Added support for a custom list of default trackers (#8446)

Author: qstokkink

src/tribler/core/libtorrent/restapi/downloads_endpoint.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
+import logging
 import mimetypes
 from asyncio import get_event_loop, shield
 from binascii import hexlify, unhexlify
+from functools import lru_cache
 from pathlib import Path, PurePosixPath
+from time import time
 from typing import TYPE_CHECKING, Any, Optional, TypedDict, cast
 
 import libtorrent as lt
@@ -39,6 +42,7 @@
 TOTAL = "total"
 LOADED = "loaded"
 ALL_LOADED = "all_loaded"
+logger = logging.getLogger(__name__)
 
 
 class JSONFilesInfo(TypedDict):
@@ -53,6 +57,21 @@ class JSONFilesInfo(TypedDict):
     progress: float
 
 
+@lru_cache(maxsize=1)
+def cached_read(tracker_file: str, _: int) -> list[bytes]:
+    """
+    Keep one cache for one tracker file at a time (by default: for a max of 120 seconds, see caller).
+
+    When adding X torrents at once, this avoids reading the same file X times.
+    """
+    try:
+        with open(tracker_file, "rb") as f:
+            return [line.rstrip() for line in f if line.rstrip()]  # uTorrent format contains blank lines between URLs
+    except OSError:
+        logger.exception("Failed to read tracker file!")
+        return []
+
+
 class DownloadsEndpoint(RESTEndpoint):
     """
     This endpoint is responsible for all requests regarding downloads. Examples include getting all downloads,
@@ -359,6 +378,17 @@ async def get_downloads(self, request: Request) -> RESTResponse:  # noqa: C901
             result.append(info)
         return RESTResponse({"downloads": result, "checkpoints": checkpoints})
 
+    def _get_default_trackers(self) -> list[bytes]:
+        """
+        Get the default trackers from the configured tracker file.
+
+        Tracker file format is "(<TRACKER><NEWLINE><NEWLINE>)*". We assume "<TRACKER>" does not include newlines.
+        """
+        tracker_file = self.download_manager.config.get("libtorrent/download_defaults/trackers_file")
+        if not tracker_file:
+            return []
+        return cached_read(tracker_file, int(time())//120)
+
     @docs(
         tags=["Libtorrent"],
         summary="Start a download from a provided URI.",
@@ -397,7 +427,7 @@ async def get_downloads(self, request: Request) -> RESTResponse:  # noqa: C901
         "uri*": (String, "The URI of the torrent file that should be downloaded. This URI can either represent a file "
                          "location, a magnet link or a HTTP(S) url."),
     }))
-    async def add_download(self, request: Request) -> RESTResponse:  # noqa: C901
+    async def add_download(self, request: Request) -> RESTResponse:  # noqa: C901, PLR0912
         """
         Start a download from a provided URI.
         """
@@ -436,8 +466,11 @@ async def add_download(self, request: Request) -> RESTResponse:  # noqa: C901
         try:
             if tdef:
                 download = await self.download_manager.start_download(tdef=tdef, config=download_config)
-            elif uri:
+            else:  # guaranteed to have uri
                 download = await self.download_manager.start_download_from_uri(uri, config=download_config)
+            if self.download_manager.config.get("libtorrent/download_defaults/trackers_file"):
+                await download.get_handle()  # We can only add trackers to a valid handle, wait for it.
+                download.add_trackers(self._get_default_trackers())
         except Exception as e:
             return RESTResponse({"error": {
                                     "handled": True,

src/tribler/test_unit/core/libtorrent/restapi/test_downloads_endpoint.py

@@ -5,7 +5,7 @@
 
 from io import StringIO
 from pathlib import Path
-from unittest.mock import AsyncMock, Mock, call, patch
+from unittest.mock import AsyncMock, Mock, call, mock_open, patch
 
 from aiohttp.web_urldispatcher import UrlMappingMatchInfo
 from configobj import ConfigObj
@@ -309,6 +309,48 @@ async def test_add_download_failed(self) -> None:
         self.assertEqual(HTTP_INTERNAL_SERVER_ERROR, response.status)
         self.assertEqual("invalid uri", response_body_json["error"]["message"])
 
+    async def test_add_download_with_default_trackers_bad_file(self) -> None:
+        """
+        Test if a bad trackers file is simply ignored.
+        """
+        download = self.create_mock_download()
+        download.get_handle = AsyncMock(return_value=None)
+        self.download_manager.start_download_from_uri = AsyncMock(return_value=download)
+        self.download_manager.config = MockTriblerConfigManager()
+        self.download_manager.config.set("libtorrent/download_defaults/trackers_file", "testpath.txt")
+        request = MockRequest("/api/downloads", "PUT", {"uri": "http://127.0.0.1/file"})
+
+        with patch("tribler.core.libtorrent.download_manager.download_config.DownloadConfig.from_defaults",
+                   lambda _: download.config):
+            response = await self.endpoint.add_download(request)
+        response_body_json = await response_to_json(response)
+
+        self.assertEqual(200, response.status)
+        self.assertTrue(response_body_json["started"])
+
+    async def test_add_download_with_default_trackers(self) -> None:
+        """
+        Test if the default trackers are added when adding a download.
+        """
+        download = self.create_mock_download()
+        download.handle = Mock(is_valid=Mock(return_value=True))
+        download.get_handle = AsyncMock(return_value=download.handle)
+        self.download_manager.start_download_from_uri = AsyncMock(return_value=download)
+        self.download_manager.config = MockTriblerConfigManager()
+        self.download_manager.config.set("libtorrent/download_defaults/trackers_file", "testpath.txt")
+        request = MockRequest("/api/downloads", "PUT", {"uri": "http://127.0.0.1/file"})
+
+        with patch("tribler.core.libtorrent.download_manager.download_config.DownloadConfig.from_defaults",
+                   lambda _: download.config), patch("builtins.open", mock_open(read_data=b"http://1\n\nudp://2/\n\n")):
+            response = await self.endpoint.add_download(request)
+        response_body_json = await response_to_json(response)
+
+        self.assertEqual(200, response.status)
+        self.assertTrue(response_body_json["started"])
+        self.assertListEqual([call({"url": b"http://1", "verified": False}),
+                              call({"url": b"udp://2/", "verified": False})],
+                             download.handle.add_tracker.call_args_list)
+
     async def test_delete_download_no_remove_data(self) -> None:
         """
         Test if a graceful error is returned when no remove data is supplied when deleting a download.

src/tribler/tribler_config.py

@@ -77,6 +77,7 @@ class DownloadDefaultsConfig(TypedDict):
     seeding_time: float
     channel_download: bool
     add_download_to_channel: bool
+    trackers_file: str
 
 
 class LibtorrentConfig(TypedDict):
@@ -231,7 +232,8 @@ class TriblerConfig(TypedDict):
             seeding_ratio=2.0,
             seeding_time=60.0,
             channel_download=False,
-            add_download_to_channel=False)
+            add_download_to_channel=False,
+            trackers_file="")
         ),
     "recommender": RecommenderConfig(enabled=True),
     "rendezvous": RendezvousConfig(enabled=True),

src/tribler/ui/public/locales/en_US.json

@@ -22,6 +22,7 @@
     "CreateTorrent": "Create torrent from file(s)",
     "DefaultDownloadSettings": "Default download settings",
     "SaveFilesTo": "Save files to:",
+    "DefaultTrackersFile": "Default trackers file:",
     "AlwaysAsk": "Always ask download settings",
     "DownloadAnon": "Download anonymously using proxies",
     "SeedAnon": "Encrypted anonymous seeding using proxies",

src/tribler/ui/public/locales/es_ES.json

@@ -22,6 +22,7 @@
     "CreateTorrent": "Crear torrent a partir de archivo(s)",
     "DefaultDownloadSettings": "Configuración de descargas por defecto",
     "SaveFilesTo": "Guardar archivos en:",
+    "DefaultTrackersFile": "Archivo de rastreadores predeterminados:",
     "AlwaysAsk": "¿Preguntar siempre donde guardar las descargas?",
     "DownloadAnon": "Descarga anónima mediante proxies",
     "SeedAnon": "Siembra anónima cifrada mediante proxies",

src/tribler/ui/public/locales/ko_KR.json

@@ -22,6 +22,7 @@
     "CreateTorrent": "파일에서 토렌트 만들기",
     "DefaultDownloadSettings": "기본 내려받기 설정",
     "SaveFilesTo": "파일을 저장할 위치 :",
+    "DefaultTrackersFile": "기본 추적기 파일 :",
     "AlwaysAsk": "항상 내려받기 설정 확인",
     "DownloadAnon": "프록시를 사용하여 익명으로 내려받기",
     "SeedAnon": "프록시를 사용하여 암호화된 익명 시딩",

src/tribler/ui/public/locales/pt_BR.json

@@ -22,6 +22,7 @@
     "CreateTorrent": "Criar torrent de um arquivo(s)",
     "DefaultDownloadSettings": "Configuração de download padrão",
     "SaveFilesTo": "Salvar arquivos em:",
+    "DefaultTrackersFile": "Arquivo de rastreadores padrão:",
     "AlwaysAsk": "Sempre peça configurações de download",
     "DownloadAnon": "Baixe anonimamente usando proxies",
     "SeedAnon": "Semear anonimamente usando proxies",

src/tribler/ui/public/locales/ru_RU.json

@@ -22,6 +22,7 @@
     "CreateTorrent": "Создать торрент",
     "DefaultDownloadSettings": "Настройки закачки по умолчанию",
     "SaveFilesTo": "Сохранять файлы в:",
+    "DefaultTrackersFile": "Файл трекеров по умолчанию:",
     "AlwaysAsk": "Всегда спрашивать настройки закачки",
     "DownloadAnon": "Загружать торренты анонимно (через других участников)",
     "SeedAnon": "Раздавать торренты анонимно (через других участников)",

src/tribler/ui/public/locales/zh_CN.json

@@ -22,6 +22,7 @@
     "CreateTorrent": "从文件创建种子",
     "DefaultDownloadSettings": "默认下载设置",
     "SaveFilesTo": "保存文件到：",
+    "DefaultTrackersFile": "默认跟踪器文件：",
     "AlwaysAsk": "总是询问下载设置",
     "DownloadAnon": "使用代理匿名下载",
     "SeedAnon": "使用代理加密地匿名做种",

src/tribler/ui/src/models/settings.model.tsx

@@ -78,6 +78,7 @@ export interface Settings {
             seeding_time: number;
             channel_download: boolean;
             add_download_to_channel: boolean;
+            trackers_file: string;
         },
     },
     rendezvous: {

src/tribler/ui/src/pages/Settings/General.tsx

@@ -84,6 +84,29 @@ export default function General() {
                     }}
                 />
             </div>
+            <div className="py-2 flex items-center">
+                <Label htmlFor="trackers_file" className="whitespace-nowrap pr-5">
+                    {t('DefaultTrackersFile')}
+                </Label>
+                <PathInput
+                    path={settings?.libtorrent?.download_defaults?.trackers_file}
+                    directory={false}
+                    onPathChange={(path) => {
+                        if (settings) {
+                            setSettings({
+                                ...settings,
+                                libtorrent: {
+                                    ...settings.libtorrent,
+                                    download_defaults: {
+                                        ...settings.libtorrent.download_defaults,
+                                        trackers_file: path
+                                    }
+                                }
+                            });
+                        }
+                    }}
+                />
+            </div>
             <div className="flex items-center">
                 <div className="w-64 flex items-center">
                     <Checkbox