diff --git a/bookmarks/services/bookmarks.py b/bookmarks/services/bookmarks.py index 9bc9ea45..d2b05674 100644 --- a/bookmarks/services/bookmarks.py +++ b/bookmarks/services/bookmarks.py @@ -69,7 +69,7 @@ def update_bookmark(bookmark: Bookmark, tag_string, current_user: User): return bookmark -def enhance_with_website_metadata(bookmark: Bookmark): +def enhance_with_website_metadata(bookmark: Bookmark, save: bool = True): metadata = website_loader.load_website_metadata(bookmark.url) if not bookmark.title: bookmark.title = metadata.title or "" @@ -77,7 +77,8 @@ def enhance_with_website_metadata(bookmark: Bookmark): if not bookmark.description: bookmark.description = metadata.description or "" - bookmark.save() + if save: + bookmark.save() def archive_bookmark(bookmark: Bookmark): diff --git a/bookmarks/services/importer.py b/bookmarks/services/importer.py index 41fda084..6d7464de 100644 --- a/bookmarks/services/importer.py +++ b/bookmarks/services/importer.py @@ -1,6 +1,7 @@ import logging from dataclasses import dataclass from typing import List +from concurrent.futures import ThreadPoolExecutor from django.contrib.auth.models import User from django.utils import timezone @@ -9,6 +10,7 @@ from bookmarks.services import tasks from bookmarks.services.parser import parse, NetscapeBookmark from bookmarks.utils import parse_timestamp +from bookmarks.services.bookmarks import enhance_with_website_metadata logger = logging.getLogger(__name__) @@ -23,6 +25,7 @@ class ImportResult: @dataclass class ImportOptions: map_private_flag: bool = False + get_metadata_flag: bool = False class TagCache: @@ -132,8 +135,8 @@ def _import_batch( existing_bookmarks = Bookmark.objects.filter(owner=user, url__in=batch_urls) # Create or update bookmarks from parsed Netscape bookmarks - bookmarks_to_create = [] - bookmarks_to_update = [] + bookmarks_to_create: List[Bookmark] = [] + bookmarks_to_update: List[Bookmark] = [] for netscape_bookmark in netscape_bookmarks: result.total = result.total + 1 @@ -169,6 +172,18 @@ def _import_batch( logging.exception("Error importing bookmark: " + shortened_bookmark_tag_str) result.failed = result.failed + 1 + if options.get_metadata_flag: + bookmarks_missing_metadata = filter( + lambda b: not b.title or not b.description, + bookmarks_to_create + bookmarks_to_update, + ) + + with ThreadPoolExecutor(5) as executor: + executor.map( + lambda b: enhance_with_website_metadata(b, save=False), + bookmarks_missing_metadata, + ) + # Bulk update bookmarks in DB Bookmark.objects.bulk_update( bookmarks_to_update, diff --git a/bookmarks/templates/settings/general.html b/bookmarks/templates/settings/general.html index dc6f0702..130223bb 100644 --- a/bookmarks/templates/settings/general.html +++ b/bookmarks/templates/settings/general.html @@ -325,6 +325,13 @@