From 199796088b8d826aad3488b54f384fbb706e4ba9 Mon Sep 17 00:00:00 2001 From: Riley Carter Date: Fri, 21 Jul 2023 07:23:53 -0400 Subject: [PATCH 1/2] Completions: Move standard parse() to BaseEngine --- background_scripts/completion_engines.js | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/background_scripts/completion_engines.js b/background_scripts/completion_engines.js index e2d2322f9..c72e9a6fa 100644 --- a/background_scripts/completion_engines.js +++ b/background_scripts/completion_engines.js @@ -27,7 +27,7 @@ // A base class for common regexp-based matching engines. "options" must define: // options.engineUrl: the URL to use for the completion engine. This must be a string. -// options.regexps: one or regular expressions. This may either a single string or a list of strings. +// options.regexps: Regexes matching search URLs this engine handles. List of strings or RegExps. // options.example: an example object containing at least "keyword" and "searchUrl", and optional "description". // TODO(philc): This base class is doing very little. We should remove it and use composition. class BaseEngine { @@ -42,6 +42,9 @@ class BaseEngine { getUrl(queryTerms) { return Utils.createSearchUrl(queryTerms, this.engineUrl); } + parse(text) { + return JSON.parse(text)[1]; + } } class Google extends BaseEngine { @@ -55,10 +58,6 @@ class Google extends BaseEngine { }, }); } - - parse(text) { - return JSON.parse(text)[1]; - } } const googleMapsPrefix = "map of "; @@ -99,10 +98,6 @@ class Youtube extends BaseEngine { }, }); } - - parse(text) { - return JSON.parse(text)[1]; - } } class Wikipedia extends BaseEngine { @@ -116,10 +111,6 @@ class Wikipedia extends BaseEngine { }, }); } - - parse(text) { - return JSON.parse(text)[1]; - } } class Bing extends BaseEngine { @@ -133,10 +124,6 @@ class Bing extends BaseEngine { }, }); } - - parse(text) { - return JSON.parse(text)[1]; - } } class Amazon extends BaseEngine { From ff67283a4423d583b0ec92ea217606a0b2c90b24 Mon Sep 17 00:00:00 2001 From: Riley Carter Date: Fri, 21 Jul 2023 07:32:05 -0400 Subject: [PATCH 2/2] Completions: Allow adding completion providers in settings --- background_scripts/bg_utils.js | 41 ++++++++++++++++++++++++ background_scripts/completion_engines.js | 4 +-- background_scripts/completion_search.js | 13 ++++++-- lib/settings.js | 11 ++++++- 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/background_scripts/bg_utils.js b/background_scripts/bg_utils.js index 46a331740..81a69e171 100644 --- a/background_scripts/bg_utils.js +++ b/background_scripts/bg_utils.js @@ -1,3 +1,5 @@ +import {BaseEngine} from "./completion_engines.js"; + const TIME_DELTA = 500; // Milliseconds. // TabRecency associates a logical timestamp with each tab id. These are used to provide an initial @@ -124,12 +126,51 @@ const SearchEngines = { if ((this.previousSearchEngines == null) || (searchEngines !== this.previousSearchEngines)) { this.previousSearchEngines = searchEngines; this.searchEngines = new AsyncDataFetcher(function (callback) { + CompletionSearch.clearCache(); const engines = {}; + CompletionEngines = [...BuiltinCompletionEngines]; for (let line of BgUtils.parseLines(searchEngines)) { const tokens = line.split(/\s+/); if (2 <= tokens.length) { const keyword = tokens[0].split(":")[0]; const searchUrl = tokens[1]; + // Build and register a new completion engine using the @URL + if (3 <= tokens.length && tokens[2].startsWith("@")) { + const [_, engineUrl, jsonPath] = tokens[2].split("@"); + const engineRegexp = "^" + searchUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + class NewCompletionEngine extends BaseEngine { + constructor() { + super({ + engineUrl: engineUrl, + regexps: [engineRegexp], + example: { + searchUrl: searchUrl, + keyword: keyword, + }, + }); + } + parse(text) { + if (!jsonPath) return super.parse(text); + let data = JSON.parse(text); + let star = false; + for (const key of jsonPath.split(".")) { + if (key === "*") { + // TODO when ES2019: Replace with data=data.flat(1) + if (star) data = [].concat.apply([], data); + star = true; + } else if (star && data instanceof Array) { + star = false; + data = data.map(val => val[key]); + } else { + data = data[key]; + } + } + return data; + } + } + CompletionEngines.unshift(NewCompletionEngine); + tokens.shift(); + } const description = tokens.slice(2).join(" ") || `search (${keyword})`; if (Utils.hasFullUrlPrefix(searchUrl) || Utils.hasJavascriptPrefix(searchUrl)) { engines[keyword] = { keyword, searchUrl, description }; diff --git a/background_scripts/completion_engines.js b/background_scripts/completion_engines.js index c72e9a6fa..7a7fd0ecd 100644 --- a/background_scripts/completion_engines.js +++ b/background_scripts/completion_engines.js @@ -224,6 +224,6 @@ const CompletionEngines = [ DummyCompletionEngine, ]; -globalThis.CompletionEngines = CompletionEngines; +globalThis.CompletionEngines = globalThis.BuiltinCompletionEngines = CompletionEngines; -export { Amazon, DuckDuckGo, Qwant, Webster }; +export { Amazon, BaseEngine, DuckDuckGo, Qwant, Webster }; diff --git a/background_scripts/completion_search.js b/background_scripts/completion_search.js index fb653f672..5bed19d48 100644 --- a/background_scripts/completion_search.js +++ b/background_scripts/completion_search.js @@ -45,8 +45,13 @@ class EnginePrefixWrapper { const CompletionSearch = { debug: false, inTransit: {}, - completionCache: new SimpleCache(2 * 60 * 60 * 1000, 5000), // Two hours, 5000 entries. - engineCache: new SimpleCache(1000 * 60 * 60 * 1000), // 1000 hours. + completionCache: undefined, + engineCache: undefined, + + clearCache() { + this.completionCache = new SimpleCache(2 * 60 * 60 * 1000, 5000); // Two hours, 5000 entries. + this.engineCache = new SimpleCache(1000 * 60 * 60 * 1000); // 1000 hours. + }, // The amount of time to wait for new requests before launching the current request (for example, // if the user is still typing). @@ -221,7 +226,8 @@ const CompletionSearch = { () => this.completionCache.set(completionCacheKey, null), ); if (this.debug) { - console.log("fail", url); + console.error("completion failed:", url); + console.error(error); } } @@ -255,4 +261,5 @@ const CompletionSearch = { }, }; +CompletionSearch.clearCache(); globalThis.CompletionSearch = CompletionSearch; diff --git a/lib/settings.js b/lib/settings.js index cf494bcf3..8ffda002e 100644 --- a/lib/settings.js +++ b/lib/settings.js @@ -51,7 +51,7 @@ w: https://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedi # More examples. # -# (Vimium supports search completion Wikipedia, as +# (Vimium supports search completion for Wikipedia, as # above, and for these.) # # g: https://www.google.com/search?q=%s Google @@ -62,6 +62,15 @@ w: https://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedi # d: https://duckduckgo.com/?q=%s DuckDuckGo # az: https://www.amazon.com/s/?field-keywords=%s # qw: https://www.qwant.com/?q=%s Qwant\ +# +# You can also configure a custom search completion provider. +# shortcut: search-url?search=%s @autocomplete-url?search=%s Description +# By default data[1] is used, but you can also set a path: +# shortcut: search-url?search=%s @autocomplete-url?search=%s@path.2.*.list Description +# Examples: +# rs: https://runescape.wiki/?search=%s @https://runescape.wiki/api.php?action=opensearch&format=json&search=%s Runescape Wiki +# os: https://oldschool.runescape.wiki/?search=%s @https://oldschool.runescape.wiki/api.php?action=opensearch&format=json&search=%s Old School Runescape wiki +# man: https://www.mankier.com/?q=%s @https://www.mankier.com/api/v2/suggest/?q=%s@results.*.text ManKier Man Pages `, newTabUrl: "about:newtab", // Equal to the value of Utils.chromeNewTabUrl. grabBackFocus: false,