completion_search.js

// This is a wrapper class for completion engines. It handles the case where a custom search engine
// includes a prefix query term (or terms). For example:
//
//   https://www.google.com/search?q=javascript+%s
//
// In this case, we get better suggestions if we include the term "javascript" in queries sent to
// the completion engine. This wrapper handles adding such prefixes to completion-engine queries and
// removing them from the resulting suggestions.
class EnginePrefixWrapper {
  constructor(searchUrl, engine) {
    this.searchUrl = searchUrl;
    this.engine = engine;
  }

  getUrl(queryTerms) {
    // This tests whether @searchUrl contains something of the form "...=abc+def+%s...", from which
    // we extract a prefix of the form "abc def ".
    if (/\=.+\+%s/.test(this.searchUrl)) {
      let terms = this.searchUrl.replace(/\+%s.*/, "");
      terms = terms.replace(/.*=/, "");
      terms = terms.replace(/\+/g, " ");

      queryTerms = [...terms.split(" "), ...queryTerms];
      const prefix = `${terms} `;

      this.postprocessSuggestions = (suggestions) => {
        return suggestions
          .filter((s) => s.startsWith(prefix))
          .map((s) => s.slice(prefix.length));
      };
    }

    return this.engine.getUrl(queryTerms);
  }

  parse(responseText) {
    return this.postprocessSuggestions(this.engine.parse(responseText));
  }

  postprocessSuggestions(suggestions) {
    return suggestions;
  }
}

const CompletionSearch = {
  debug: false,
  inTransit: {},
  completionCache: undefined,
  engineCache: undefined,

  clearCache() {
    this.completionCache = new SimpleCache(2 * 60 * 60 * 1000, 5000); // Two hours, 5000 entries.
    this.engineCache = new SimpleCache(1000 * 60 * 60 * 1000); // 1000 hours.
  },

  // The amount of time to wait for new requests before launching the current request (for example,
  // if the user is still typing).
  delay: 100,

  async get(searchUrl, url, callback) {
    const timeoutDuration = 2500;
    const controller = new AbortController();
    let isError = false;
    let responseText;
    const timer = Utils.setTimeout(timeoutDuration, () => controller.abort());

    try {
      const response = await fetch(url, { signal: controller.signal });
      responseText = await response.text();
    } catch (error) {
      // Fetch throws an error if the network is unreachable, etc.
      isError = true;
    }

    clearTimeout(timer);

    if (isError) {
      callback(null);
    } else {
      callback(responseText);
    }
  },

  // Look up the completion engine for this searchUrl. Because of DummyCompletionEngine, we know
  // there will always be a match.
  lookupEngine(searchUrl) {
    if (this.engineCache.has(searchUrl)) {
      return this.engineCache.get(searchUrl);
    } else {
      for (let engine of Array.from(CompletionEngines)) {
        engine = new engine();
        if (engine.match(searchUrl)) {
          return this.engineCache.set(searchUrl, engine);
        }
      }
    }
  },

  // True if we have a completion engine for this search URL, false otherwise.
  haveCompletionEngine(searchUrl) {
    return !this.lookupEngine(searchUrl).dummy;
  },

  // This is the main entry point.
  //  - searchUrl is the search engine's URL, e.g. Settings.get("searchUrl"), or a custom search
  //    engine's URL. This is only used as a key for determining the relevant completion engine.
  //  - queryTerms are the query terms.
  //  - callback will be applied to a list of suggestion strings (which may be an empty list, if
  //    anything goes wrong).
  //
  // If no callback is provided, then we're to provide suggestions only if we can do so
  // synchronously (ie. from a cache). In this case we just return the results. Returns null if we
  // cannot service the request synchronously.
  //
  // TODO(philc): This function is large and nestedand could use refactoring.
  complete(searchUrl, queryTerms, callback = null) {
    let handler;
    const query = queryTerms.join(" ").toLowerCase();

    const returnResultsOnlyFromCache = callback == null;
    if (callback == null) {
      callback = (suggestions) => suggestions;
    }

    // We don't complete queries which are too short: the results are usually useless.
    if (query.length < 4) {
      return callback([]);
    }

    // We don't complete regular URLs or Javascript URLs.
    if (queryTerms.length == 1 && Utils.isUrl(query)) {
      return callback([]);
    }
    if (Utils.hasJavascriptPrefix(query)) {
      return callback([]);
    }

    const completionCacheKey = JSON.stringify([searchUrl, queryTerms]);
    if (this.completionCache.has(completionCacheKey)) {
      if (this.debug) {
        console.log("hit", completionCacheKey);
      }
      return callback(this.completionCache.get(completionCacheKey));
    }

    // If the user appears to be typing a continuation of the characters of the most recent query,
    // then we can sometimes re-use the previous suggestions.
    if (
      (this.mostRecentQuery != null) && (this.mostRecentSuggestions != null) &&
      (this.mostRecentSearchUrl != null)
    ) {
      if (searchUrl === this.mostRecentSearchUrl) {
        const reusePreviousSuggestions = (() => {
          // Verify that the previous query is a prefix of the current query.
          if (!query.startsWith(this.mostRecentQuery.toLowerCase())) {
            return false;
          }
          // Verify that every previous suggestion contains the text of the new query.
          // Note: @mostRecentSuggestions may also be empty, in which case we drop though. The
          // effect is that previous queries with no suggestions suppress subsequent no-hope HTTP
          // requests as the user continues to type.
          for (const suggestion of this.mostRecentSuggestions) {
            if (!suggestion.includes(query)) {
              return false;
            }
          }
          // Ok. Re-use the suggestion.
          return true;
        })();

        if (reusePreviousSuggestions) {
          if (this.debug) {
            console.log(
              "reuse previous query:",
              this.mostRecentQuery,
              this.mostRecentSuggestions.length,
            );
          }
          return callback(this.completionCache.set(completionCacheKey, this.mostRecentSuggestions));
        }
      }
    }

    // That's all of the caches we can try. Bail if the caller is only requesting synchronous
    // results. We signal that we haven't found a match by returning null.
    if (returnResultsOnlyFromCache) {
      return callback(null);
    }

    // We pause in case the user is still typing.
    Utils.setTimeout(
      this.delay,
      handler = this.mostRecentHandler = () => {
        if (handler !== this.mostRecentHandler) {
          return;
        }
        this.mostRecentHandler = null;

        // Elide duplicate requests. First fetch the suggestions...
        if (this.inTransit[completionCacheKey] == null) {
          this.inTransit[completionCacheKey] = new AsyncDataFetcher(async (callback) => {
            const engine = new EnginePrefixWrapper(searchUrl, this.lookupEngine(searchUrl));
            const url = engine.getUrl(queryTerms);

            await this.get(searchUrl, url, (responseText) => {
              // Parsing the response may fail if we receive an unexpected or an
              // unexpectedly-formatted response. In all cases, we fall back to the catch clause,
              // below. Therefore, we "fail safe" in the case of incorrect or out-of-date completion
              // engines.
              let suggestions;
              try {
                suggestions = engine.parse(responseText)
                  // Make all suggestions lower case. It looks odd when suggestions from one
                  // completion engine are upper case, and those from another are lower case.
                  .map((s) => s.toLowerCase())
                  // Filter out the query itself. It's not adding anything.
                  .filter((s) => s !== query);
                if (this.debug) {
                  console.log("GET", url);
                }
              } catch (error) {
                suggestions = [];
                // We allow failures to be cached too, but remove them after just thirty seconds.
                Utils.setTimeout(
                  30 * 1000,
                  () => this.completionCache.set(completionCacheKey, null),
                );
                if (this.debug) {
                  console.error("completion failed:", url);
                  console.error(error);
                }
              }

              callback(suggestions);
              delete this.inTransit[completionCacheKey];
            });
          });
        }

        // ... then use the suggestions.
        this.inTransit[completionCacheKey].use((suggestions) => {
          this.mostRecentSearchUrl = searchUrl;
          this.mostRecentQuery = query;
          this.mostRecentSuggestions = suggestions;
          // TODO(philc): Is this return necessary?
          return callback(this.completionCache.set(completionCacheKey, suggestions));
        });
      },
    );
  },

  // Cancel any pending (ie. blocked on @delay) queries. Does not cancel in-flight queries. This is
  // called whenever the user is typing.
  cancel() {
    if (this.mostRecentHandler != null) {
      this.mostRecentHandler = null;
      if (this.debug) {
        console.log("cancel (user is typing)");
      }
    }
  },
};

CompletionSearch.clearCache();
globalThis.CompletionSearch = CompletionSearch;