Merge pull request #5617 from apache/nouveau-search-opt

Nouveau search optimization

Author: rnewson

src/nouveau/src/nouveau_rpc.erl

@@ -24,45 +24,56 @@
 -include("nouveau.hrl").
 -import(nouveau_util, [index_path/1]).
 
-search(DbName, #index{} = Index0, QueryArgs0) ->
+search(DbName, #index{} = Index, #{} = QueryArgs) ->
+    search(DbName, #index{} = Index, QueryArgs, 0).
+
+search(DbName, #index{} = Index0, QueryArgs0, UpdateLatency) ->
     %% Incorporate the shard name into the record.
     Index1 = Index0#index{dbname = DbName},
 
-    %% get minimum seqs for search
-    {MinUpdateSeq, MinPurgeSeq} = nouveau_index_updater:get_db_info(Index1),
+    Update = maps:get(update, QueryArgs0, true),
 
     %% Incorporate min seqs into the query args.
-    QueryArgs1 = QueryArgs0#{
-        min_update_seq => MinUpdateSeq,
-        min_purge_seq => MinPurgeSeq
-    },
-    Update = maps:get(update, QueryArgs1, true),
-
-    %% check if index is up to date
-    T0 = erlang:monotonic_time(),
-    case Update andalso nouveau_index_updater:outdated(Index1) of
-        true ->
-            case nouveau_index_manager:update_index(Index1) of
-                ok ->
-                    ok;
-                {error, Reason} ->
-                    rexi:reply({error, Reason})
-            end;
-        false ->
-            ok;
-        {error, Reason} ->
-            rexi:reply({error, Reason})
-    end,
-    T1 = erlang:monotonic_time(),
-    UpdateLatency = erlang:convert_time_unit(T1 - T0, native, millisecond),
+    QueryArgs1 =
+        case Update of
+            true ->
+                %% get minimum seqs for search
+                {MinUpdateSeq, MinPurgeSeq} = nouveau_index_updater:get_db_info(Index1),
+                QueryArgs0#{
+                    min_update_seq => MinUpdateSeq,
+                    min_purge_seq => MinPurgeSeq
+                };
+            false ->
+                QueryArgs0#{
+                    min_update_seq => 0,
+                    min_purge_seq => 0
+                }
+        end,
 
     %% Run the search
     case nouveau_api:search(Index1, QueryArgs1) of
         {ok, Response} ->
             rexi:reply({ok, Response#{update_latency => UpdateLatency}});
-        {error, stale_index} ->
-            %% try again.
-            search(DbName, Index0, QueryArgs0);
+        {error, stale_index} when Update ->
+            update_and_retry(DbName, Index0, QueryArgs0, UpdateLatency);
+        {error, {not_found, _}} when Update ->
+            update_and_retry(DbName, Index0, QueryArgs0, UpdateLatency);
+        Else ->
+            rexi:reply(Else)
+    end.
+
+update_and_retry(DbName, Index, QueryArgs, UpdateLatency) ->
+    T0 = erlang:monotonic_time(),
+    case nouveau_index_manager:update_index(Index#index{dbname = DbName}) of
+        ok ->
+            T1 = erlang:monotonic_time(),
+            search(
+                DbName,
+                Index,
+                QueryArgs,
+                UpdateLatency +
+                    erlang:convert_time_unit(T1 - T0, native, millisecond)
+            );
         Else ->
             rexi:reply(Else)
     end.

test/elixir/test/config/nouveau.elixir

@@ -30,6 +30,7 @@
     "purge with conflicts",
     "index same field with different field types",
     "index not found",
-    "meta"
+    "meta",
+    "stale search"
   ]
 }

test/elixir/test/nouveau_test.exs

@@ -694,6 +694,23 @@ defmodule NouveauTest do
     assert resp.body["update_latency"] > 0
   end
 
+  @tag :with_db
+  test "stale search", context do
+    db_name = context[:db_name]
+    url = "/#{db_name}/_design/foo/_nouveau/bar"
+    create_ddoc(db_name)
+
+    resp = Couch.get(url, query: %{q: "*:*", update: false, include_docs: true})
+    assert_status_code(resp, 404)
+
+    create_search_docs(db_name)
+    resp = Couch.get(url, query: %{q: "*:*", include_docs: true})
+    assert_status_code(resp, 200)
+    ids = get_ids(resp)
+    # nouveau sorts by _id as tie-breaker
+    assert ids == ["doc1", "doc2", "doc3", "doc4"]
+  end
+
   def seq(str) do
     String.to_integer(hd(Regex.run(~r/^[0-9]+/, str)))
   end