account for > 10,000 hits

Author: pedro-cf

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py

@@ -574,15 +574,32 @@ async def execute_search(
             )
         )
 
+        count_task = asyncio.create_task(
+            self.client.count(
+                index=index_param,
+                ignore_unavailable=ignore_unavailable,
+                body=search.to_dict(count=True),
+            )
+        )
+
         try:
             es_response = await search_task
         except exceptions.NotFoundError:
             raise NotFoundError(f"Collections '{collection_ids}' do not exist")
 
-        matched = es_response["hits"]["total"]["value"]
         hits = es_response["hits"]["hits"]
         items = (hit["_source"] for hit in hits)
 
+        matched = es_response["hits"]["total"]["value"]
+        if es_response["hits"]["total"]["relation"] != "eq":
+            if count_task.done():
+                try:
+                    matched = count_task.result().get("count")
+                except Exception as e:
+                    logger.error(f"Count task failed: {e}")
+        else:
+            count_task.cancel()
+
         next_token = None
         if matched > page * limit:
             if hits and (sort_array := hits[-1].get("sort")):

stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py

@@ -605,14 +605,31 @@ async def execute_search(
             )
         )
 
+        count_task = asyncio.create_task(
+            self.client.count(
+                index=index_param,
+                ignore_unavailable=ignore_unavailable,
+                body=search.to_dict(count=True),
+            )
+        )
+
         try:
             es_response = await search_task
         except exceptions.NotFoundError:
             raise NotFoundError(f"Collections '{collection_ids}' do not exist")
 
         hits = es_response["hits"]["hits"]
         items = (hit["_source"] for hit in hits)
+
         matched = es_response["hits"]["total"]["value"]
+        if es_response["hits"]["total"]["relation"] != "eq":
+            if count_task.done():
+                try:
+                    matched = count_task.result().get("count")
+                except Exception as e:
+                    logger.error(f"Count task failed: {e}")
+        else:
+            count_task.cancel()
 
         next_token = None
         if matched > page * limit: