Merge pull request #61 from Kinto/fix-reindex-command

Stop as soon as we reached last page.

CHANGELOG.rst

@@ -2,10 +2,12 @@ Changelog
 =========
 
 
-0.4.0 (unreleased)
+0.3.1 (unreleased)
 ------------------
 
-- Nothing changed yet.
+**Bug fixes**
+
+- Fix the reindex get_paginated_records function. (fixes #61)
 
 
 0.3.0 (2017-09-12)

kinto_elasticsearch/command_reindex.py

@@ -96,11 +96,12 @@ def get_paginated_records(storage, bucket_id, collection_id, limit=5000):
                                      pagination_rules=pagination_rules,
                                      sorting=sorting,
                                      limit=limit)
-        if len(records) == 0:
-            break  # Done.
 
         yield records
 
+        if len(records) < limit:
+            break  # Done.
+
         smallest_timestamp = records[-1]["last_modified"]
         pagination_rules = [
             [Filter("last_modified", smallest_timestamp, COMPARISON.LT)]

tests/test_cli.py

@@ -2,7 +2,7 @@
 import mock
 import os
 import unittest
-from kinto_elasticsearch.command_reindex import main, reindex_records
+from kinto_elasticsearch.command_reindex import main, reindex_records, get_paginated_records
 from . import BaseWebTest
 
 HERE = os.path.abspath(os.path.dirname(__file__))
@@ -83,3 +83,17 @@ def test_cli_default_to_sys_argv(self):
         with mock.patch('sys.argv', ['cli', '--ini', os.path.join(HERE, 'wrong_config.ini')]):
             exit_code = main()
             assert exit_code == 62
+
+    def test_get_paginated_records(self):
+        # Create collection or bucket
+        self.app.put("/buckets/bid", headers=self.headers)
+        body = {"data": {"index:schema": self.schema}}
+        self.app.put_json("/buckets/bid/collections/cid", body, headers=self.headers)
+        for i in range(5):
+            self.app.post_json("/buckets/bid/collections/cid/records",
+                               {"data": {"build": {"id": "efg%d" % i, "date": "2017-02-01"}}},
+                               headers=self.headers)
+        page_count = 0
+        for records in get_paginated_records(self.app.app.registry.storage, 'bid', 'cid', limit=3):
+            page_count += 1
+        assert page_count == 2