diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e72b57a..b0518f9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,10 +2,12 @@ Changelog ========= -0.4.0 (unreleased) +0.3.1 (unreleased) ------------------ -- Nothing changed yet. +**Bug fixes** + +- Fix the reindex get_paginated_records function. (fixes #61) 0.3.0 (2017-09-12) diff --git a/kinto_elasticsearch/command_reindex.py b/kinto_elasticsearch/command_reindex.py index 9f33fa5..a193264 100644 --- a/kinto_elasticsearch/command_reindex.py +++ b/kinto_elasticsearch/command_reindex.py @@ -96,11 +96,12 @@ def get_paginated_records(storage, bucket_id, collection_id, limit=5000): pagination_rules=pagination_rules, sorting=sorting, limit=limit) - if len(records) == 0: - break # Done. yield records + if len(records) < limit: + break # Done. + smallest_timestamp = records[-1]["last_modified"] pagination_rules = [ [Filter("last_modified", smallest_timestamp, COMPARISON.LT)] diff --git a/tests/test_cli.py b/tests/test_cli.py index be29818..baee656 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,7 +2,7 @@ import mock import os import unittest -from kinto_elasticsearch.command_reindex import main, reindex_records +from kinto_elasticsearch.command_reindex import main, reindex_records, get_paginated_records from . import BaseWebTest HERE = os.path.abspath(os.path.dirname(__file__)) @@ -83,3 +83,17 @@ def test_cli_default_to_sys_argv(self): with mock.patch('sys.argv', ['cli', '--ini', os.path.join(HERE, 'wrong_config.ini')]): exit_code = main() assert exit_code == 62 + + def test_get_paginated_records(self): + # Create collection or bucket + self.app.put("/buckets/bid", headers=self.headers) + body = {"data": {"index:schema": self.schema}} + self.app.put_json("/buckets/bid/collections/cid", body, headers=self.headers) + for i in range(5): + self.app.post_json("/buckets/bid/collections/cid/records", + {"data": {"build": {"id": "efg%d" % i, "date": "2017-02-01"}}}, + headers=self.headers) + page_count = 0 + for records in get_paginated_records(self.app.app.registry.storage, 'bid', 'cid', limit=3): + page_count += 1 + assert page_count == 2