From a9cceefc0b5f62c69a93a98b9fd1fabdcba55e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20HUBSCHER?= Date: Tue, 10 Apr 2018 11:42:40 +0200 Subject: [PATCH 1/3] Stop as soon as we reached last page. --- kinto_elasticsearch/command_reindex.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kinto_elasticsearch/command_reindex.py b/kinto_elasticsearch/command_reindex.py index 9f33fa5..a193264 100644 --- a/kinto_elasticsearch/command_reindex.py +++ b/kinto_elasticsearch/command_reindex.py @@ -96,11 +96,12 @@ def get_paginated_records(storage, bucket_id, collection_id, limit=5000): pagination_rules=pagination_rules, sorting=sorting, limit=limit) - if len(records) == 0: - break # Done. yield records + if len(records) < limit: + break # Done. + smallest_timestamp = records[-1]["last_modified"] pagination_rules = [ [Filter("last_modified", smallest_timestamp, COMPARISON.LT)] From fa02ffdc4a8b7c6f10509592738d25a4c96c80bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20HUBSCHER?= Date: Tue, 10 Apr 2018 17:26:31 +0200 Subject: [PATCH 2/3] Add a test. --- tests/test_cli.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index be29818..2089355 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,7 +2,7 @@ import mock import os import unittest -from kinto_elasticsearch.command_reindex import main, reindex_records +from kinto_elasticsearch.command_reindex import main, reindex_records, get_paginated_records from . import BaseWebTest HERE = os.path.abspath(os.path.dirname(__file__)) @@ -83,3 +83,17 @@ def test_cli_default_to_sys_argv(self): with mock.patch('sys.argv', ['cli', '--ini', os.path.join(HERE, 'wrong_config.ini')]): exit_code = main() assert exit_code == 62 + + def test_get_paginated_records(self): + # Create collection or bucket + self.app.put("/buckets/bid", headers=self.headers) + body = {"data": {"index:settings": self.schema}} + self.app.put_json("/buckets/bid/collections/cid", body, headers=self.headers) + for i in range(5): + self.app.post_json("/buckets/bid/collections/cid/records", + {"data": {"build": {"id": "efg%d" % i, "date": "2017-02-01"}}}, + headers=self.headers) + count = 0 + for records in get_paginated_records(self.app.app.registry.storage, 'bid', 'cid', limit=3): + count += 1 + assert count == 2 From 3269de35b5349f88a3001db0751b689cc22eada3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20HUBSCHER?= Date: Tue, 10 Apr 2018 17:38:40 +0200 Subject: [PATCH 3/3] @leplatrem review. --- CHANGELOG.rst | 6 ++++-- tests/test_cli.py | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e72b57a..b0518f9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,10 +2,12 @@ Changelog ========= -0.4.0 (unreleased) +0.3.1 (unreleased) ------------------ -- Nothing changed yet. +**Bug fixes** + +- Fix the reindex get_paginated_records function. (fixes #61) 0.3.0 (2017-09-12) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2089355..baee656 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -87,13 +87,13 @@ def test_cli_default_to_sys_argv(self): def test_get_paginated_records(self): # Create collection or bucket self.app.put("/buckets/bid", headers=self.headers) - body = {"data": {"index:settings": self.schema}} + body = {"data": {"index:schema": self.schema}} self.app.put_json("/buckets/bid/collections/cid", body, headers=self.headers) for i in range(5): self.app.post_json("/buckets/bid/collections/cid/records", {"data": {"build": {"id": "efg%d" % i, "date": "2017-02-01"}}}, headers=self.headers) - count = 0 + page_count = 0 for records in get_paginated_records(self.app.app.registry.storage, 'bid', 'cid', limit=3): - count += 1 - assert count == 2 + page_count += 1 + assert page_count == 2