Added optional scroll_size argument (#27)

* Added optional scroll_size argument

* Some improvement

Author: eduardofv

Diff for: README.rst

@@ -31,10 +31,10 @@ Usage
 
  $ es2csv [-h] -q QUERY [-u URL] [-a AUTH] [-i INDEX [INDEX ...]]
           [-D DOC_TYPE [DOC_TYPE ...]] [-t TAGS [TAGS ...]] -o FILE
-          [-f FIELDS [FIELDS ...]] [-d DELIMITER] [-m INTEGER] [-k]
-          [-r] [-e] [--verify-certs] [--ca-certs CA_CERTS]
-          [--client-cert CLIENT_CERT] [--client-key CLIENT_KEY] [-v]
-          [--debug]
+          [-f FIELDS [FIELDS ...]] [-d DELIMITER] [-m INTEGER]
+          [-s INTEGER] [-k] [-r] [-e] [--verify-certs]
+          [--ca-certs CA_CERTS] [--client-cert CLIENT_CERT]
+          [--client-key CLIENT_KEY] [-v] [--debug]
 
  Arguments:
   -q, --query QUERY                        Query string in Lucene syntax.               [required]
@@ -47,6 +47,7 @@ Usage
   -f, --fields FIELDS [FIELDS ...]         List of selected fields in output. Default is ['_all'].
   -d, --delimiter DELIMITER                Delimiter to use in CSV file. Default is ",".
   -m, --max INTEGER                        Maximum number of results to return. Default is 0.
+  -s, --scroll_size INTEGER                Scroll size for each batch of results. Default is 100.
   -k, --kibana_nested                      Format nested fields in Kibana style.
   -r, --raw_query                          Switch query format in the Query DSL.
   -e, --meta_fields                        Add meta-fields in output.
@@ -204,6 +205,12 @@ Max results count
 
   $ es2csv -m 6283185 -q '*' -i twitter -o database.csv
 
+Retrieve 2000 results in just 2 requests (two scrolls 1000 each):
+
+.. code-block:: bash
+
+  $ es2csv -m 2000 -s 1000 -q '*' -i twitter -o database.csv
+
 Changing column delimiter in CSV file, by default ','
 
 .. code-block:: bash
@@ -254,4 +261,4 @@ A CSV file in default format
 .. code-block::
 
   body,comments.0.age,comments.0.comment,comments.0.date,comments.0.name,comments.0.stars,comments.1.age,comments.1.comment,comments.1.date,comments.1.name,comments.1.stars,tags.0,tags.1,title
-  Making your money work...,28,Great article,2014-09-01,John Smith,4,31,More like this please,2014-10-22,Alice White,5,cash,shares,Nest eggs
+  Making your money work...,28,Great article,2014-09-01,John Smith,4,31,More like this please,2014-10-22,Alice White,5,cash,shares,Nest eggs

Diff for: es2csv.py

@@ -63,7 +63,6 @@ def __init__(self, opts):
 
         self.num_results = 0
         self.scroll_ids = []
-        self.scroll_size = 100
         self.scroll_time = '30m'
 
         self.csv_headers = list(META_FIELDS) if self.opts.meta_fields else []
@@ -96,7 +95,7 @@ def next_scroll(scroll_id):
             index=','.join(self.opts.index_prefixes),
             search_type='scan',
             scroll=self.scroll_time,
-            size=self.scroll_size,
+            size=self.opts.scroll_size,
             terminate_after=self.opts.max_results
         )
 
@@ -264,6 +263,7 @@ def main():
     p.add_argument('-f', '--fields', dest='fields', default=['_all'], type=str, nargs='+', help='List of selected fields in output. Default is %(default)s.')
     p.add_argument('-d', '--delimiter', dest='delimiter', default=',', type=str, help='Delimiter to use in CSV file. Default is "%(default)s".')
     p.add_argument('-m', '--max', dest='max_results', default=0, type=int, metavar='INTEGER', help='Maximum number of results to return. Default is %(default)s.')
+    p.add_argument('-s', '--scroll_size', dest='scroll_size', default=100, type=int, metavar='INTEGER', help='Scroll size for each batch of results. Default is %(default)s.')
     p.add_argument('-k', '--kibana_nested', dest='kibana_nested', action='store_true', help='Format nested fields in Kibana style.')
     p.add_argument('-r', '--raw_query', dest='raw_query', action='store_true', help='Switch query format in the Query DSL.')
     p.add_argument('-e', '--meta_fields', dest='meta_fields', action='store_true', help='Add meta-fields in output.')