native Query DSL format

Author: taraslayshchuk

README.rst

@@ -4,7 +4,7 @@ es2csv
 A CLI tool for exporting data from Elasticsearch into a CSV file
 ----------------------------------------------------------------
 
-Command line utility, written in Python, for querying Elasticsearch in Lucene query syntax and for exporting documents into a CSV file. This tool can query bulk docs in multiple indices and get only selected fields, this reduces query execution time.
+Command line utility, written in Python, for querying Elasticsearch in Lucene query syntax or Query DSL syntax and exporting result as documents into a CSV file. This tool can query bulk docs in multiple indices and get only selected fields, this reduces query execution time.
 
 Quick Look Demo
 ---------------
@@ -31,7 +31,7 @@ Usage
 
  $ es2csv [-h] -q QUERY [-u URL] [-i INDEX [INDEX ...]]
           [-t TAGS [TAGS ...]] -o FILE [-f FIELDS [FIELDS ...]]
-          [-d DELIMITER] [-m INTEGER] [-k] [--debug]
+          [-d DELIMITER] [-m INTEGER] [-k] [-r] [-v] [--debug]
 
  Arguments:
   -q, --query QUERY                        Query string in Lucene syntax.               [required]
@@ -43,6 +43,8 @@ Usage
   -d, --delimiter DELIMITER                Delimiter to use in CSV file. Default is ",".
   -m, --max INTEGER                        Maximum number of results to return. Default is 0.
   -k, --kibana_nested                      Format nested fields in Kibana style.
+  -r, --raw_query                          Switch query format in the Query DSL.
+  -v, --version                            Show version and exit.
   --debug                                  Debug mode on.
   -h, --help                               show this help message and exit
 
@@ -53,6 +55,12 @@ Searching on localhost and save to database.csv
 .. code-block:: bash
 
   $ es2csv -q 'host: localhost' -o database.csv
+
+Same in Query DSL syntax
+
+.. code-block:: bash
+
+  $ es2csv -r -q '{"query": {"match": {"host": "localhost"}}}' -o database.csv
   
 With tag
 

es2csv.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python
 """
 title:           A CLI tool for exporting data from Elasticsearch into a CSV file.
-description:     Command line utility, written in Python, for querying Elasticsearch in Lucene query syntax and for exporting documents into a CSV file.
+description:     Command line utility, written in Python, for querying Elasticsearch in Lucene query syntax or Query DSL syntax and exporting result as documents into a CSV file.
 usage:           es2csv -q '*' -i _all -o ~/file.csv -k -m 100
+                 es2csv -q '{"query": {"match_all": {}}}' -r -i _all -o ~/file.csv
                  es2csv -q '*' -i logstash-2015-01-* -f host status message -o ~/file.csv
                  es2csv -q 'host: localhost' -i logstash-2015-01-01 logstash-2015-01-02 -f host status message -o ~/file.csv
                  es2csv -q 'host: localhost AND status: GET' -u http://kibana.com:80/es/ -o ~/file.csv
                  es2csv -q '*' -t dev prod -u http://login:[email protected]:6666/es/ -o ~/file.csv
+                 es2csv -q '{"query": {"match_all": {}}, "filter":{"term": {"tags": "dev"}}}' -r -u http://login:[email protected]:6666/es/ -o ~/file.csv
 """
 import os
 import sys
@@ -85,25 +87,28 @@ def check_indexes(self):
 
     @retry(elasticsearch.exceptions.ConnectionError, tries=TIMES_TO_TRY)
     def search_query(self):
-        query = self.opts.query if not self.opts.tags else '%s AND tags:%s' % (
-            self.opts.query, '(%s)' % ' AND '.join(self.opts.tags))
-
-        if self.opts.debug_mode:
-            print('Using these indices: %s' % ', '.join(self.opts.index_prefixes))
-            print('Query: %s' % query)
-            print('Output field(s): %s' % ', '.join(self.opts.fields))
-
         search_args = dict(
             index=','.join(self.opts.index_prefixes),
-            q=query,
             search_type='scan',
             scroll=self.scroll_time,
             size=self.scroll_size
         )
+        if self.opts.raw_query:
+            query = json.loads(self.opts.query)
+            search_args['body'] = query
+        else:
+            query = self.opts.query if not self.opts.tags else '%s AND tags:%s' % (
+                 self.opts.query, '(%s)' % ' AND '.join(self.opts.tags))
+            search_args['q'] = query
 
         if '_all' not in self.opts.fields:
             search_args['fields'] = ','.join(self.opts.fields)
 
+        if self.opts.debug_mode:
+            print('Using these indices: %s' % ', '.join(self.opts.index_prefixes))
+            print('Query[%s]: %s' % (('Query DSL', json.dumps(query)) if self.opts.raw_query else ('Lucene', query)))
+            print('Output field(s): %s' % ', '.join(self.opts.fields))
+
         res = self.es_conn.search(**search_args)
 
         self.scroll_ids.append(res['_scroll_id'])
@@ -236,6 +241,7 @@ def main():
     p.add_argument('-d', '--delimiter', dest='delimiter', default=',', type=str, help='Delimiter to use in CSV file. Default is "%(default)s".')
     p.add_argument('-m', '--max', dest='max_results', default=0, type=int, metavar='INTEGER', help='Maximum number of results to return. Default is %(default)s.')
     p.add_argument('-k', '--kibana_nested', dest='kibana_nested', action='store_true', help='Format nested fields in Kibana style.')
+    p.add_argument('-r', '--raw_query', dest='raw_query', action='store_true', help='Switch query format in the Query DSL.')
     p.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__, help='Show version and exit.')
     p.add_argument('--debug', dest='debug_mode', action='store_true', help='Debug mode on.')