configurable shards

Author: Bek

base.ini

@@ -5,6 +5,8 @@ retry.attempts = 3
 file_upload_bucket = snowflakes-files-dev
 file_upload_profile_name = snowflakes-files-upload
 elasticsearch.server = localhost:9201
+elasticsearch.shards.primary = 1
+elasticsearch.shards.replicate = 0
 aws_ip_ranges_path = %(here)s/aws-ip-ranges.json
 download_proxy = https://download.encodeproject.org/
 annotations_path = %(here)s/annotations.json

src/snovault/elasticsearch/create_mapping.py

@@ -164,13 +164,13 @@ def schema_mapping(name, schema):
         }
 
 
-def index_settings():
+def index_settings(primary_shards=3, replicate_shards=1):
     return {
         'settings': {
             'index.max_result_window': 99999,
             'index.mapping.total_fields.limit': 5000,
-            'index.number_of_shards': 1,
-            'index.number_of_replicas': 0,
+            'index.number_of_shards': primary_shards,
+            'index.number_of_replicas': replicate_shards,
             'index.max_ngram_diff': 32,
             'analysis': {
                 'filter': {
@@ -484,6 +484,8 @@ def create_snovault_index_alias(es, indices):
 def run(app, collections=None, dry_run=False):
     index = app.registry.settings['snovault.elasticsearch.index']
     registry = app.registry
+    primary_shards = registry.settings.get('elasticsearch.shards.primary', 3)
+    replicate_shards = registry.settings.get('elasticsearch.shards.replicate', 1)
     if not dry_run:
         es = app.registry[ELASTIC_SEARCH]
         print('CREATE MAPPING RUNNING')
@@ -503,9 +505,9 @@ def run(app, collections=None, dry_run=False):
         if mapping is None:
             continue  # Testing collections
         if dry_run:
-            print(json.dumps(sorted_dict({index: {doc_type: mapping}}), indent=4))
+            print(json.dumps(sorted_dict({index: {collection_name: mapping}}), indent=4))
             continue
-        create_elasticsearch_index(es, index, index_settings())
+        create_elasticsearch_index(es, index, index_settings(primary_shards, replicate_shards))
         set_index_mapping(es, index, mapping)
         if collection_name != 'meta':
             indices.append(index)