SNO-179-allow-devserver-set-es-config-dir

Author: caseylitton

src/snovault/dev_servers.py

@@ -23,7 +23,7 @@
 
 
 EPILOG = __doc__
-
+ENCD_CONFIG_DIR = os.environ.get('ENCD_CONFIG_DIR')
 logger = logging.getLogger(__name__)
 
 
@@ -60,6 +60,14 @@ def main():
     parser.add_argument('--init', action="store_true", help="Init database")
     parser.add_argument('--load', action="store_true", help="Load test set")
     parser.add_argument('--datadir', default='/tmp/snovault', help="path to datadir")
+    parser.add_argument('-c', '--encd-config-dir', default=ENCD_CONFIG_DIR, help="ES dir with config")
+    parser.add_argument(
+        '-e',
+        '--es-config-version',
+        default=5,
+        type=int,
+        help="ES dir with config"
+    )
     args = parser.parse_args()
 
     appsettings = get_appsettings(args.config_uri, name='app')
@@ -77,17 +85,38 @@ def main():
     from snovault.elasticsearch import create_mapping
     datadir = os.path.abspath(args.datadir)
     pgdata = os.path.join(datadir, 'pgdata')
-    esdata = os.path.join(datadir, 'esdata')
+    esdata = os.path.join(datadir, 'data')
+    es_data_dir = None
+    esconfig = None
+    if args.encd_config_dir:
+        es_data_dir = f"{ENCD_CONFIG_DIR}/elasticsearch/es{args.es_config_version}"
+        es_config_dir = f"{es_data_dir}/config"
+        es_yml_path = f"{es_config_dir}/elasticsearch.yml"
+        if os.path.exists(es_yml_path):
+            # es yml config overrides logs/data/host/port vars
+            esconfig = es_config_dir
+            esdata = es_data_dir
     redisdata = os.path.join(datadir, 'redisdata')
     if args.clear:
-        for dirname in [pgdata, esdata, redisdata]:
+        rm_dirs = [args.datadir]
+        if esconfig:
+            rm_dirs.extend([f"{es_data_dir}/data", f"{es_data_dir}/logs", f"{esconfig}/scripts"])
+            es_keystore_path = f"{esconfig}/elasticsearch.keystore"
+            if os.path.exists(es_keystore_path):
+                os.remove(es_keystore_path)
+        for dirname in rm_dirs:
             if os.path.exists(dirname):
                 shutil.rmtree(dirname)
     if args.init:
         postgresql_fixture.initdb(pgdata, echo=True)
 
     postgres = postgresql_fixture.server_process(pgdata, echo=True)
-    elasticsearch = elasticsearch_fixture.server_process(esdata, echo=True)
+    elasticsearch = elasticsearch_fixture.server_process(
+        esdata,
+        configdir=esconfig,
+        echo=True,
+        version=args.es_config_version,
+    )
     nginx = nginx_server_process(echo=True)
     redis_config_path = redis_storage_fixture.initdb(redisdata, local_storage_port, echo=True)
     redis = redis_storage_fixture.server_process(redis_config_path, local_storage_port, local_storage_redis_index, echo=True)

src/snovault/tests/elasticsearch_fixture.py

@@ -7,31 +7,49 @@
     import subprocess
 
 
-def server_process(datadir, host='127.0.0.1', port=9201, prefix='', echo=False):
-    args = [
-        os.path.join(prefix, 'elasticsearch'),
-        '-Enetwork.host=%s' % host,
-        '-Ehttp.port=%d' % port,
-        '-Epath.data=%s' % os.path.join(datadir, 'data'),
-        '-Epath.logs=%s' % os.path.join(datadir, 'logs'),
-    ]
-    if os.environ.get('TRAVIS'):
-        print('IN TRAVIS')
-        echo = True
-        args.append('-Epath.conf=%s/conf' % os.environ['TRAVIS_BUILD_DIR'])
-    elif os.path.exists('/etc/elasticsearch'):
-        print('NOT IN TRAVIS')
-        args.append('-Epath.conf=./conf')
-    print(args)
-    process = subprocess.Popen(
-        args,
-        close_fds=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
+def server_process(
+        datadir,
+        configdir=None,
+        echo=False,
+        host='127.0.0.1',
+        port=9201,
+        prefix='',
+        version=5,
+    ):
+    if configdir:
+        env = os.environ.copy()
+        # How elasticsearch 6+ sets config path
+        env["ES_PATH_CONF"] = configdir
+        args = [
+            'elasticsearch',
+        ]
+        if version <= 5:
+            # How elasticsearch 5 sets config path
+            args.append(f"-Epath.conf={configdir}")
+        process = subprocess.Popen(
+            args,
+            close_fds=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            env=env,
+        )
+    else:
+        args = [
+            os.path.join(prefix, 'elasticsearch'),
+            '-Enetwork.host=%s' % host,
+            '-Ehttp.port=%d' % port,
+            '-Epath.data=%s' % os.path.join(datadir, 'data'),
+            '-Epath.logs=%s' % os.path.join(datadir, 'logs'),
+            '-Epath.conf=./conf',
+        ]
+        process = subprocess.Popen(
+            args,
+            close_fds=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
 
     SUCCESS_LINE = b'started\n'
-
     lines = []
     for line in iter(process.stdout.readline, b''):
         if echo: