add some utils for common tasks

Author: spompea-cfa

Dockerfile.consumer

@@ -3,7 +3,7 @@ FROM senzing/senzingsdk-runtime:4.0.0
 USER root
 
 RUN apt-get update \
- && apt-get -y install --no-install-recommends curl python3 python3-pip python3-boto3 \
+ && apt-get -y install --no-install-recommends python3 python3-pip python3-boto3 \
  && apt-get -y autoremove \
  && apt-get -y clean
 

Dockerfile.exporter

@@ -3,7 +3,7 @@ FROM senzing/senzingsdk-runtime:4.0.0
 USER root
 
 RUN apt-get update \
- && apt-get -y install --no-install-recommends curl python3 python3-pip python3-boto3 \
+ && apt-get -y install --no-install-recommends python3 python3-pip python3-boto3 \
  && apt-get -y autoremove \
  && apt-get -y clean
 

Dockerfile.util

@@ -0,0 +1,26 @@
+FROM senzing/senzingsdk-runtime:4.0.0
+
+USER root
+
+RUN apt-get update \
+ && apt-get -y install --no-install-recommends python3 python3-pip python3-boto3 \
+ && apt-get -y autoremove \
+ && apt-get -y clean
+
+WORKDIR /app
+COPY middleware/* .
+
+# Add a new user and switch to it.
+RUN useradd -m -u 1001 senzing
+USER senzing
+
+ENV PYTHONPATH=/opt/senzing/er/sdk/python:/app
+
+# Flush buffer - helps with print statements.
+ENV PYTHONUNBUFFERED=1
+
+ENV AWS_PROFILE=localstack
+ENV Q_URL=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/sqs-senzing-local-ingest
+ENV S3_BUCKET_NAME=sqs-senzing-local-export
+
+ENTRYPOINT ["python3"]

README.md

@@ -111,6 +111,24 @@ sz_command -C add_record \
   PEOPLE 1 '{"NAME_FULL":"Robert Smith", "DATE_OF_BIRTH":"7/4/1976", "PHONE_NUMBER":"555-555-2088"}'
 ```
 
+### Utilities
+
+Load a single record as a simple test:
+
+    docker compose run util util_load_record.py
+
+Purge the database:
+
+    docker compose run util util_purge_db.py
+
+Copy a file out of the LocalStack S3 bucket into `~/tmp` on your machine (be 
+sure this folder already exists):
+
+    docker compose run util util_s3_retrieve.py hemingway.txt
+
+Purge the LocalStack S3 bucket:
+
+    docker compose run util util_purge_s3.py
 
 [awslocal]: https://docs.localstack.cloud/aws/integrations/aws-native-tools/aws-cli/#localstack-aws-cli-awslocal
 [localstack]: https://www.localstack.cloud/

docker-compose.yaml

@@ -125,6 +125,31 @@ services:
       - ~/.aws:/home/senzing/.aws
       - ~/tmp:/tmp # Should you wish to write files to host.
 
+  util:
+    build:
+      context: .
+      dockerfile: Dockerfile.util
+    depends_on:
+      - db
+    environment:
+      AWS_ENDPOINT_URL: http://localstack:4566
+      SENZING_ENGINE_CONFIGURATION_JSON: >-
+        {
+          "PIPELINE": {
+            "CONFIGPATH": "/etc/opt/senzing",
+            "LICENSESTRINGBASE64": "${SENZING_LICENSE_BASE64_ENCODED}",
+            "RESOURCEPATH": "/opt/senzing/er/resources",
+            "SUPPORTPATH": "/opt/senzing/data"
+          },
+          "SQL": {
+            "BACKEND": "SQL",
+            "CONNECTION": "postgresql://${POSTGRES_USERNAME:-senzing}:${POSTGRES_PASSWORD:-senzing}@db:5432:${POSTGRES_DB:-G2}/?sslmode=disable"
+          }
+        }
+    volumes:
+      - ~/.aws:/home/senzing/.aws
+      - ~/tmp:/tmp # Should you wish to write files to host.
+
 volumes:
   localstack:
   postgres:

middleware/exporter.py

@@ -128,23 +128,3 @@ def main():
 
 if __name__ == '__main__': main()
 
-#-------------------------------------------------------------------------------
-# ad-hoc test funcs - might move later
-
-def _upload_test_file_to_s3():
-    print("Starting test upload to S3 ...")
-    s3 = make_s3_client()
-    print(s3)
-    fname = 'hemingway.txt'
-    resp = s3.upload_file(fname, S3_BUCKET_NAME, fname)
-    print(resp) 
-    print('Upload successful.')
-
-def _get_file_from_s3(key):
-    '''Get file from S3 and write to /tmp (use docker-compose to map this
-    to desired directory on host machine).'''
-    print('Grabbing file...') 
-    s3 = make_s3_client()
-    resp = s3.download_file(S3_BUCKET_NAME, key, '/tmp/'+key)
-    print(resp)
-    print('Done.')

middleware/testutils.py

@@ -1,4 +1,5 @@
 import json
+import os
 import sys
 import time
 import boto3
@@ -7,8 +8,9 @@
 # sample-data/customers.jsonl into SQS.
 
 # For live AWS, can set these to other values.
-AWS_PROFILE_NAME = 'localstack'
+AWS_PROFILE = 'localstack'
 Q_URL = 'http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/sqs-senzing-local-ingest'
+S3_BUCKET_NAME = 'sqs-senzing-local-export'
 
 #------------------------------------------------------------------------------
 
@@ -23,7 +25,9 @@ def make_boto_session(fpath=None):
     if fpath:
         return boto3.Session(**json.load(open(fpath)))
     else:
-        return boto3.Session(profile_name=AWS_PROFILE_NAME)
+        # Here we pass in profile_name explicitly since it's not necessarily an env
+        # var in this context.
+        return boto3.Session(profile_name=AWS_PROFILE)
 
 def make_sqs_client(boto_session):
     return boto_session.client('sqs')
@@ -71,3 +75,25 @@ def _get_1_msg(sqs, q_url):
     print(resp)
     return resp
 
+#-------------------------------------------------------------------------------
+
+def make_s3_client():
+    try:
+        # Here we pass in profile_name explicitly since it's not necessarily an env
+        # var in this context.
+        sess = boto3.Session(profile_name=AWS_PROFILE)
+        if 'AWS_ENDPOINT_URL' in os.environ:
+            return sess.client('s3', endpoint_url=os.environ['AWS_ENDPOINT_URL'])
+        else:
+            return sess.client('s3')
+    except Exception as e:
+        print(e)
+
+def upload_test_file_to_s3():
+    print("Starting test upload to S3 ...")
+    s3 = make_s3_client()
+    print(s3)
+    fname = 'sample-data/hemingway.txt'
+    resp = s3.upload_file(fname, S3_BUCKET_NAME, fname[fname.rfind('/')+1:])
+    print(resp) 
+    print('Upload successful.')

middleware/sz_test.py renamed to middleware/util_load_record.py

@@ -16,9 +16,9 @@
 sz_factory = sz_core.SzAbstractFactoryCore("sz_factory_1", senzing_engine_configuration_json)
 sz_diagnostic = sz_factory.create_diagnostic()
 
-print('Are you sure you want to purge the repository? If so, type YES:')
+print('Are you sure you want to purge the database? If so, type YES:')
 ans = input('>')
 if ans == 'YES':
     sz_diagnostic.purge_repository()
 else:
-    print('Everything left as-is.')
+    print('Nothing was done. Everything was left as-is.')

middleware/sz_purge.py renamed to middleware/util_purge_db.py

@@ -0,0 +1,16 @@
+import os
+import boto3
+
+def purge_s3():
+    s3 = boto3.resource('s3', endpoint_url=os.environ['AWS_ENDPOINT_URL'])
+    buck = s3.Bucket(os.environ['S3_BUCKET_NAME'])
+    print('Purging...') 
+    buck.objects.all().delete()
+
+print('Are you sure you want to purge the S3 bucket (' + os.environ['S3_BUCKET_NAME'] + ')? If so, type YES:')
+ans = input('>')
+if ans == 'YES':
+    purge_s3()
+    print('Done.')
+else:
+    print('Nothing was done. Everything was left as-is.')