add otel metrics to exporter

Author: spompea-cfa

Dockerfile.exporter

@@ -7,6 +7,11 @@ RUN apt-get update \
  && apt-get -y autoremove \
  && apt-get -y clean
 
+RUN pip3 install --break-system-packages opentelemetry-distro
+# Above, `--break-system-packages` flag overrides the
+# "This environment is externally managed" error that calling pip
+# would otherwise incur here.
+
 WORKDIR /app
 COPY middleware/* .
 

README.md

@@ -265,6 +265,8 @@ docker compose run --env AWS_PROFILE=localstack --env S3_BUCKET_NAME=sqs-senzing
 - `FOLDER_NAME` -- optional (defaults to `exporter-outputs`); folder inside S3
   where the file will be placed.
 - `LOG_LEVEL` -- optional; defaults to `INFO`.
+- `RUNTIME_ENV` -- the runtime enviroment (e.g., "Dev", "Prod", etc.). Optional;
+  defaults to "unknown".
 
 _Mounts in docker-compose.yaml:_
 

middleware/consumer.py

@@ -171,9 +171,11 @@ def clean_up(signum, frm):
         log.error(fmterr(e))
 
     # OTel setup #
+    log.info('Starting OTel setup.')
     meter = otel.init('consumer')
     otel_msgs_counter = meter.create_counter('consumer.messages.count')
     otel_durations = meter.create_histogram('consumer.messages.duration')
+    log.info('Finished OTel setup.')
     # end OTel setup #
 
     while 1:

middleware/exporter.py

@@ -10,6 +10,8 @@
 from loglib import *
 log = retrieve_logger()
 
+import otel
+
 try:
     log.info('Importing senzing_core library . . .')
     import senzing_core as sz_core
@@ -28,6 +30,7 @@
     sys.exit(1)
 S3_BUCKET_NAME = os.environ['S3_BUCKET_NAME']
 FOLDER_NAME = os.environ.get('FOLDER_NAME', 'exporter-outputs')
+RUNTIME_ENV = os.environ.get('RUNTIME_ENV', 'unknown') # For OTel
 
 EXPORT_FLAGS =  sz.SzEngineFlags.SZ_EXPORT_DEFAULT_FLAGS
 
@@ -82,13 +85,25 @@ def go():
     except Exception as e:
         log.error(fmterr(e))
 
+    # OTel setup #
+    log.info('Starting OTel setup.')
+    meter = otel.init('exporter')
+    otel_exp_counter = meter.create_counter('exporter.export.count')
+    otel_duration = meter.create_histogram('exporter.export.duration')
+    log.info('Finished OTel setup.')
+    # end OTel setup #
+
     # init buffer
     buff = io.BytesIO()
 
     # Retrieve output from sz into buff
     # sz will export JSONL lines; we add the chars necessary to make
     # the output as a whole be a single JSON blob.
     log.info(SZ_TAG + 'Starting export from Senzing.')
+
+    start = time.perf_counter()
+    success_status = otel.FAILURE # initial default state
+
     try:
         export_handle = sz_eng.export_json_entity_report(EXPORT_FLAGS)
         log.info(SZ_TAG + 'Obtained export_json_entity_report handle.')
@@ -121,9 +136,20 @@ def go():
     try:
         s3.upload_fileobj(buff, S3_BUCKET_NAME, full_path)
         log.info(AWS_TAG + 'Successfully uploaded file.')
+        success_status = otel.SUCCESS
     except Exception as e:
         log.error(AWS_TAG + fmterr(e))
 
+    finish = time.perf_counter()
+    otel_exp_counter.add(1,
+        {'status': success_status,
+        'service': 'exporter',
+        'environment': RUNTIME_ENV})
+    otel_duration.record(finish - start,
+        {'status':  success_status,
+         'service': 'exporter',
+         'environment': RUNTIME_ENV})
+
 #-------------------------------------------------------------------------------
 
 def main():
@@ -134,4 +160,3 @@ def main():
     go()
 
 if __name__ == '__main__': main()
-