From e4b14940324dac22e6bc285899c42cdeca30f985 Mon Sep 17 00:00:00 2001 From: ke-zhang-rd Date: Wed, 9 Jan 2019 15:55:38 -0500 Subject: [PATCH 01/10] MNT: Create loggers.py and add ElasticHandler --- nslsii/loggers.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 nslsii/loggers.py diff --git a/nslsii/loggers.py b/nslsii/loggers.py new file mode 100644 index 00000000..938b3411 --- /dev/null +++ b/nslsii/loggers.py @@ -0,0 +1,51 @@ +import logging +import requests + + +class ElasticHandler(logging.Handler): + ''' + Logging handler that submits data to Elastic + + Parameters + ---------- + url: string + url to Elastic server + + level: Logging Levels, optional + Handler logging level + ''' + def __init__(self, url, level=logging.INFO): + self.url = url + self.level = level + super().__init__(self.level) + + def emit(self, record): + ''' + method is used to send the message to its destination and + logging for debug + ''' + # Extract useful info from the record and put them into a dict. + try: + record_dict = {'name': record.name, + 'levelno': record.levelno, + 'pathname': record.pathname, + 'lineno': record.lineno, + 'msg': record.msg, + 'args': record.args, + 'exc_info': record.exc_info, + 'sinfo': record.sinfo} + response = requests.post(self.url, json=record_dict) + response.raise_for_status() + except Exception: + self.handleError(record) + + +bluesky_logger = logging.getLogger('bluesky') +carproto_logger = logging.getLogger('carproto') +ophyd_logger = logging.getLogger('ophyd') + +elastic_hdr = ElasticHandler('http://elasticsearch.cs.nsls2.local/test/_doc/1') + +bluesky_logger.addHandler(elastic_hdr) +carproto_logger.addHandler(elastic_hdr) +ophyd_logger.addHandler(elastic_hdr) From a4ca2845bf2c8035573ed7e2481b965a548d10f6 Mon Sep 17 00:00:00 2001 From: ke-zhang-rd Date: Fri, 11 Jan 2019 12:08:39 -0500 Subject: [PATCH 02/10] MNT: Add .flake8 --- .flake8 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.flake8 b/.flake8 index 44b8a5e3..783a9263 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,8 @@ [flake8] +<<<<<<< HEAD exclude = .git,__pycache__,build,dist,versioneer.py,nslsii/_version.py,docs/source/conf.py +======= +ignore = E402,W504 +exclude = .git,__pycache__,doc/source/conf.py,build,dist,versioneer.py,caproto/_headers.py,caproto/tests/test_pyepics_compat.py,caproto/_version.py +max-line-length = 115 +>>>>>>> MNT: Add .flake8 From b3dcf8c298ae9281c19ac3a3389349fad8007019 Mon Sep 17 00:00:00 2001 From: ke-zhang-rd Date: Fri, 11 Jan 2019 12:20:13 -0500 Subject: [PATCH 03/10] STY: PEP8 --- nslsii/common/ipynb/animation.py | 2 +- nslsii/common/ipynb/info.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nslsii/common/ipynb/animation.py b/nslsii/common/ipynb/animation.py index fc34ceb6..a4365d2b 100644 --- a/nslsii/common/ipynb/animation.py +++ b/nslsii/common/ipynb/animation.py @@ -52,7 +52,7 @@ def view_frame(i, vmin, vmax): plt.show() - interact(view_frame, i=(0, n-1), vmin=minmax, vmax=minmax) + interact(view_frame, i=(0, n - 1), vmin=minmax, vmax=minmax) def image_stack_to_movie(images, frames=None, vmin=None, vmax=None, diff --git a/nslsii/common/ipynb/info.py b/nslsii/common/ipynb/info.py index 1af3af0b..818d8a0d 100644 --- a/nslsii/common/ipynb/info.py +++ b/nslsii/common/ipynb/info.py @@ -16,11 +16,11 @@ def get_sys_info(): mem = psutil.virtual_memory() html += 'Total System Memory' - html += '{:.4} Mb'.format(mem.total/1024**3) + html += '{:.4} Mb'.format(mem.total / 1024**3) html += 'Total Memory Used' - html += '{:.4} Mb'.format(mem.used/1024**3) + html += '{:.4} Mb'.format(mem.used / 1024**3) html += 'Total Memory Free' - html += '{:.4} Mb'.format(mem.free/1024**3) + html += '{:.4} Mb'.format(mem.free / 1024**3) html += 'Number of CPU Cores{}'.format( psutil.cpu_count()) From 4d27ea957dd525e3e024047bb6a0c38883b1cf49 Mon Sep 17 00:00:00 2001 From: ke-zhang-rd Date: Fri, 11 Jan 2019 15:54:00 -0500 Subject: [PATCH 04/10] MNT: delete caproto related file --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 783a9263..4c11b1c2 100644 --- a/.flake8 +++ b/.flake8 @@ -3,6 +3,6 @@ exclude = .git,__pycache__,build,dist,versioneer.py,nslsii/_version.py,docs/source/conf.py ======= ignore = E402,W504 -exclude = .git,__pycache__,doc/source/conf.py,build,dist,versioneer.py,caproto/_headers.py,caproto/tests/test_pyepics_compat.py,caproto/_version.py +exclude = .git,__pycache__,doc/source/conf.py,build,dist,versioneer.py max-line-length = 115 >>>>>>> MNT: Add .flake8 From a0169b92680dce4b9aefdaa88f698dfdc7f95f4b Mon Sep 17 00:00:00 2001 From: ke-zhang-rd Date: Mon, 14 Jan 2019 15:26:30 -0500 Subject: [PATCH 05/10] MNT: Add requests --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 17f26884..507762c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ numpy ophyd psutil pyolog +requests From 6bd6648c81b491b872493e6ff179754cd06d3a44 Mon Sep 17 00:00:00 2001 From: ke-zhang-rd Date: Tue, 15 Jan 2019 12:06:42 -0500 Subject: [PATCH 06/10] MNT: Update elastic index --- nslsii/__init__.py | 4 ++++ nslsii/loggers.py | 18 +++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/nslsii/__init__.py b/nslsii/__init__.py index b5d85732..b679a355 100644 --- a/nslsii/__init__.py +++ b/nslsii/__init__.py @@ -1,5 +1,6 @@ from IPython import get_ipython from ._version import get_versions +import .loggers import configure_elastic __version__ = get_versions()['version'] del get_versions @@ -148,6 +149,9 @@ def configure_base(user_ns, broker_name, *, ch.setLevel(logging.ERROR) ophyd.ophydobj.logger.addHandler(ch) + configure_elastic(broker_name) + #Configure elastic log handler including bluesky, carproto and ophyd + # convenience imports # some of the * imports are for 'back-compatibility' of a sort -- we have # taught BL staff to expect LiveTable and LivePlot etc. to be in their diff --git a/nslsii/loggers.py b/nslsii/loggers.py index 938b3411..2b091b73 100644 --- a/nslsii/loggers.py +++ b/nslsii/loggers.py @@ -39,13 +39,17 @@ def emit(self, record): except Exception: self.handleError(record) +def configure_elastic(beamline): + url = 'http://elasticsearch.cs.nsls2.local/' + beamline + '.blueksy' -bluesky_logger = logging.getLogger('bluesky') -carproto_logger = logging.getLogger('carproto') -ophyd_logger = logging.getLogger('ophyd') + bluesky_logger = logging.getLogger('bluesky') + bluesky_elastic_hdr = ElasticHandler(url, level = logging.DEBUG) + bluesky_logger.addHandler(elastic_hdr) -elastic_hdr = ElasticHandler('http://elasticsearch.cs.nsls2.local/test/_doc/1') + carproto_logger = logging.getLogger('carproto') + carproto_elastic_hdr = ElasticHandler(url, level = logging.INFO) + carproto_logger.addHandler(elastic_hdr) -bluesky_logger.addHandler(elastic_hdr) -carproto_logger.addHandler(elastic_hdr) -ophyd_logger.addHandler(elastic_hdr) + ophyd_logger = logging.getLogger('ophyd') + ophyd_elastic_hdr = ElasticHandler(url, level = logging.INFO) + ophyd_logger.addHandler(elastic_hdr) From 5dbb1ff629e0b461f4eae4ab6200e1879466ed30 Mon Sep 17 00:00:00 2001 From: ke-zhang-rd Date: Thu, 14 Mar 2019 13:33:52 -0400 Subject: [PATCH 07/10] MNT: Update .flaske8 --- .flake8 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.flake8 b/.flake8 index 4c11b1c2..51b7df2b 100644 --- a/.flake8 +++ b/.flake8 @@ -1,8 +1,4 @@ [flake8] -<<<<<<< HEAD -exclude = .git,__pycache__,build,dist,versioneer.py,nslsii/_version.py,docs/source/conf.py -======= ignore = E402,W504 exclude = .git,__pycache__,doc/source/conf.py,build,dist,versioneer.py max-line-length = 115 ->>>>>>> MNT: Add .flake8 From c99bc5df7d8949943e520db13be64f258ec73d59 Mon Sep 17 00:00:00 2001 From: ke-zhang-rd Date: Tue, 26 Mar 2019 15:18:06 -0400 Subject: [PATCH 08/10] MNT: typos --- nslsii/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nslsii/__init__.py b/nslsii/__init__.py index b679a355..ac12dfae 100644 --- a/nslsii/__init__.py +++ b/nslsii/__init__.py @@ -1,6 +1,6 @@ from IPython import get_ipython from ._version import get_versions -import .loggers import configure_elastic +from .loggers import configure_elastic __version__ = get_versions()['version'] del get_versions From 1c7749420bfc45b77a5ab5fef134f8b677387215 Mon Sep 17 00:00:00 2001 From: ke-zhang-rd Date: Tue, 26 Mar 2019 15:20:36 -0400 Subject: [PATCH 09/10] MNT: Update dict for es --- nslsii/loggers.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/nslsii/loggers.py b/nslsii/loggers.py index 2b091b73..721d5a3c 100644 --- a/nslsii/loggers.py +++ b/nslsii/loggers.py @@ -1,6 +1,6 @@ import logging import requests - +import json class ElasticHandler(logging.Handler): ''' @@ -27,13 +27,16 @@ def emit(self, record): # Extract useful info from the record and put them into a dict. try: record_dict = {'name': record.name, - 'levelno': record.levelno, + 'levelname': record.levelname, 'pathname': record.pathname, 'lineno': record.lineno, - 'msg': record.msg, - 'args': record.args, - 'exc_info': record.exc_info, - 'sinfo': record.sinfo} + 'msg': self.format(record), + 'args': list(map(str, record.args)), + 'threadName': record.threadName} + + if hasattr(record, 'pv'): record_dict['pv'] = record.pv + if hasattr(record, 'address'): record_dict['address'] = record.address + if hasattr(record, 'role'): record_dict['role'] = record.role response = requests.post(self.url, json=record_dict) response.raise_for_status() except Exception: @@ -41,13 +44,14 @@ def emit(self, record): def configure_elastic(beamline): url = 'http://elasticsearch.cs.nsls2.local/' + beamline + '.blueksy' + local_url = 'http://localhost:9200/test1/_doc' bluesky_logger = logging.getLogger('bluesky') bluesky_elastic_hdr = ElasticHandler(url, level = logging.DEBUG) bluesky_logger.addHandler(elastic_hdr) carproto_logger = logging.getLogger('carproto') - carproto_elastic_hdr = ElasticHandler(url, level = logging.INFO) + carproto_elastic_hdr = ElasticHandler(local_url, level = logging.DEBUG) carproto_logger.addHandler(elastic_hdr) ophyd_logger = logging.getLogger('ophyd') From 94b3de856640a94a2294c99fbbf7b77b0d679fa4 Mon Sep 17 00:00:00 2001 From: ke-zhang-rd Date: Fri, 29 Mar 2019 16:09:09 -0400 Subject: [PATCH 10/10] MNH: Assume record.address as a (host,port) tuple --- nslsii/loggers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nslsii/loggers.py b/nslsii/loggers.py index 721d5a3c..db6ba3f5 100644 --- a/nslsii/loggers.py +++ b/nslsii/loggers.py @@ -35,7 +35,7 @@ def emit(self, record): 'threadName': record.threadName} if hasattr(record, 'pv'): record_dict['pv'] = record.pv - if hasattr(record, 'address'): record_dict['address'] = record.address + if hasattr(record, 'address'): record_dict['address'] = record.address[0]+':'+str(record.address[1]) if hasattr(record, 'role'): record_dict['role'] = record.role response = requests.post(self.url, json=record_dict) response.raise_for_status()