SSL + arguments + python3 + skip collectors #10
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,31 +1,50 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| from __future__ import absolute_import | ||
| from __future__ import division | ||
| from __future__ import print_function | ||
| from __future__ import unicode_literals | ||
|
|
||
| import sys | ||
| is_python_3 = sys.version_info > (3, 0) | ||
|
|
||
| import argparse, base64, re, struct, time, socket, datetime, os.path | ||
|
|
||
| if is_python_3: | ||
|
Owner
There was a problem hiding this comment. It's better use Something like that: # For Python 2 and 3 compatibility
try:
import urllib2
except ImportError:
# Since Python 3, urllib2.Request and urlopen were moved to
# the urllib.request.
import urllib.request as urllib2 |
||
| import urllib.request as urllib # Python 3 | ||
| import codecs | ||
| else: | ||
| import urllib2 as urllib # Python 2 | ||
|
|
||
| try: | ||
| import json | ||
| except: | ||
| import simplejson as json | ||
|
|
||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument('--use-ssl', default=False, action="store_true", help='Enable SSL') | ||
| parser.add_argument('--ssl-cafile', default='/etc/zabbix/certs/cacert.pem', type=str, help='CA PEM file path') | ||
| parser.add_argument('--ssl-certfile', default='/etc/zabbix/certs/agent.crt', type=str, help='Client/Agent certificate file path') | ||
| parser.add_argument('--ssl-keyfile', default='/etc/zabbix/certs/agent.key', type=str, help='Client/Agent private key file path') | ||
| parser.add_argument('--zabbix-server', default='127.0.0.1', type=str, help='Zabbix server host') | ||
| parser.add_argument('--zabbix-port', default=10051, type=int, help='Zabbix server port') | ||
| parser.add_argument('--monitored-hostname', default='Zabbix agent', type=str, help='Name of monitored host, like it shows in zabbix web ui') | ||
| parser.add_argument('--nginx-auth-username', default=None, type=str, help='Nginx authentication username') | ||
| parser.add_argument('--nginx-auth-password', default=None, type=str, help='Nginx authentication password') | ||
| parser.add_argument('--nginx-status-module-url', default='http://127.0.0.1:55777/nginx-status', type=str, help='URL to retrieve Nginx status (http_stub_status_module)') | ||
|
Owner
There was a problem hiding this comment. why the port is 55777 is that default port for stats or in your specific case?
Author
There was a problem hiding this comment. Sorry, it was something specific of our nginx configuration as we don't expose the stats module to the world. I should have changed it back to the default HTTP port. |
||
| parser.add_argument('--nginx-accesslog', default='/var/log/nginx/access.log', type=str, help='Nginx access log file path') | ||
| parser.add_argument('--print-metric', default=-1, type=int, help='Print an specific metric') | ||
| parser.add_argument('--skip-nginx-accesslog', default=False, action="store_true", help='Don\'t gather data from the Nginx access log') | ||
| parser.add_argument('--skip-nginx-status-module', default=False, action="store_true", help='Don\'t gather data from the Nginx status module (http_stub_status_module)') | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| # grep interval in minutes | ||
| time_delta = 1 | ||
|
|
||
| # Temp file, with log file cursor position | ||
| seek_file = '/tmp/nginx_log_stat' | ||
|
|
||
| class Metric(object): | ||
| def __init__(self, host, key, value, clock=None): | ||
| self.host = host | ||
|
|
@@ -38,81 +57,106 @@ def __repr__(self): | |
| return 'Metric(%r, %r, %r)' % (self.host, self.key, self.value) | ||
| return 'Metric(%r, %r, %r, %r)' % (self.host, self.key, self.value, self.clock) | ||
|
|
||
| def send_to_zabbix(metrics, zabbix_host, zabbix_port): | ||
| j = json.dumps | ||
| metrics_data = [] | ||
| for m in metrics: | ||
| clock = m.clock or ('%d' % time.time()) | ||
| #print('%s, %s, %s, %s' % (m.host, m.key, m.value, clock)) | ||
| metrics_data.append('{"host":%s,"key":%s,"value":%s,"clock":%s}' % (j(m.host), j(m.key), j(m.value), j(clock))) | ||
|
|
||
| # Zabbix 3.0 protocol - https://www.zabbix.com/documentation/3.0/manual/appendix/items/activepassive | ||
| data = ('{"request":"sender data","data":[%s]}' % (','.join(metrics_data))) | ||
| data_length = len(data) | ||
| data_header = struct.pack('<Q', data_length) # 8 bytes | ||
| data_to_send = b'ZBXD\x01' + data_header + data.encode('utf-8') | ||
| #print(repr(data_to_send)) | ||
|
|
||
| try: | ||
| zabbix = socket.socket() | ||
| sock = zabbix | ||
| if args.use_ssl: | ||
| import ssl | ||
| context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) | ||
| context.options |= ssl.OP_NO_SSLv2 | ||
| context.options |= ssl.OP_NO_SSLv3 | ||
| context.verify_mode = ssl.CERT_REQUIRED | ||
| context.load_verify_locations(cafile=args.ssl_cafile) | ||
| context.load_cert_chain(certfile=args.ssl_certfile, keyfile=args.ssl_keyfile) | ||
|
|
||
| sock = context.wrap_socket(zabbix, server_hostname=zabbix_host) | ||
|
Owner
There was a problem hiding this comment. so, is it support send metrics over ssl? it's very nice. Thanks for idea about use SSL. I didn't know that zabbix support trappers over ssl. It's good idea add it to
Author
There was a problem hiding this comment. Correct. SSL using certificates - I did try to support TLS using PSK (pre-shared key) as well, but the Python's SSL module doesn't support PSK, and we'd need a custom extension which would introduce extra steps and requisites (Makefile, C compiler, etc), so I avoided touching this. No worries :) Thank you for indicating py-zabbix, I didn't know it ;-) |
||
|
|
||
| sock.connect((zabbix_host, zabbix_port)) | ||
| sock.sendall(data_to_send) | ||
|
|
||
| resp_hdr = _recv_all(sock, 13) | ||
| if not resp_hdr.decode('ascii').startswith('ZBXD\x01') or len(resp_hdr) != 13: | ||
| print('Wrong zabbix response') | ||
| return False | ||
|
|
||
| resp_body_len = struct.unpack('<Q', resp_hdr[5:])[0] | ||
| resp_body = sock.recv(resp_body_len).decode('ascii') | ||
|
|
||
| #sock.shutdown(socket.SHUT_RDWR) | ||
| sock.close() | ||
|
|
||
| resp = json.loads(resp_body) | ||
| #print(resp) | ||
| if resp.get('response') != 'success': | ||
| print('Got error from Zabbix: %s' % resp) | ||
| return False | ||
| return True | ||
| except Exception as e: | ||
| print('Error while sending data to Zabbix %s:' % e) | ||
| #sock.shutdown(socket.SHUT_RDWR) | ||
| sock.close() | ||
| return False | ||
|
|
||
| def _recv_all(sock, count): | ||
| buf = b'' | ||
| while len(buf)<count: | ||
| chunk = sock.recv(count-len(buf)) | ||
| if not chunk: | ||
| return buf | ||
| buf += chunk | ||
| return buf # type(buf) == bytes | ||
|
|
||
| def get(url, login, passwd): | ||
| req = urllib.Request(url) | ||
| if login and passwd: | ||
| base64string = base64.encodestring('%s:%s' % (login, passwd)).replace('\n', '') | ||
| req.add_header("Authorization", "Basic %s" % base64string) | ||
| q = urllib.urlopen(req) | ||
| if is_python_3: | ||
| res = codecs.decode(q.read(), 'utf-8') | ||
| else: | ||
| res = q.read().decode('utf-8') | ||
| q.close() | ||
| return res | ||
|
|
||
| def parse_nginx_stat(data): | ||
| a = {} | ||
|
|
||
| # Active connections | ||
| a['active_connections'] = re.match(r'(.*):\s(\d*)', data[0], re.M | re.I).group(2) | ||
| # Accepts | ||
| a['accepted_connections'] = re.match(r'\s(\d*)\s(\d*)\s(\d*)', data[2], re.M | re.I).group(1) | ||
| # Handled | ||
| a['handled_connections'] = re.match(r'\s(\d*)\s(\d*)\s(\d*)', data[2], re.M | re.I).group(2) | ||
| # Requests | ||
| a['handled_requests'] = re.match(r'\s(\d*)\s(\d*)\s(\d*)', data[2], re.M | re.I).group(3) | ||
| # Reading | ||
| a['header_reading'] = re.match(r'(.*):\s(\d*)(.*):\s(\d*)(.*):\s(\d*)', data[3], re.M | re.I).group(2) | ||
| # Writing | ||
| a['body_reading'] = re.match(r'(.*):\s(\d*)(.*):\s(\d*)(.*):\s(\d*)', data[3], re.M | re.I).group(4) | ||
| # Waiting | ||
| a['keepalive_connections'] = re.match(r'(.*):\s(\d*)(.*):\s(\d*)(.*):\s(\d*)', data[3], re.M | re.I).group(6) | ||
|
|
||
| if not is_python_3: | ||
| for key, value in a.iteritems(): | ||
| a[key] = value.decode('utf-8') | ||
|
|
||
| return a | ||
|
|
||
| def read_seek(file): | ||
| if os.path.isfile(file): | ||
|
|
@@ -131,66 +175,68 @@ def write_seek(file, value): | |
| f.write(value) | ||
| f.close() | ||
|
|
||
| #print('[12/Mar/2014:03:21:13 +0400]') | ||
|
|
||
| if not args.skip_nginx_accesslog: | ||
| d = datetime.datetime.now()-datetime.timedelta(minutes=time_delta) | ||
| minute = int(time.mktime(d.timetuple()) / 60)*60 | ||
| d = d.strftime('%d/%b/%Y:%H:%M') | ||
|
|
||
| total_rps = 0 | ||
| rps = [0]*60 | ||
| tps = [0]*60 | ||
| res_code = {} | ||
|
|
||
| nf = open(args.nginx_accesslog, 'r') | ||
|
|
||
| new_seek = seek = read_seek(seek_file) | ||
|
|
||
| # if new log file, don't do seek | ||
| if os.path.getsize(args.nginx_accesslog) > seek: | ||
| nf.seek(seek) | ||
|
|
||
| line = nf.readline() | ||
| while line: | ||
| if d in line: | ||
| new_seek = nf.tell() | ||
| total_rps += 1 | ||
| sec = int(re.match('(.*):(\d+):(\d+):(\d+)\s.*]', line).group(4)) | ||
| code = re.match(r'(.*)"\s(\d*)\s', line).group(2) | ||
| if code in res_code: | ||
| res_code[code] += 1 | ||
| else: | ||
| res_code[code] = 1 | ||
|
|
||
| rps[sec] += 1 | ||
| line = nf.readline() | ||
|
|
||
| if total_rps != 0: | ||
| write_seek(seek_file, str(new_seek)) | ||
|
|
||
| nf.close() | ||
|
|
||
| should_print_metric = (args.print_metric != -1) and re.match(r'nginx\[(.*)\]', args.print_metric, re.M | re.I).group(1) or False | ||
| data = get(args.nginx_status_module_url, args.nginx_auth_username, args.nginx_auth_password).split('\n') | ||
| data = parse_nginx_stat(data) | ||
|
|
||
| data_to_send = [] | ||
|
|
||
| # Adding the metrics to response | ||
| if should_print_metric: | ||
| print(data[metric]) | ||
|
|
||
| if not args.skip_nginx_status_module: | ||
| for i in data: | ||
| data_to_send.append(Metric(args.monitored_hostname, ('nginx[%s]' % i), data[i])) | ||
|
|
||
| if not args.skip_nginx_accesslog: | ||
| # Add the request per seconds to response | ||
| for t in range(0,60): | ||
| data_to_send.append(Metric(args.monitored_hostname, 'nginx[rps]', rps[t], minute+t)) | ||
|
|
||
| # Add the response codes stats to response | ||
| for t in res_code: | ||
| data_to_send.append(Metric(args.monitored_hostname, ('nginx[%s]' % t), res_code[t])) | ||
|
|
||
|
|
||
| send_to_zabbix(data_to_send, args.zabbix_server, args.zabbix_port) | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is it works only for 3.0?
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It works for >=2.7.9 and >=3 as far as I tested.