Merge pull request #39 from sysdiglabs/master

Master fetch

Author: Dlorite

.circleci/config.yml

@@ -2,13 +2,13 @@
 version: 2.1
 
 orbs:
-  prometheus: prometheus/prometheus@0.11.0
+  prometheus: prometheus/prometheus@0.16.0
 
 executors:
   # This must match .promu.yml.
   golang:
     docker:
-    - image: circleci/golang:1.16
+    - image: cimg/go:1.18
 
 jobs:
   test:
@@ -22,7 +22,7 @@ jobs:
 
   integration:
     docker:
-    - image: circleci/golang:1.16
+    - image: cimg/go:1.18
     - image: << parameters.postgres_image >>
       environment:
         POSTGRES_DB: circle_test
@@ -56,13 +56,11 @@ workflows:
         matrix:
           parameters:
             postgres_image:
-            - circleci/postgres:9.4
-            - circleci/postgres:9.5
-            - circleci/postgres:9.6
             - circleci/postgres:10
             - circleci/postgres:11
             - circleci/postgres:12
             - circleci/postgres:13
+            - cimg/postgres:14.1
     - prometheus/build:
         name: build
         filters:

.github/ISSUE_TEMPLATE/bug_report.md

@@ -37,7 +37,7 @@ assignees: ''
 insert list of flags used here
 ```
 
-* PostgresSQL version:
+* PostgreSQL version:
 
 	insert PostgreSQL version here
 

.github/workflows/golangci-lint.yml

@@ -0,0 +1,30 @@
+name: golangci-lint
+on:
+  push:
+    paths:
+      - "go.sum"
+      - "go.mod"
+      - "**.go"
+      - "scripts/errcheck_excludes.txt"
+      - ".github/workflows/golangci-lint.yml"
+      - ".golangci.yml"
+  pull_request:
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.x
+      - name: Install snmp_exporter/generator dependencies
+        run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
+        if: github.repository == 'prometheus/snmp_exporter'
+      - name: Lint
+        uses: golangci/[email protected]
+        with:
+          version: v1.45.2

.promu.yml

@@ -1,6 +1,6 @@
 go:
     # This must match .circle/config.yml.
-    version: 1.16
+    version: 1.18
 repository:
     path: github.com/prometheus-community/postgres_exporter
 build:

.yamllint

@@ -0,0 +1,28 @@
+---
+extends: default
+
+rules:
+  braces:
+    max-spaces-inside: 1
+    level: error
+  brackets:
+    max-spaces-inside: 1
+    level: error
+  commas: disable
+  comments: disable
+  comments-indentation: disable
+  document-start: disable
+  indentation:
+    spaces: consistent
+    indent-sequences: consistent
+  key-duplicates:
+    ignore: |
+      config/testdata/section_key_dup.bad.yml
+  line-length: disable
+  truthy:
+    ignore: |
+      .github/workflows/codeql-analysis.yml
+      .github/workflows/funcbench.yml
+      .github/workflows/fuzzing.yml
+      .github/workflows/prombench.yml
+      .github/workflows/golangci-lint.yml

CHANGELOG.md

@@ -1,3 +1,12 @@
+## master / unreleased
+
+* [CHANGE] pg_stat_bgwriter counter metrics had the `_total` suffix added #556
+* [ENHANCEMENT] Add pg_database_size_bytes metric #613
+
+## 0.10.1 / 2022-01-14
+
+* [BUGFIX] Fix broken log-level for values other than debug. #560
+
 ## 0.10.0 / 2021-07-08
 
 * [ENHANCEMENT] Add ability to set included databases when autoDiscoverDatabases is enabled #499

CODE_OF_CONDUCT.md

@@ -1,3 +1,3 @@
-## Prometheus Community Code of Conduct
+# Prometheus Community Code of Conduct
 
-Prometheus follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
+Prometheus follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).

MAINTAINERS.md

@@ -1,2 +1,3 @@
 * Ben Kochie <[email protected]> @SuperQ
 * William Rouesnel <[email protected]> @wrouesnel
+* Joe Adams <[email protected]> @sysadmind

Makefile.common

@@ -78,17 +78,23 @@ ifneq ($(shell which gotestsum),)
 endif
 endif
 
-PROMU_VERSION ?= 0.12.0
+PROMU_VERSION ?= 0.13.0
 PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
 
 GOLANGCI_LINT :=
 GOLANGCI_LINT_OPTS ?=
-GOLANGCI_LINT_VERSION ?= v1.39.0
+GOLANGCI_LINT_VERSION ?= v1.45.2
 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
 # windows isn't included here because of the path separator being different.
 ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
 	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386))
-		GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint
+		# If we're in CI and there is an Actions file, that means the linter
+		# is being run in Actions, so we don't need to run it here.
+		ifeq (,$(CIRCLE_JOB))
+			GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint
+		else ifeq (,$(wildcard .github/workflows/golangci-lint.yml))
+			GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint
+		endif
 	endif
 endif
 
@@ -118,7 +124,7 @@ endif
 %: common-% ;
 
 .PHONY: common-all
-common-all: precheck style check_license lint unused build test
+common-all: precheck style check_license lint yamllint unused build test
 
 .PHONY: common-style
 common-style:
@@ -154,7 +160,7 @@ endif
 update-go-deps:
 	@echo ">> updating Go dependencies"
 	@for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \
-		$(GO) get $$m; \
+		$(GO) get -d $$m; \
 	done
 	GO111MODULE=$(GO111MODULE) $(GO) mod tidy
 ifneq (,$(wildcard vendor))
@@ -198,6 +204,15 @@ else
 endif
 endif
 
+.PHONY: common-yamllint
+common-yamllint:
+	@echo ">> running yamllint on all YAML files in the repository"
+ifeq (, $(shell which yamllint))
+	@echo "yamllint not installed so skipping"
+else
+	yamllint .
+endif
+
 # For backward-compatibility.
 .PHONY: common-staticcheck
 common-staticcheck: lint

README.md

@@ -7,7 +7,7 @@
 
 Prometheus exporter for PostgreSQL server metrics.
 
-CI Tested PostgreSQL versions: `9.4`, `9.5`, `9.6`, `10`, `11`, `12`, `13`
+CI Tested PostgreSQL versions: `9.4`, `9.5`, `9.6`, `10`, `11`, `12`, `13`, `14`
 
 ## Quick Start
 This package is available for Docker:
@@ -41,20 +41,26 @@ This will build the docker image as `prometheuscommunity/postgres_exporter:${bra
 * `help`
   Show context-sensitive help (also try --help-long and --help-man).
 
+* `collector.database`
+  Enable the pg_database collector. Default is `enabled`
+
+* `collector.bgwriter`
+  Enable the pg_stat_bgwriter collector. Default is `enabled`
+
 * `web.listen-address`
   Address to listen on for web interface and telemetry. Default is `:9187`.
 
 * `web.telemetry-path`
   Path under which to expose metrics. Default is `/metrics`.
 
 * `disable-default-metrics`
-  Use only metrics supplied from `queries.yaml` via `--extend.query-path`.
+  Use only metrics supplied from `queries.yaml` via `--extend.query-path`.  Default is `false`.
 
 * `disable-settings-metrics`
-  Use the flag if you don't want to scrape `pg_settings`.
+  Use the flag if you don't want to scrape `pg_settings`.  Default is `false`.
 
 * `auto-discover-databases`
-  Whether to discover the databases on a server dynamically.
+  Whether to discover the databases on a server dynamically.  Default is `false`.
 
 * `extend.query-path`
   Path to a YAML file containing custom queries to run. Check out [`queries.yaml`](queries.yaml)
@@ -186,8 +192,8 @@ The -extend.query-path command-line argument specifies a YAML file containing ad
 Some examples are provided in [queries.yaml](queries.yaml).
 
 ### Disabling default metrics
-To work with non-officially-supported postgres versions you can try disabling (e.g. 8.2.15)
-or a variant of postgres (e.g. Greenplum) you can disable the default metrics with the `--disable-default-metrics`
+To work with non-officially-supported postgres versions (e.g. 8.2.15),
+or variants of postgres (e.g. Greenplum), you can disable the default metrics with the `--disable-default-metrics`
 flag. This removes all built-in metrics, and uses only metrics defined by queries in the `queries.yaml` file you supply
 (so you must supply one, otherwise the exporter will return nothing but internal statuses and not your database).
 
@@ -203,14 +209,10 @@ If you want to include only subset of databases, you can use option `--include-d
 
 ### Running as non-superuser
 
-To be able to collect metrics from `pg_stat_activity` and `pg_stat_replication`
-as  non-superuser you have to create functions and views as a superuser, and
-assign permissions separately to those.
-
-In PostgreSQL, views run with the permissions of the user that created them so
-they can act as security barriers. Functions need to be created to share this
-data with the non-superuser. Only creating the views will leave out the most
-important bits of data.
+To be able to collect metrics from `pg_stat*` views as non-superuser in PostgreSQL
+server versions >= 10 you can grant the `pg_monitor` or `pg_read_all_stats` [built-in roles](https://www.postgresql.org/docs/current/predefined-roles.html) to the user. If
+you need to monitor older PostgreSQL servers, you will have to create functions
+and views as a superuser, and assign permissions separately to those.
 
 ```sql
 -- To use IF statements, hence to be able to check if the user exists before
@@ -239,9 +241,23 @@ ALTER USER postgres_exporter SET SEARCH_PATH TO postgres_exporter,pg_catalog;
 -- If deploying as non-superuser (for example in AWS RDS), uncomment the GRANT
 -- line below and replace <MASTER_USER> with your root user.
 -- GRANT postgres_exporter TO <MASTER_USER>;
+
+GRANT CONNECT ON DATABASE postgres TO postgres_exporter;
+```
+
+Run following command if you use PostgreSQL versions >= 10
+```sql
+GRANT pg_monitor to postgres_exporter;
+```
+
+Run following SQL commands only if you use PostgreSQL versions older than 10.
+In PostgreSQL, views run with the permissions of the user that created them so
+they can act as security barriers. Functions need to be created to share this
+data with the non-superuser. Only creating the views will leave out the most
+important bits of data.
+```sql
 CREATE SCHEMA IF NOT EXISTS postgres_exporter;
 GRANT USAGE ON SCHEMA postgres_exporter TO postgres_exporter;
-GRANT CONNECT ON DATABASE postgres TO postgres_exporter;
 
 CREATE OR REPLACE FUNCTION get_pg_stat_activity() RETURNS SETOF pg_stat_activity AS
 $$ SELECT * FROM pg_catalog.pg_stat_activity; $$
@@ -267,6 +283,7 @@ AS
 
 GRANT SELECT ON postgres_exporter.pg_stat_replication TO postgres_exporter;
 
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
 CREATE OR REPLACE FUNCTION get_pg_stat_statements() RETURNS SETOF pg_stat_statements AS
 $$ SELECT * FROM public.pg_stat_statements; $$
 LANGUAGE sql

SECURITY.md

@@ -3,4 +3,4 @@
 The Prometheus security policy, including how to report vulnerabilities, can be
 found here:
 
-https://prometheus.io/docs/operating/security/
+<https://prometheus.io/docs/operating/security/>

VERSION

@@ -1 +1 @@
-0.10.0
+0.10.1

cmd/postgres_exporter/datasource.go

@@ -21,7 +21,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 )
 

cmd/postgres_exporter/main.go

@@ -17,8 +17,9 @@ import (
 	"net/http"
 	"os"
 
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/prometheus-community/postgres_exporter/collector"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/common/promlog"
@@ -114,6 +115,17 @@ func main() {
 
 	prometheus.MustRegister(exporter)
 
+	pe, err := collector.NewPostgresCollector(
+		logger,
+		dsn,
+		[]string{},
+	)
+	if err != nil {
+		level.Error(logger).Log("msg", "Failed to create PostgresCollector", "err", err.Error())
+		os.Exit(1)
+	}
+	prometheus.MustRegister(pe)
+
 	http.Handle(*metricPath, promhttp.Handler())
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/html; charset=UTF-8") // nolint: errcheck

cmd/postgres_exporter/namespace.go

@@ -20,7 +20,7 @@ import (
 	"time"
 
 	"github.com/blang/semver"
-	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/log/level"
 	"github.com/lib/pq"
 	"github.com/prometheus/client_golang/prometheus"
 )