diff --git a/grafana/postgres/v13/global-health.json b/grafana/postgres/v13/global-health.json index a1e8c26497..488f95f3b3 100644 --- a/grafana/postgres/v13/global-health.json +++ b/grafana/postgres/v13/global-health.json @@ -58,7 +58,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT * FROM (\nSELECT\n dbname,\n max((data->>'longest_tx_seconds')::int) AS longest_tx_seconds\nFROM\n backends\nWHERE\n $__timeFilter(\"time\")\nGROUP BY\n dbname\n) x\nWHERE longest_tx_seconds > 0\nORDER BY 2 DESC\nLIMIT $top_limit", + "rawSql": "SELECT * FROM (\nSELECT\n dbname,\n max((data->>'longest_tx_seconds')::float) AS longest_tx_seconds\nFROM\n backends\nWHERE\n $__timeFilter(\"time\")\nGROUP BY\n dbname\n) x\nWHERE longest_tx_seconds > 0\nORDER BY 2 DESC\nLIMIT $top_limit", "select": [ [ { diff --git a/grafana/postgres/v13/recommendations.json b/grafana/postgres/v13/recommendations.json index 2f55296901..335980245b 100644 --- a/grafana/postgres/v13/recommendations.json +++ b/grafana/postgres/v13/recommendations.json @@ -284,7 +284,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n dbname AS \"Source\",\n (SELECT (data->>'value')::int8::text FROM settings_num s WHERE s.dbname = x.dbname AND s.tag_data->>'name' = 'server_version_num' ORDER BY time DESC LIMIT 1) AS \"PostgreSQL Version\",\n 'AUTOVACUUM Workers' AS \"Category\",\n 'If most / all autovacuum worker slots are busy or AV runs for hours then this could hint at incorrect AV params or too much workload' AS \"Recommendation\",\n 'Max. active autovacuum workers: ' || max_av_workers || ' (of total ' || (SELECT (data->>'value')::int FROM settings_num s WHERE s.dbname = x.dbname AND s.tag_data->>'name' = 'autovacuum_max_workers' ORDER BY s.time DESC LIMIT 1) || '); Max. AV duration (seconds): ' || longest_autovacuum_seconds AS \"Additional Info\"\nFROM (\nSELECT\n dbname,\n max((data->>'av_workers')::int) as max_av_workers,\n coalesce(max((data->>'longest_autovacuum_seconds')::int), 0) as longest_autovacuum_seconds\nFROM\n backends\nWHERE\n $__timeFilter(time)\n AND dbname IN ($dbname)\nGROUP BY\n dbname\n) x\nWHERE\n (max_av_workers > 1 AND max_av_workers::numeric / (SELECT (data->>'value')::int FROM settings_num s WHERE s.dbname = x.dbname AND s.tag_data->>'name' = 'autovacuum_max_workers' ORDER BY s.time DESC LIMIT 1) >= 0.6)\n OR\n (longest_autovacuum_seconds > 1800)", + "rawSql": "SELECT\n dbname AS \"Source\",\n (SELECT (data->>'value')::int8::text FROM settings_num s WHERE s.dbname = x.dbname AND s.tag_data->>'name' = 'server_version_num' ORDER BY time DESC LIMIT 1) AS \"PostgreSQL Version\",\n 'AUTOVACUUM Workers' AS \"Category\",\n 'If most / all autovacuum worker slots are busy or AV runs for hours then this could hint at incorrect AV params or too much workload' AS \"Recommendation\",\n 'Max. active autovacuum workers: ' || max_av_workers || ' (of total ' || (SELECT (data->>'value')::int FROM settings_num s WHERE s.dbname = x.dbname AND s.tag_data->>'name' = 'autovacuum_max_workers' ORDER BY s.time DESC LIMIT 1) || '); Max. AV duration (seconds): ' || longest_autovacuum_seconds AS \"Additional Info\"\nFROM (\nSELECT\n dbname,\n max((data->>'av_workers')::int) as max_av_workers,\n coalesce(max((data->>'longest_autovacuum_seconds')::float), 0) as longest_autovacuum_seconds\nFROM\n backends\nWHERE\n $__timeFilter(time)\n AND dbname IN ($dbname)\nGROUP BY\n dbname\n) x\nWHERE\n (max_av_workers > 1 AND max_av_workers::numeric / (SELECT (data->>'value')::int FROM settings_num s WHERE s.dbname = x.dbname AND s.tag_data->>'name' = 'autovacuum_max_workers' ORDER BY s.time DESC LIMIT 1) >= 0.6)\n OR\n (longest_autovacuum_seconds > 1800)", "select": [ [ { @@ -590,7 +590,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n dbname AS \"Source\",\n 'Performance Issue' AS \"Category\",\n 'Long Running Queries' AS \"Objects\",\n 'Long running queries block autovacuum and thereby cause bloat' AS \"Recommendation\",\n 'Longest query duration: '|| max((data->>'longest_query_seconds')::int)::text || ' seconds' AS \"Additional Info\"\nFROM\n backends\nWHERE\n $__timeFilter(time)\n AND dbname IN ($dbname)\n AND (data->>'longest_query_seconds')::int > 0\nGROUP BY\n dbname\nORDER BY\n max((data->>'longest_query_seconds')::int) DESC NULLS LAST, dbname", + "rawSql": "SELECT\n dbname AS \"Source\",\n 'Performance Issue' AS \"Category\",\n 'Long Running Queries' AS \"Objects\",\n 'Long running queries block autovacuum and thereby cause bloat' AS \"Recommendation\",\n 'Longest query duration: '|| max((data->>'longest_query_seconds')::float)::text || ' seconds' AS \"Additional Info\"\nFROM\n backends\nWHERE\n $__timeFilter(time)\n AND dbname IN ($dbname)\n AND (data->>'longest_query_seconds')::float > 0\nGROUP BY\n dbname\nORDER BY\n max((data->>'longest_query_seconds')::float) DESC NULLS LAST, dbname", "select": [ [ { diff --git a/grafana/postgres/v13/sessions-overview.json b/grafana/postgres/v13/sessions-overview.json index 7938bd8926..c3ca0a2706 100644 --- a/grafana/postgres/v13/sessions-overview.json +++ b/grafana/postgres/v13/sessions-overview.json @@ -651,7 +651,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_query_seconds')::int8) as longest_query,\n max((data->>'avg_query_seconds')::float) as avg_query\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_query_seconds')::float) as longest_query,\n max((data->>'avg_query_seconds')::float) as avg_query\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", "select": [ [ { @@ -817,7 +817,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_tx_seconds')::int8) as longest_tx,\n max((data->>'avg_tx_seconds')::float) as avg_tx\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_tx_seconds')::float) as longest_tx,\n max((data->>'avg_tx_seconds')::float) as avg_tx\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", "select": [ [ { @@ -972,6 +972,157 @@ } } }, + "panel-15": { + "kind": "Panel", + "spec": { + "id": 15, + "title": "Longest idle-in-transaction duration", + "description": "Seconds since the oldest idle-in-transaction client session (aborted ones included) entered that state. Such a session holds its snapshot and locks, so vacuum cannot remove dead rows.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-postgresql-datasource", + "version": "v0", + "datasource": { + "name": "pgwatch-metrics" + }, + "spec": { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_idle_in_tx_seconds')::float) as longest_idle_in_tx\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.1.0", + "spec": { + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "longest_idle_in_tx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-orange", + "mode": "fixed" + } + } + ] + } + ] + } + } + } + } + }, "panel-6": { "kind": "Panel", "spec": { @@ -998,7 +1149,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_autovacuum_seconds')::int8) as longest_autovacuum_seconds\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_autovacuum_seconds')::float) as longest_autovacuum_seconds\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", "select": [ [ { @@ -1205,7 +1356,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_waiting_seconds')::int8) as longest_waiting,\n max((data->>'avg_waiting_seconds')::float) as avg_waiting\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_waiting_seconds')::float) as longest_waiting,\n max((data->>'avg_waiting_seconds')::float) as avg_waiting\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", "select": [ [ { @@ -1372,7 +1523,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_session_seconds')::int8) as longest_session,\n max((data->>'avg_session_seconds')::float) as avg_session\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->>'longest_session_seconds')::float) as longest_session,\n max((data->>'avg_session_seconds')::float) as avg_session\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", "select": [ [ { @@ -1624,7 +1775,7 @@ "height": 6, "element": { "kind": "ElementReference", - "name": "panel-8" + "name": "panel-15" } } }, @@ -1637,7 +1788,7 @@ "height": 6, "element": { "kind": "ElementReference", - "name": "panel-6" + "name": "panel-8" } } }, @@ -1650,7 +1801,7 @@ "height": 6, "element": { "kind": "ElementReference", - "name": "panel-10" + "name": "panel-6" } } }, @@ -1661,6 +1812,19 @@ "y": 50, "width": 24, "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-10" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 56, + "width": 24, + "height": 6, "element": { "kind": "ElementReference", "name": "panel-11" diff --git a/grafana/prometheus/v13/sessions-overview-prometheus.json b/grafana/prometheus/v13/sessions-overview-prometheus.json index 14dbc3f81b..d0f09814a7 100644 --- a/grafana/prometheus/v13/sessions-overview-prometheus.json +++ b/grafana/prometheus/v13/sessions-overview-prometheus.json @@ -798,6 +798,61 @@ } } }, + "panel-10": { + "kind": "Panel", + "spec": { + "id": 10, + "title": "Longest idle-in-transaction duration", + "description": "Seconds since the oldest idle-in-transaction client session (aborted ones included) entered that state. Such a session holds its snapshot and locks, so vacuum cannot remove dead rows.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "pgwatch-prometheus" + }, + "spec": { + "exemplar": true, + "expr": "max(max_over_time(pgwatch_backends_longest_idle_in_tx_seconds{dbname='$dbname'}[$agg_interval]))", + "interval": "", + "legendFormat": "Longest idle in TX" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "interval": "$agg_interval" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "7.5.9", + "spec": { + "options": { + "alertThreshold": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + }, "panel-8": { "kind": "Panel", "spec": { @@ -1062,7 +1117,7 @@ "height": 9, "element": { "kind": "ElementReference", - "name": "panel-9" + "name": "panel-10" } } }, @@ -1075,7 +1130,7 @@ "height": 9, "element": { "kind": "ElementReference", - "name": "panel-6" + "name": "panel-9" } } }, @@ -1086,6 +1141,19 @@ "y": 54, "width": 24, "height": 9, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 63, + "width": 24, + "height": 9, "element": { "kind": "ElementReference", "name": "panel-8" diff --git a/pkg/metrics/metrics.yaml b/pkg/metrics/metrics.yaml index 8bd78b0cc9..3a57e6b429 100644 --- a/pkg/metrics/metrics.yaml +++ b/pkg/metrics/metrics.yaml @@ -77,7 +77,8 @@ metrics: This metric gathers detailed information from the PostgreSQL pg_stat_activity view, providing an overview of the database's current session and activity state. It tracks the total number of client backends, active sessions, idle sessions, sessions waiting on locks, and background workers. The metric also calculates statistics on blocked sessions, most extended waiting times, average and longest session durations, transaction times, - and query durations. Additionally, it monitors autovacuum worker activity and provides the age of the oldest transaction (measured by xmin). + and query durations. It also reports how long the oldest idle-in-transaction session, aborted ones included, has been idle since its last state change. + Additionally, it monitors autovacuum worker activity and provides the age of the oldest transaction (measured by xmin). This metric helps administrators monitor session states, detect bottlenecks, and ensure the system is within its connection limits, providing visibility into database performance and contention. sqls: @@ -96,6 +97,7 @@ metrics: (select count(*) from sa_snapshot where state = 'active' and backend_type = 'client backend') as active, (select count(*) from sa_snapshot where state = 'idle' and backend_type = 'client backend') as idle, (select count(*) from sa_snapshot where state = 'idle in transaction' and backend_type = 'client backend') as idleintransaction, + (select round(extract(epoch from max(now() - state_change)), 3)::float from sa_snapshot where state in ('idle in transaction', 'idle in transaction (aborted)') and backend_type = 'client backend') as longest_idle_in_tx_seconds, (select count(*) from sa_snapshot where wait_event_type in ('LWLock', 'Lock', 'BufferPin') and backend_type = 'client backend') as waiting, (select coalesce(sum(case when coalesce(array_length(pg_blocking_pids(pid), 1), 0) >= 1 then 1 else 0 end), 0) from sa_snapshot where backend_type = 'client backend' and state = 'active') as blocked, (select round(extract(epoch from max(now() - query_start)), 3)::float from sa_snapshot where wait_event_type in ('LWLock', 'Lock', 'BufferPin') and backend_type = 'client backend') as longest_waiting_seconds, diff --git a/pkg/metrics/metrics_yaml_test.go b/pkg/metrics/metrics_yaml_test.go index b71fa82f25..36f0567625 100644 --- a/pkg/metrics/metrics_yaml_test.go +++ b/pkg/metrics/metrics_yaml_test.go @@ -384,3 +384,46 @@ func TestBlockingLocksColumns(t *testing.T) { a.Regexp(re, sql, "column %q missing from the blocking_locks select list", col) } } + +// backendsColumns is the column list of backends. Dashboards query these +// names, so a rewrite of the SQL must keep every one of them. +var backendsColumns = []string{ + "epoch_ns", + "total", + "instance_total", + "max_connections", + "background_workers", + "active", + "idle", + "idleintransaction", + "longest_idle_in_tx_seconds", + "waiting", + "blocked", + "longest_waiting_seconds", + "avg_waiting_seconds", + "longest_session_seconds", + "avg_session_seconds", + "longest_tx_seconds", + "avg_tx_seconds", + "longest_autovacuum_seconds", + "longest_query_seconds", + "avg_query_seconds", + "max_xmin_age_tx", + "av_workers", +} + +func TestBackendsColumns(t *testing.T) { + a := assert.New(t) + m, ok := GetDefaultMetrics().MetricDefs["backends"] + if !a.True(ok, "backends must exist in metrics.yaml") { + return + } + sql, ok := m.SQLs[14] + if !a.True(ok, "backends must keep SQL key 14") { + return + } + for _, col := range backendsColumns { + re := regexp.MustCompile(`(?i)\bAS\s+` + col + `\b`) + a.Regexp(re, sql, "column %q missing from the backends select list", col) + } +}