Skip to content

Added n_tup_newpage_upd metrics (postgres_table_tuples_new_page_updated_total) #383

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cherts wants to merge 3 commits into
base: release/0.15
Choose a base branch
from
Open

Added n_tup_newpage_upd metrics (postgres_table_tuples_new_page_updated_total) #383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d2e23ef
Added n_tup_newpage_upd metrics (postgres_table_tuples_new_page_updat…
Aug 11, 2026
495fdd2
Fixed query
Aug 11, 2026
96a8fb1
Fixed test
Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 88 additions & 4 deletions internal/collector/postgres_tables.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
package collector

import (
"fmt"
"strconv"
"strings"

Expand All @@ -13,7 +14,7 @@ import (
)

const (
userTablesQuery = "WITH pg_locked AS (SELECT relation FROM pg_locks WHERE mode = 'AccessExclusiveLock' AND relation IS NOT NULL) " +
userTablesQuery15 = "WITH pg_locked AS (SELECT relation FROM pg_locks WHERE mode = 'AccessExclusiveLock' AND relation IS NOT NULL) " +
"SELECT current_database() AS database, s1.schemaname AS schema, s1.relname AS table, " +
"seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, " +
"n_live_tup, n_dead_tup, n_mod_since_analyze, " +
Expand All @@ -30,7 +31,24 @@ const (
"LEFT JOIN pg_locked l ON l.relation = s1.relid " +
"WHERE l.relation IS NULL"

userTablesQueryTopK = "WITH pg_locked AS (SELECT relation FROM pg_locks WHERE mode = 'AccessExclusiveLock' AND relation IS NOT NULL), " +
userTablesQueryLatest = "WITH pg_locked AS (SELECT relation FROM pg_locks WHERE mode = 'AccessExclusiveLock' AND relation IS NOT NULL) " +
"SELECT current_database() AS database, s1.schemaname AS schema, s1.relname AS table, " +
"seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, " +
"n_tup_newpage_upd, n_live_tup, n_dead_tup, n_mod_since_analyze, " +
"EXTRACT(EPOCH FROM AGE(now(), GREATEST(last_vacuum, last_autovacuum))) AS last_vacuum_seconds, " +
"EXTRACT(EPOCH FROM AGE(now(), GREATEST(last_analyze, last_autoanalyze))) AS last_analyze_seconds, " +
"EXTRACT(EPOCH FROM GREATEST(last_vacuum, last_autovacuum)) AS last_vacuum_time, " +
"EXTRACT(EPOCH FROM GREATEST(last_analyze, last_autoanalyze)) AS last_analyze_time, " +
"vacuum_count, autovacuum_count, analyze_count, autoanalyze_count, heap_blks_read, heap_blks_hit, idx_blks_read, " +
"idx_blks_hit, toast_blks_read, toast_blks_hit, tidx_blks_read, tidx_blks_hit, " +
"pg_table_size(s1.relid) AS size_bytes, reltuples " +
"FROM pg_stat_user_tables s1 " +
"JOIN pg_statio_user_tables s2 USING (schemaname, relname) " +
"JOIN pg_class c ON s1.relid = c.oid " +
"LEFT JOIN pg_locked l ON l.relation = s1.relid " +
"WHERE l.relation IS NULL"

userTablesQuery15TopK = "WITH pg_locked AS (SELECT relation FROM pg_locks WHERE mode = 'AccessExclusiveLock' AND relation IS NOT NULL), " +
"stat AS (SELECT s1.schemaname AS schema, s1.relname AS table, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, " +
"n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze, " +
"EXTRACT(EPOCH FROM AGE(now(), GREATEST(last_vacuum, last_autovacuum))) AS last_vacuum_seconds, " +
Expand Down Expand Up @@ -68,6 +86,46 @@ const (
"NULLIF(SUM(COALESCE(toast_blks_hit,0)),0), NULLIF(SUM(COALESCE(tidx_blks_read,0)),0), NULLIF(SUM(COALESCE(tidx_blks_hit, 0)),0), " +
"NULLIF(SUM(COALESCE(size_bytes,0)),0), NULLIF(SUM(COALESCE(reltuples,0)),0) FROM stat " +
"WHERE NOT visible HAVING EXISTS (SELECT 1 FROM stat WHERE NOT visible))"

userTablesQueryLatestTopK = "WITH pg_locked AS (SELECT relation FROM pg_locks WHERE mode = 'AccessExclusiveLock' AND relation IS NOT NULL), " +
"stat AS (SELECT s1.schemaname AS schema, s1.relname AS table, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, " +
"n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_tup_newpage_upd, n_live_tup, n_dead_tup, n_mod_since_analyze, " +
"EXTRACT(EPOCH FROM AGE(now(), GREATEST(last_vacuum, last_autovacuum))) AS last_vacuum_seconds, " +
"EXTRACT(EPOCH FROM AGE(now(), GREATEST(last_analyze, last_autoanalyze))) AS last_analyze_seconds, " +
"EXTRACT(EPOCH FROM GREATEST(last_vacuum, last_autovacuum)) AS last_vacuum_time, " +
"EXTRACT(EPOCH FROM GREATEST(last_analyze, last_autoanalyze)) AS last_analyze_time, " +
"vacuum_count, autovacuum_count, analyze_count, autoanalyze_count, heap_blks_read, heap_blks_hit, idx_blks_read, " +
"idx_blks_hit, toast_blks_read, toast_blks_hit, tidx_blks_read, tidx_blks_hit, pg_table_size(s1.relid) AS size_bytes, " +
"reltuples, (row_number() OVER (ORDER BY seq_scan DESC NULLS LAST) < $1) OR (row_number() OVER (ORDER BY seq_tup_read DESC NULLS LAST) < $1) OR " +
"(row_number() OVER (ORDER BY idx_scan DESC NULLS LAST) < $1) OR (row_number() OVER (ORDER BY idx_tup_fetch DESC NULLS LAST) < $1) OR " +
"(row_number() OVER (ORDER BY n_tup_ins DESC NULLS LAST) < $1) OR (row_number() OVER (ORDER BY n_tup_upd DESC NULLS LAST) < $1) OR " +
"(row_number() OVER (ORDER BY n_tup_del DESC NULLS LAST) < $1) OR (row_number() OVER (ORDER BY n_tup_hot_upd DESC NULLS LAST) < $1) OR " +
"(row_number() OVER (ORDER BY n_tup_newpage_upd DESC NULLS LAST) < $1) OR " +
"(row_number() OVER (ORDER BY n_live_tup DESC NULLS LAST) < $1) OR (row_number() OVER (ORDER BY n_dead_tup DESC NULLS LAST) < $1) OR " +
"(row_number() OVER (ORDER BY n_mod_since_analyze DESC NULLS LAST) < $1) OR (row_number() OVER (ORDER BY vacuum_count DESC NULLS LAST) < $1) OR " +
"(row_number() OVER (ORDER BY autovacuum_count DESC NULLS LAST) < $1) OR (row_number() OVER (ORDER BY analyze_count DESC NULLS LAST) < $1) OR " +
"(row_number() OVER (ORDER BY heap_blks_read DESC NULLS LAST) < $1) OR (row_number() OVER (ORDER BY idx_blks_hit DESC NULLS LAST) < $1) OR " +
"(row_number() OVER (ORDER BY toast_blks_read DESC NULLS LAST) < $1) OR (row_number() OVER (ORDER BY toast_blks_hit DESC NULLS LAST) < $1) OR " +
"(row_number() OVER (ORDER BY pg_table_size(s1.relid) DESC NULLS LAST) < $1) OR (row_number() OVER (ORDER BY reltuples DESC NULLS LAST) < $1) AS visible " +
"FROM pg_stat_user_tables s1 " +
"JOIN pg_statio_user_tables s2 USING (schemaname, relname) " +
"JOIN pg_class c ON s1.relid = c.oid " +
"LEFT JOIN pg_locked l ON l.relation = s1.relid " +
"WHERE l.relation IS NULL) " +
"SELECT current_database() AS database, schema, \"table\", seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, " +
"n_tup_hot_upd, n_tup_newpage_upd, n_live_tup, n_dead_tup, n_mod_since_analyze, last_vacuum_seconds, last_analyze_seconds, last_vacuum_time, last_analyze_time, " +
"vacuum_count, autovacuum_count, analyze_count, autoanalyze_count, heap_blks_read, heap_blks_hit, idx_blks_read, idx_blks_hit, toast_blks_read, " +
"toast_blks_hit, tidx_blks_read, tidx_blks_hit, size_bytes, reltuples FROM stat WHERE visible UNION ALL (SELECT current_database() AS database, " +
"'all_shemas', 'all_other_tables', NULLIF(SUM(COALESCE(seq_scan,0)),0), NULLIF(SUM(COALESCE(seq_tup_read,0)),0), NULLIF(SUM(COALESCE(idx_scan,0)),0), " +
"NULLIF(SUM(COALESCE(idx_tup_fetch,0)),0), NULLIF(SUM(COALESCE(n_tup_ins,0)),0), NULLIF(SUM(COALESCE(n_tup_upd,0)),0), " +
"NULLIF(SUM(COALESCE(n_tup_del,0)),0), NULLIF(SUM(COALESCE(n_tup_hot_upd,0)),0), NULLIF(SUM(COALESCE(n_tup_newpage_upd,0)),0), " +
"NULLIF(SUM(COALESCE(n_live_tup,0)),0), NULLIF(SUM(COALESCE(n_dead_tup,0)),0), NULLIF(SUM(COALESCE(n_mod_since_analyze,0)),0), NULL, NULL, NULL, NULL, " +
"NULLIF(SUM(COALESCE(vacuum_count,0)),0), NULLIF(SUM(COALESCE(autovacuum_count,0)),0), NULLIF(SUM(COALESCE(analyze_count,0)),0), " +
"NULLIF(SUM(COALESCE(autoanalyze_count,0)),0), NULLIF(SUM(COALESCE(heap_blks_read,0)),0), NULLIF(SUM(COALESCE(heap_blks_hit,0)),0), " +
"NULLIF(SUM(COALESCE(idx_blks_read,0)),0), NULLIF(SUM(COALESCE(idx_blks_hit,0)),0), NULLIF(SUM(COALESCE(toast_blks_read,0)),0), " +
"NULLIF(SUM(COALESCE(toast_blks_hit,0)),0), NULLIF(SUM(COALESCE(tidx_blks_read,0)),0), NULLIF(SUM(COALESCE(tidx_blks_hit, 0)),0), " +
"NULLIF(SUM(COALESCE(size_bytes,0)),0), NULLIF(SUM(COALESCE(reltuples,0)),0) FROM stat " +
"WHERE NOT visible HAVING EXISTS (SELECT 1 FROM stat WHERE NOT visible))"
)

// postgresTablesCollector defines metric descriptors and stats store.
Expand All @@ -79,6 +137,7 @@ type postgresTablesCollector struct {
tupInserted typedDesc
tupUpdated typedDesc
tupHotUpdated typedDesc
tupNewPageUpdated typedDesc
tupDeleted typedDesc
tupLive typedDesc
tupDead typedDesc
Expand Down Expand Up @@ -145,6 +204,12 @@ func NewPostgresTablesCollector(constLabels labels, settings model.CollectorSett
labels, constLabels,
settings.Filters,
),
tupNewPageUpdated: newBuiltinTypedDesc(
descOpts{"postgres", "table", "tuples_new_page_updated_total", "Total number of tuples (rows) have been updated where the successor version goes onto a new heap page, leaving behind an original version with a t_ctid field that points to a different heap page. These are always non-HOT updates.", 0},
prometheus.CounterValue,
labels, constLabels,
settings.Filters,
),
tupDeleted: newBuiltinTypedDesc(
descOpts{"postgres", "table", "tuples_deleted_total", "Total number of tuples (rows) have been deleted in the table.", 0},
prometheus.CounterValue,
Expand Down Expand Up @@ -233,9 +298,10 @@ func (c *postgresTablesCollector) Update(config Config, ch chan<- prometheus.Met
collect := func(conn *store.DB) error {
var res *model.PGResult
if config.CollectTopTable > 0 {
res, err = conn.Query(userTablesQueryTopK, config.CollectTopTable)
res, err = conn.Query(selectTableQuery(config.pgVersion.Numeric, config.CollectTopTable), config.CollectTopTable)

} else {
res, err = conn.Query(userTablesQuery)
res, err = conn.Query(selectTableQuery(config.pgVersion.Numeric, config.CollectTopTable))
}
if err != nil {
log.Warnf("get tables stat failed: %s; skip", err)
Expand All @@ -256,6 +322,7 @@ func (c *postgresTablesCollector) Update(config Config, ch chan<- prometheus.Met
ch <- c.tupUpdated.newConstMetric(stat.updated, stat.database, stat.schema, stat.table)
ch <- c.tupDeleted.newConstMetric(stat.deleted, stat.database, stat.schema, stat.table)
ch <- c.tupHotUpdated.newConstMetric(stat.hotUpdated, stat.database, stat.schema, stat.table)
ch <- c.tupNewPageUpdated.newConstMetric(stat.newPageUpdated, stat.database, stat.schema, stat.table)

// tuples total stats
ch <- c.tupLive.newConstMetric(stat.live, stat.database, stat.schema, stat.table)
Expand Down Expand Up @@ -369,6 +436,7 @@ type postgresTableStat struct {
updated float64
deleted float64
hotUpdated float64
newPageUpdated float64
live float64
dead float64
modified float64
Expand Down Expand Up @@ -455,6 +523,8 @@ func parsePostgresTableStats(r *model.PGResult, labelNames []string) map[string]
s.deleted = v
case "n_tup_hot_upd":
s.hotUpdated = v
case "n_tup_newpage_upd":
s.newPageUpdated = v
case "n_live_tup":
s.live = v
case "n_dead_tup":
Expand Down Expand Up @@ -507,3 +577,17 @@ func parsePostgresTableStats(r *model.PGResult, labelNames []string) map[string]

return stats
}

// selectTableQuery returns suitable table query depending on passed version.
func selectTableQuery(version int, topK int) string {
if version < PostgresV16 {
if topK > 0 {
return fmt.Sprintf(userTablesQuery15TopK)
}
return fmt.Sprintf(userTablesQuery15)
}
if topK > 0 {
return fmt.Sprintf(userTablesQueryLatestTopK)
}
return fmt.Sprintf(userTablesQueryLatest)
}
83 changes: 77 additions & 6 deletions internal/collector/postgres_tables_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ package collector

import (
"database/sql"
"github.com/jackc/pgx/v5/pgconn"
"fmt"
"testing"

"github.com/cherts/pgscv/internal/model"
"github.com/jackc/pgx/v5/pgconn"
"github.com/stretchr/testify/assert"
"testing"
)

func TestPostgresTablesCollector_Update(t *testing.T) {
Expand All @@ -18,6 +20,7 @@ func TestPostgresTablesCollector_Update(t *testing.T) {
"postgres_table_tuples_inserted_total",
"postgres_table_tuples_updated_total",
"postgres_table_tuples_hot_updated_total",
"postgres_table_tuples_new_page_updated_total",
"postgres_table_tuples_deleted_total",
"postgres_table_tuples_live_total",
"postgres_table_tuples_dead_total",
Expand Down Expand Up @@ -47,7 +50,7 @@ func Test_parsePostgresTableStats(t *testing.T) {
want map[string]postgresTableStat
}{
{
name: "normal output",
name: "normal output, Postgres 15",
res: &model.PGResult{
Nrows: 1,
Ncols: 32,
Expand Down Expand Up @@ -80,9 +83,56 @@ func Test_parsePostgresTableStats(t *testing.T) {
"testdb/testschema/testrelname": {
database: "testdb", schema: "testschema", table: "testrelname",
seqscan: 100, seqtupread: 1000, idxscan: 200, idxtupfetch: 2000,
inserted: 300, updated: 400, deleted: 500, hotUpdated: 150, live: 600, dead: 100, modified: 500,
lastvacuumAge: 700, lastanalyzeAge: 800, lastvacuumTime: 12345678, lastanalyzeTime: 87654321, vacuum: 910, autovacuum: 920, analyze: 930, autoanalyze: 940,
heapread: 4528, heaphit: 5845, idxread: 458, idxhit: 698, toastread: 125, toasthit: 825, tidxread: 699, tidxhit: 375,
inserted: 300, updated: 400, deleted: 500, hotUpdated: 150,
live: 600, dead: 100, modified: 500,
lastvacuumAge: 700, lastanalyzeAge: 800, lastvacuumTime: 12345678, lastanalyzeTime: 87654321,
vacuum: 910, autovacuum: 920, analyze: 930, autoanalyze: 940,
heapread: 4528, heaphit: 5845, idxread: 458, idxhit: 698,
toastread: 125, toasthit: 825, tidxread: 699, tidxhit: 375,
sizebytes: 458523, reltuples: 50000,
},
},
},
{
name: "normal output, Postgres 16",
res: &model.PGResult{
Nrows: 1,
Ncols: 33,
Colnames: []pgconn.FieldDescription{
{Name: "database"}, {Name: "schema"}, {Name: "table"},
{Name: "seq_scan"}, {Name: "seq_tup_read"}, {Name: "idx_scan"}, {Name: "idx_tup_fetch"},
{Name: "n_tup_ins"}, {Name: "n_tup_upd"}, {Name: "n_tup_del"}, {Name: "n_tup_hot_upd"},
{Name: "n_tup_newpage_upd"}, {Name: "n_live_tup"}, {Name: "n_dead_tup"}, {Name: "n_mod_since_analyze"},
{Name: "last_vacuum_seconds"}, {Name: "last_analyze_seconds"}, {Name: "last_vacuum_time"}, {Name: "last_analyze_time"},
{Name: "vacuum_count"}, {Name: "autovacuum_count"}, {Name: "analyze_count"}, {Name: "autoanalyze_count"},
{Name: "heap_blks_read"}, {Name: "heap_blks_hit"}, {Name: "idx_blks_read"}, {Name: "idx_blks_hit"},
{Name: "toast_blks_read"}, {Name: "toast_blks_hit"}, {Name: "tidx_blks_read"}, {Name: "tidx_blks_hit"},
{Name: "size_bytes"}, {Name: "reltuples"},
},
Rows: [][]sql.NullString{
{
{String: "testdb", Valid: true}, {String: "testschema", Valid: true}, {String: "testrelname", Valid: true},
{String: "100", Valid: true}, {String: "1000", Valid: true}, {String: "200", Valid: true}, {String: "2000", Valid: true},
{String: "300", Valid: true}, {String: "400", Valid: true}, {String: "500", Valid: true}, {String: "150", Valid: true},
{String: "10", Valid: true}, {String: "600", Valid: true}, {String: "100", Valid: true}, {String: "500", Valid: true},
{String: "700", Valid: true}, {String: "800", Valid: true}, {String: "12345678", Valid: true}, {String: "87654321", Valid: true},
{String: "910", Valid: true}, {String: "920", Valid: true}, {String: "930", Valid: true}, {String: "940", Valid: true},
{String: "4528", Valid: true}, {String: "5845", Valid: true}, {String: "458", Valid: true}, {String: "698", Valid: true},
{String: "125", Valid: true}, {String: "825", Valid: true}, {String: "699", Valid: true}, {String: "375", Valid: true},
{String: "458523", Valid: true}, {String: "50000", Valid: true},
},
},
},
want: map[string]postgresTableStat{
"testdb/testschema/testrelname": {
database: "testdb", schema: "testschema", table: "testrelname",
seqscan: 100, seqtupread: 1000, idxscan: 200, idxtupfetch: 2000,
inserted: 300, updated: 400, deleted: 500, hotUpdated: 150,
newPageUpdated: 10, live: 600, dead: 100, modified: 500,
lastvacuumAge: 700, lastanalyzeAge: 800, lastvacuumTime: 12345678, lastanalyzeTime: 87654321,
vacuum: 910, autovacuum: 920, analyze: 930, autoanalyze: 940,
heapread: 4528, heaphit: 5845, idxread: 458, idxhit: 698,
toastread: 125, toasthit: 825, tidxread: 699, tidxhit: 375,
sizebytes: 458523, reltuples: 50000,
},
},
Expand All @@ -96,3 +146,24 @@ func Test_parsePostgresTableStats(t *testing.T) {
})
}
}

func Test_selectTableQuery(t *testing.T) {
testcases := []struct {
version int
want string
topK int
}{
{version: PostgresV12, want: fmt.Sprintf(userTablesQuery15), topK: 0},
{version: PostgresV12, want: fmt.Sprintf(userTablesQuery15TopK), topK: 100},
{version: PostgresV15, want: fmt.Sprintf(userTablesQuery15), topK: 0},
{version: PostgresV15, want: fmt.Sprintf(userTablesQuery15TopK), topK: 100},
{version: PostgresV16, want: fmt.Sprintf(userTablesQueryLatest), topK: 0},
{version: PostgresV16, want: fmt.Sprintf(userTablesQueryLatestTopK), topK: 100},
{version: PostgresV18, want: fmt.Sprintf(userTablesQueryLatest), topK: 0},
{version: PostgresV18, want: fmt.Sprintf(userTablesQueryLatestTopK), topK: 100},
}

for _, tc := range testcases {
assert.Equal(t, tc.want, selectTableQuery(tc.version, tc.topK))
}
}
Loading