From 2615359e36f2605fe83925639897e186fe1bb3fb Mon Sep 17 00:00:00 2001 From: roaldm153 Date: Mon, 3 Aug 2026 18:09:27 +0500 Subject: [PATCH 01/17] feat(gpsc): apply runtime query-state core patches to tree Fold three PostgreSQL core changes the gp_stats_collector extension depends on directly into the sources: custom ProcSignal handlers (procsignal.c/.h, postgres.c), end-of-node instrumentation (instrument.c/.h), and runtime EXPLAIN entry points (explain.c/.h). configure enables the extension by default, so the tree must build without a manual patch step. --- src/backend/commands/explain.c | 156 ++++++++++++++++++++++----- src/backend/executor/instrument.c | 4 + src/backend/storage/ipc/procsignal.c | 106 ++++++++++++++++++ src/backend/tcop/postgres.c | 8 +- src/include/commands/explain.h | 2 + src/include/executor/instrument.h | 3 + src/include/storage/procsignal.h | 18 +++- 7 files changed, 270 insertions(+), 27 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 0d63d374128..99ff8ef747e 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -1312,15 +1312,37 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) Instrumentation *instr = rInfo->ri_TrigInstrument + nt; char *relname; char *conname = NULL; + instr_time starttimespan; + double total; + double ntuples; + double ncalls; + if (!es->runtime) + { /* Must clean up instrumentation state */ InstrEndLoop(instr); + } + + /* Collect statistic variables */ + if (!INSTR_TIME_IS_ZERO(instr->starttime)) + { + INSTR_TIME_SET_CURRENT(starttimespan); + INSTR_TIME_SUBTRACT(starttimespan, instr->starttime); + } + else + INSTR_TIME_SET_ZERO(starttimespan); + + total = instr->total + INSTR_TIME_GET_DOUBLE(instr->counter) + + INSTR_TIME_GET_DOUBLE(starttimespan); + ntuples = instr->ntuples + instr->tuplecount; + ncalls = ntuples + !INSTR_TIME_IS_ZERO(starttimespan); + /* * We ignore triggers that were never invoked; they likely aren't * relevant to the current query type. */ - if (instr->ntuples == 0) + if (ncalls == 0) continue; ExplainOpenGroup("Trigger", NULL, true, es); @@ -1345,10 +1367,10 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) if (show_relname) appendStringInfo(es->str, " on %s", relname); if (es->timing) - appendStringInfo(es->str, ": time=%.3f calls=%.ld\n", - 1000.0 * instr->total, instr->ntuples); + appendStringInfo(es->str, ": time=%.3f calls=%.0f\n", + 1000.0 * total, ncalls); else - appendStringInfo(es->str, ": calls=%.ld\n", instr->ntuples); + appendStringInfo(es->str, ": calls=%.0f\n", ncalls); } else { @@ -1357,9 +1379,8 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) ExplainPropertyText("Constraint Name", conname, es); ExplainPropertyText("Relation", relname, es); if (es->timing) - ExplainPropertyFloat("Time", "ms", 1000.0 * instr->total, 3, - es); - ExplainPropertyFloat("Calls", NULL, instr->ntuples, 0, es); + ExplainPropertyFloat("Time", "ms", 1000.0 * total, 3, es); + ExplainPropertyFloat("Calls", NULL, ncalls, 0, es); } if (conname) @@ -2259,8 +2280,11 @@ ExplainNode(PlanState *planstate, List *ancestors, * instrumentation results the user didn't ask for. But we do the * InstrEndLoop call anyway, if possible, to reduce the number of cases * auto_explain has to contend with. + * + * If flag es->stateinfo is set, i.e. when printing the current execution + * state, this step of cleaning up is missed. */ - if (planstate->instrument) + if (planstate->instrument && !es->runtime) InstrEndLoop(planstate->instrument); /* GPDB_90_MERGE_FIXME: In GPDB, these are printed differently. But does that work @@ -2297,7 +2321,7 @@ ExplainNode(PlanState *planstate, List *ancestors, ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); } } - else if (es->analyze) + else if (es->analyze && !es->runtime) { if (es->format == EXPLAIN_FORMAT_TEXT) appendStringInfoString(es->str, " (never executed)"); @@ -2313,6 +2337,75 @@ ExplainNode(PlanState *planstate, List *ancestors, } } + /* + * Print the progress of node execution at current loop. + */ + if (planstate->instrument && es->analyze && es->runtime) + { + instr_time starttimespan; + double startup_sec; + double total_sec; + double rows; + double loop_num; + bool finished; + + if (!INSTR_TIME_IS_ZERO(planstate->instrument->starttime)) + { + INSTR_TIME_SET_CURRENT(starttimespan); + INSTR_TIME_SUBTRACT(starttimespan, planstate->instrument->starttime); + } + else + INSTR_TIME_SET_ZERO(starttimespan); + startup_sec = 1000.0 * planstate->instrument->firsttuple; + total_sec = 1000.0 * (INSTR_TIME_GET_DOUBLE(planstate->instrument->counter) + + INSTR_TIME_GET_DOUBLE(starttimespan)); + rows = planstate->instrument->tuplecount; + loop_num = planstate->instrument->nloops + 1; + + finished = planstate->instrument->nloops > 0 + && !planstate->instrument->running + && INSTR_TIME_IS_ZERO(starttimespan); + + if (!finished) + { + ExplainOpenGroup("Current loop", "Current loop", true, es); + if (es->format == EXPLAIN_FORMAT_TEXT) + { + if (es->timing) + { + if (planstate->instrument->running) + appendStringInfo(es->str, + " (Current loop: actual time=%.3f..%.3f rows=%.0f, loop number=%.0f)", + startup_sec, total_sec, rows, loop_num); + else + appendStringInfo(es->str, + " (Current loop: running time=%.3f actual rows=0, loop number=%.0f)", + total_sec, loop_num); + } + else + appendStringInfo(es->str, + " (Current loop: actual rows=%.0f, loop number=%.0f)", + rows, loop_num); + } + else + { + ExplainPropertyFloat("Actual Loop Number", NULL, loop_num, 0, es); + if (es->timing) + { + if (planstate->instrument->running) + { + ExplainPropertyFloat("Actual Startup Time", NULL, startup_sec, 3, es); + ExplainPropertyFloat("Actual Total Time", NULL, total_sec, 3, es); + } + else + ExplainPropertyFloat("Running Time", NULL, total_sec, 3, es); + } + ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); + } + ExplainCloseGroup("Current loop", "Current loop", true, es); + } + } + /* in text format, first line ends here */ if (es->format == EXPLAIN_FORMAT_TEXT) appendStringInfoChar(es->str, '\n'); @@ -2867,8 +2960,9 @@ ExplainNode(PlanState *planstate, List *ancestors, if (es->wal && planstate->instrument) show_wal_usage(es, &planstate->instrument->walusage); - /* Prepare per-worker buffer/WAL usage */ - if (es->workers_state && (es->buffers || es->wal) && es->verbose) + /* Show worker detail after query execution */ + if (es->analyze && es->verbose && planstate->worker_instrument + && !es->runtime) { WorkerInstrumentation *w = planstate->worker_instrument; @@ -4005,6 +4099,11 @@ show_hash_info(HashState *hashstate, ExplainState *es) if (hashstate->hinstrument) memcpy(&hinstrument, hashstate->hinstrument, sizeof(HashInstrumentation)); + + if (hashstate->hashtable) + { + ExecHashAccumInstrumentation(&hinstrument, hashstate->hashtable); + } /* * Merge results from workers. In the parallel-oblivious case, the @@ -4396,21 +4495,16 @@ show_instrumentation_count(const char *qlabel, int which, if (!es->analyze || !planstate->instrument) return; - + nloops = planstate->instrument->nloops; if (which == 2) - nfiltered = planstate->instrument->nfiltered2; + nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered2 / nloops : 0); else - nfiltered = planstate->instrument->nfiltered1; + nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered1 / nloops : 0); nloops = planstate->instrument->nloops; /* In text mode, suppress zero counts; they're not interesting enough */ if (nfiltered > 0 || es->format != EXPLAIN_FORMAT_TEXT) - { - if (nloops > 0) - ExplainPropertyFloat(qlabel, NULL, nfiltered / nloops, 0, es); - else - ExplainPropertyFloat(qlabel, NULL, 0.0, 0, es); - } + ExplainPropertyFloat(qlabel, NULL, nfiltered, 0, es); } /* @@ -5068,15 +5162,27 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, double insert_path; double other_path; - InstrEndLoop(outerPlanState(mtstate)->instrument); + if (!es->runtime) + InstrEndLoop(outerPlanState(mtstate)->instrument); /* count the number of source rows */ - total = outerPlanState(mtstate)->instrument->ntuples; other_path = mtstate->ps.instrument->ntuples2; - insert_path = total - other_path; - ExplainPropertyFloat("Tuples Inserted", NULL, - insert_path, 0, es); + /* + * Insert occurs after extracting row from subplan and in runtime mode + * we can appear between these two operations - situation when + * total > insert_path + other_path. Therefore we don't know exactly + * whether last row from subplan is inserted. + * We don't print inserted tuples in runtime mode in order to not print + * inconsistent data + */ + if (!es->runtime) + { + total = outerPlanState(mtstate)->instrument->ntuples; + insert_path = total - other_path; + ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); + } + ExplainPropertyFloat("Conflicting Tuples", NULL, other_path, 0, es); } diff --git a/src/backend/executor/instrument.c b/src/backend/executor/instrument.c index 12561e0c051..e85013a4cd5 100644 --- a/src/backend/executor/instrument.c +++ b/src/backend/executor/instrument.c @@ -117,6 +117,9 @@ InstrStopNodeSync(Instrumentation *instr, uint64 nTuples) /* count the returned tuples */ instr->tuplecount += nTuples; + /* A zero-tuple stop means the node is exhausted for this cycle. */ + instr->eof = (nTuples == 0); + /* let's update the time only if the timer was requested */ if (instr->need_timer) { @@ -207,6 +210,7 @@ InstrEndLoop(Instrumentation *instr) /* Reset for next cycle (if any) */ instr->running = false; + instr->eof = false; INSTR_TIME_SET_ZERO(instr->starttime); INSTR_TIME_SET_ZERO(instr->counter); instr->firsttuple = 0; diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index 669b5465d73..fc9694bcc14 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -99,12 +99,20 @@ typedef struct #define BARRIER_CLEAR_BIT(flags, type) \ ((flags) &= ~(((uint32) 1) << (uint32) (type))) +#define IsCustomProcSignalReason(reason) \ + ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) + +static bool CustomSignalPendings[NUM_CUSTOM_PROCSIGNALS]; +static bool CustomSignalProcessing[NUM_CUSTOM_PROCSIGNALS]; +static ProcSignalHandler_type CustomInterruptHandlers[NUM_CUSTOM_PROCSIGNALS]; + static ProcSignalHeader *ProcSignal = NULL; static ProcSignalSlot *MyProcSignalSlot = NULL; static bool CheckProcSignal(ProcSignalReason reason); static void CleanupProcSignalState(int status, Datum arg); static void ResetProcSignalBarrierBits(uint32 flags); +static void CheckAndSetCustomSignalInterrupts(void); static bool ProcessBarrierPlaceholder(void); /* @@ -250,6 +258,40 @@ CleanupProcSignalState(int status, Datum arg) slot->pss_pid = 0; } +/* RegisterCustomProcSignalHandler + * Assign specific handler of custom process signal with new + * ProcSignalReason key. + * + * This function has to be called in _PG_init function of extensions at the + * stage of loading shared preloaded libraries. Otherwise it throws fatal error. + * + * Return INVALID_PROCSIGNAL if all slots for custom signals are occupied. + */ +ProcSignalReason +RegisterCustomProcSignalHandler(ProcSignalHandler_type handler) +{ + ProcSignalReason reason; + + + if (!process_shared_preload_libraries_in_progress) + { + ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cannot register custom signal after startup"))); + } + + /* Iterate through custom signal slots to find a free one */ + for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) + { + if (!CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1]) + { + CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1] = handler; + return reason; + } + } + + return INVALID_PROCSIGNAL; +} + /* * SendProcSignal * Send a signal to a Postgres process @@ -708,7 +750,71 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_FAILED_LOGIN)) HandleLoginFailed(); + CheckAndSetCustomSignalInterrupts(); + SetLatch(MyLatch); errno = save_errno; } + +/* + * Handle receipt of an interrupt indicating any of custom process signals. + */ +static void +CheckAndSetCustomSignalInterrupts() +{ + ProcSignalReason reason; + + for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) + { + if (CheckProcSignal(reason)) + { + /* set interrupt flags */ + InterruptPending = true; + CustomSignalPendings[reason - PROCSIG_CUSTOM_1] = true; + } + } + + SetLatch(MyLatch); +} + +/* + * CheckAndHandleCustomSignals + * Check custom signal flags and call handler assigned to that signal + * if it is not NULL + * + * This function is called within CHECK_FOR_INTERRUPTS if interrupt occurred. + */ +void +CheckAndHandleCustomSignals(void) +{ + int i; + + /* + * This is invoked from ProcessInterrupts(), and since some of the + * functions it calls contain CHECK_FOR_INTERRUPTS(), there is a potential + * for recursive calls if more signals are received while this runs, so + * let's block interrupts until done. + */ + HOLD_INTERRUPTS(); + + /* Check on expiring of custom signals and call its handlers if exist */ + for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) + { + if (!CustomSignalProcessing[i] && CustomSignalPendings[i]) + { + ProcSignalHandler_type handler; + + CustomSignalPendings[i] = false; + handler = CustomInterruptHandlers[i]; + if (handler != NULL) + { + CustomSignalProcessing[i] = true; + handler(); + CustomSignalProcessing[i] = false; + } + } + } + + RESUME_INTERRUPTS(); +} diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 6ae0202b396..fd1784fa015 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -4363,6 +4363,8 @@ ProcessInterrupts(const char* filename, int lineno) if (ParallelMessagePending) HandleParallelMessages(); + CheckAndHandleCustomSignals(); + if (LogMemoryContextPending) ProcessLogMemoryContextInterrupt(); } @@ -4778,7 +4780,7 @@ process_postgres_switches(int argc, char *argv[], GucContext ctx, * postmaster/postmaster.c (the option sets should not conflict) and with * the common help() function in main/main.c. */ - while ((flag = getopt(argc, argv, "B:bc:C:D:d:EeFf:h:ijk:lMm:N:nOPp:r:R:S:sTt:v:W:-:")) != -1) + while ((flag = getopt(argc, argv, "B:bc:C:D:d:EeFf:h:ijk:lMm:N:nOPp:r:R:S:sTt:v:W:Z-:")) != -1) { switch (flag) { @@ -4946,6 +4948,10 @@ process_postgres_switches(int argc, char *argv[], GucContext ctx, SetConfigOption("post_auth_delay", optarg, ctx, gucsource); break; + case 'Z': + /* ignored for consistency with the postmaster */ + break; + case 'c': case '-': { diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index 77cb96f0cab..1078460abd4 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -51,6 +51,8 @@ typedef struct ExplainState bool summary; /* print total planning and execution timing */ bool settings; /* print modified settings */ ExplainFormat format; /* output format */ + bool runtime; /* print intermediate state of query execution, + not after completion */ /* state for output formatting --- not reset for each new plan tree */ int indent; /* current indentation level */ List *grouping_stack; /* format-specific grouping state */ diff --git a/src/include/executor/instrument.h b/src/include/executor/instrument.h index 4536df3b237..974c315b46e 100644 --- a/src/include/executor/instrument.h +++ b/src/include/executor/instrument.h @@ -81,6 +81,9 @@ typedef struct Instrumentation bool prf_work; /* true if pushdown runtime filters really work */ /* Info about current plan cycle: */ bool running; /* true if we've completed first tuple */ + bool eof; /* true if the last fetch returned no tuple + * (node exhausted for this cycle); safe to read + * mid-run, unlike nloops/ntuples */ instr_time starttime; /* Start time of current iteration of node */ instr_time counter; /* Accumulated runtime for this node */ double firsttuple; /* Time for first tuple of this cycle */ diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index 0815460c72f..bc9efd6f878 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -15,7 +15,7 @@ #define PROCSIGNAL_H #include "storage/backendid.h" - +#define NUM_CUSTOM_PROCSIGNALS 64 /* * Reasons for signaling a Postgres child process (a backend or an auxiliary @@ -29,6 +29,8 @@ */ typedef enum { + INVALID_PROCSIGNAL = -1, /* Must be first */ + PROCSIG_CATCHUP_INTERRUPT, /* sinval catchup interrupt */ PROCSIG_NOTIFY_INTERRUPT, /* listen/notify interrupt */ PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ @@ -49,6 +51,14 @@ typedef enum PROCSIG_FAILED_LOGIN, /* failed login */ + PROCSIG_CUSTOM_1, + /* + * PROCSIG_CUSTOM_2, + * ..., + * PROCSIG_CUSTOM_N-1, + */ + PROCSIG_CUSTOM_N = PROCSIG_CUSTOM_1 + NUM_CUSTOM_PROCSIGNALS - 1, + NUM_PROCSIGNALS /* Must be last! */ } ProcSignalReason; @@ -62,6 +72,9 @@ typedef enum PROCSIGNAL_BARRIER_PLACEHOLDER = 0 } ProcSignalBarrierType; +/* Handler of custom process signal */ +typedef void (*ProcSignalHandler_type) (void); + /* * prototypes for functions in procsignal.c */ @@ -69,12 +82,15 @@ extern Size ProcSignalShmemSize(void); extern void ProcSignalShmemInit(void); extern void ProcSignalInit(int pss_idx); +extern ProcSignalReason +RegisterCustomProcSignalHandler(ProcSignalHandler_type handler); extern int SendProcSignal(pid_t pid, ProcSignalReason reason, BackendId backendId); extern uint64 EmitProcSignalBarrier(ProcSignalBarrierType type); extern void WaitForProcSignalBarrier(uint64 generation); extern void ProcessProcSignalBarrier(void); +extern void CheckAndHandleCustomSignals(void); extern void procsignal_sigusr1_handler(SIGNAL_ARGS); From 9398d6135953b2a965dc463df0cd6b88bbfff676 Mon Sep 17 00:00:00 2001 From: roaldm153 Date: Mon, 3 Aug 2026 18:09:27 +0500 Subject: [PATCH 02/17] feat(gpsc): add pg_query_state signal API with owner/superuser gate Signal-dispatch module and executor-lifecycle hooks that walk the live plan tree of a running backend on demand, plus the SQL API (extension v1.2): pg_query_state(pid), pg_query_state_backends(pid), cbdb_mpp_query_state(). Functions are granted to PUBLIC and guarded in C: a caller may poll a backend only if it is a superuser or owns the target query (GetUserId() == proc->roleId), so monitoring agents can run as a non-superuser while access stays restricted. --- gpcontrib/gp_stats_collector/Makefile | 1 + gpcontrib/gp_stats_collector/README.md | 22 + .../gp_stats_collector--1.1--1.2.sql | 45 + .../gp_stats_collector--1.2.sql | 155 +++ .../gp_stats_collector.control | 2 +- gpcontrib/gp_stats_collector/src/GpscStat.cpp | 111 +- .../src/gp_stats_collector.c | 7 + .../gp_stats_collector/src/hook_wrappers.cpp | 190 ++- .../src/pg_query_state/pg_query_state.c | 1019 +++++++++++++++++ .../src/pg_query_state/pg_query_state.h | 231 ++++ .../src/pg_query_state/qs_types.h | 73 ++ .../src/pg_query_state/signal_handler.c | 627 ++++++++++ 12 files changed, 2421 insertions(+), 62 deletions(-) create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector--1.1--1.2.sql create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector--1.2.sql create mode 100644 gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.c create mode 100644 gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.h create mode 100644 gpcontrib/gp_stats_collector/src/pg_query_state/qs_types.h create mode 100644 gpcontrib/gp_stats_collector/src/pg_query_state/signal_handler.c diff --git a/gpcontrib/gp_stats_collector/Makefile b/gpcontrib/gp_stats_collector/Makefile index b3228d2c45e..67126973cec 100644 --- a/gpcontrib/gp_stats_collector/Makefile +++ b/gpcontrib/gp_stats_collector/Makefile @@ -11,6 +11,7 @@ CPP_OBJS = $(patsubst %.cpp,%.o,$(wildcard src/*.cpp src/log/*.cpp src/memory/*. OBJS = $(C_OBJS) $(CPP_OBJS) $(PROTO_OBJS) PG_CXXFLAGS += -Werror -Wall -Wno-unused-but-set-variable -std=c++17 -Isrc/protos -Isrc -Iinclude -DGPBUILD +PG_CPPFLAGS += -I$(libpq_srcdir) -Isrc/protos -Isrc -Iinclude SHLIB_LINK += -lprotobuf -lstdc++ EXTRA_CLEAN = src/protos diff --git a/gpcontrib/gp_stats_collector/README.md b/gpcontrib/gp_stats_collector/README.md index 8c2d5c6868e..8fb5ba2ce84 100644 --- a/gpcontrib/gp_stats_collector/README.md +++ b/gpcontrib/gp_stats_collector/README.md @@ -45,3 +45,25 @@ An extension for collecting query execution metrics and reporting them to an ext - **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `gpsc.ignored_users_list`. - **Trimming plans:** Query texts and execution plans are trimmed based on `gpsc.max_text_size` and `gpsc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB. - **Analyze collection:** Analyze is sent if execution time exceeds `gpsc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `gpsc.enable_analyze` is true. + +### Runtime Query State (`pg_query_state`) + +On-demand inspection of the live execution state of another running backend. The target's active plan tree is walked across the coordinator (QD) and every segment (QE), collecting per-node instrumentation, without waiting for the query to finish. This is the signal-only variant: per-node samples are written to the server log rather than sent to the UDS sink. + +The functions live in the `gpsc` schema (extension version 1.2). + +#### 1. `pg_query_state(pid)` +- **What:** Triggers runtime per-node collection for the query running on backend `pid`. Fans a poll out to every participating QE and to the QD; each backend walks its plan tree and logs a per-node snapshot. Fire-and-forget: returns `void`. +- **GUC:** `pg_query_state.enable`. + +#### 2. `pg_query_state_backends(pid)` +- **What:** Lists the QE backends participating in the query running on backend `pid`, as `(segid, pid)` rows. Returns an empty set when the target is not running a query or has the module disabled. +- **GUC:** `pg_query_state.enable`. + +#### 3. `cbdb_mpp_query_state(gp_segment_pid[])` +- **What:** QE-side dispatch target used internally by `pg_query_state()`; not intended for direct use. + +### Runtime Query State Configuration +- **Enable:** `pg_query_state.enable` (default `on`) turns the executor hooks and signal handling on or off. Additional GUCs `pg_query_state.enable_timing` and `pg_query_state.enable_buffers` control the level of instrumentation collected. +- **Permissions:** The functions are granted to `PUBLIC`, but access is checked in the server: a caller may poll a backend only if it is a superuser or owns the target query. This lets monitoring agents run under a non-superuser role while still preventing one role from observing another's queries. +- **Preload:** The module registers custom signal handlers at startup, so `gp_stats_collector` must be listed in `shared_preload_libraries`. diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.1--1.2.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1--1.2.sql new file mode 100644 index 00000000000..bcb90ffc9f8 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1--1.2.sql @@ -0,0 +1,45 @@ +/* gp_stats_collector--1.1--1.2.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION gp_stats_collector UPDATE TO '1.2'" to load this file. \quit + +-- Compact (segid, pid) identifier for a QE backend running on a segment. +-- Matches the C gp_segment_pid struct used by the pg_query_state signal layer. +CREATE TYPE gpsc.gp_segment_pid AS ( + segid int, + pid int +); + +-- pg_query_state(pid): trigger runtime per-node collection for the query +-- running on backend `pid`. Fans QueryStatePollReason out to every QE via +-- cbdb_mpp_query_state; each matching QE walks its plan tree and logs a +-- per-node snapshot. Fire-and-forget: returns void. +CREATE FUNCTION gpsc.pg_query_state(pid int) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'pg_query_state' +LANGUAGE C VOLATILE EXECUTE ON COORDINATOR; + +-- cbdb_mpp_query_state(gp_segment_pid[]): dispatched verbatim to every segment +-- by pg_query_state() via CdbDispatchCommand; runs locally on each QE, so no +-- EXECUTE ON marker. Signals the matching local backends. +CREATE FUNCTION gpsc.cbdb_mpp_query_state(gpsc.gp_segment_pid[]) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'cbdb_mpp_query_state' +LANGUAGE C VOLATILE; + +-- pg_query_state_backends(pid): list the QE backends participating in the +-- query running on backend `pid`, as (segid, pid) rows. +CREATE FUNCTION gpsc.pg_query_state_backends(pid int) +RETURNS TABLE(segid int, pid int) +AS 'MODULE_PATHNAME', 'pg_query_state_backends' +LANGUAGE C VOLATILE EXECUTE ON COORDINATOR; + +-- The runtime query-state API is callable by any role; the per-backend +-- permission gate in C (superuser or the query's owner) enforces access, so +-- these can be granted broadly. This lets monitoring agents (e.g. yagpcc) run +-- under a non-superuser role. cbdb_mpp_query_state is dispatched to the QEs +-- under the caller's role, so it needs EXECUTE too. +GRANT USAGE ON SCHEMA gpsc TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.pg_query_state(int) TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.pg_query_state_backends(int) TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.cbdb_mpp_query_state(gpsc.gp_segment_pid[]) TO PUBLIC; \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.2.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.2.sql new file mode 100644 index 00000000000..0770416643b --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.2.sql @@ -0,0 +1,155 @@ +/* gp_stats_collector--1.2.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_stats_collector" to load this file. \quit + +CREATE SCHEMA gpsc; + +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.stat_messages_reset() +RETURNS SETOF void +AS +$$ + SELECT gpsc.__stat_messages_reset_f_on_master(); + SELECT gpsc.__stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW gpsc.stat_messages AS + SELECT C.* + FROM gpsc.__stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM gpsc.__stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; + +CREATE FUNCTION gpsc.__init_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__init_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +-- Creates log table inside gpsc schema. +SELECT gpsc.__init_log_on_master(); +SELECT gpsc.__init_log_on_segments(); + +CREATE VIEW gpsc.log AS + SELECT * FROM gpsc.__log -- master + UNION ALL + SELECT * FROM gp_dist_random('gpsc.__log') -- segments +ORDER BY tmid, ssid, ccnt; + +CREATE FUNCTION gpsc.__truncate_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__truncate_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.truncate_log() +RETURNS SETOF void AS $$ +BEGIN + PERFORM gpsc.__truncate_log_on_master(); + PERFORM gpsc.__truncate_log_on_segments(); +END; +$$ LANGUAGE plpgsql VOLATILE; + +CREATE FUNCTION gpsc.__test_uds_start_server(path text) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_test_uds_start_server' +LANGUAGE C STRICT EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__test_uds_receive(timeout_ms int DEFAULT 2000) +RETURNS SETOF bigint +AS 'MODULE_PATHNAME', 'gpsc_test_uds_receive' +LANGUAGE C STRICT EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__test_uds_stop_server() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_test_uds_stop_server' +LANGUAGE C EXECUTE ON COORDINATOR; + +-- --------------------------------------------------------------------------- +-- 1.2: pg_query_state per-node runtime collection (push to yagpcc via UDS) +-- --------------------------------------------------------------------------- + +-- Compact (segid, pid) identifier for a QE backend running on a segment. +-- Matches the C gp_segment_pid struct used by the pg_query_state signal layer. +CREATE TYPE gpsc.gp_segment_pid AS ( + segid int, + pid int +); + +-- pg_query_state(pid): trigger runtime per-node collection for the query +-- running on backend `pid`. Fans QueryStatePollReason out to every QE via +-- cbdb_mpp_query_state; each matching QE walks its plan tree and logs a +-- per-node snapshot. Fire-and-forget: returns void. +CREATE FUNCTION gpsc.pg_query_state(pid int) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'pg_query_state' +LANGUAGE C VOLATILE EXECUTE ON COORDINATOR; + +-- cbdb_mpp_query_state(gp_segment_pid[]): dispatched verbatim to every segment +-- by pg_query_state() via CdbDispatchCommand; runs locally on each QE, so no +-- EXECUTE ON marker. Signals the matching local backends. +CREATE FUNCTION gpsc.cbdb_mpp_query_state(gpsc.gp_segment_pid[]) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'cbdb_mpp_query_state' +LANGUAGE C VOLATILE; + +-- pg_query_state_backends(pid): list the QE backends participating in the +-- query running on backend `pid`, as (segid, pid) rows. +CREATE FUNCTION gpsc.pg_query_state_backends(pid int) +RETURNS TABLE(segid int, pid int) +AS 'MODULE_PATHNAME', 'pg_query_state_backends' +LANGUAGE C VOLATILE EXECUTE ON COORDINATOR; + +-- The runtime query-state API is callable by any role; the per-backend +-- permission gate in C (superuser or the query's owner) enforces access, so +-- these can be granted broadly. This lets monitoring agents (e.g. yagpcc) run +-- under a non-superuser role. cbdb_mpp_query_state is dispatched to the QEs +-- under the caller's role, so it needs EXECUTE too. +GRANT USAGE ON SCHEMA gpsc TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.pg_query_state(int) TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.pg_query_state_backends(int) TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.cbdb_mpp_query_state(gpsc.gp_segment_pid[]) TO PUBLIC; \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector.control b/gpcontrib/gp_stats_collector/gp_stats_collector.control index 4aea2bd49b8..76cf6c26e2b 100644 --- a/gpcontrib/gp_stats_collector/gp_stats_collector.control +++ b/gpcontrib/gp_stats_collector/gp_stats_collector.control @@ -1,5 +1,5 @@ # gp_stats_collector extension comment = 'Intercept query and plan execution hooks and report them to Cloudberry monitor agents' -default_version = '1.1' +default_version = '1.2' module_pathname = '$libdir/gp_stats_collector' superuser = true diff --git a/gpcontrib/gp_stats_collector/src/GpscStat.cpp b/gpcontrib/gp_stats_collector/src/GpscStat.cpp index 151cfd87c02..7e326e6b44a 100644 --- a/gpcontrib/gp_stats_collector/src/GpscStat.cpp +++ b/gpcontrib/gp_stats_collector/src/GpscStat.cpp @@ -40,19 +40,57 @@ extern "C" { namespace { + +/* + * ProtectedData -- spin-lock-protected wrapper around the GpscStat counters. + * + * Lives in a shared-memory segment so all backends on the same segment host + * contribute to the same counters. + */ struct ProtectedData { - slock_t mutex; + slock_t mutex; GpscStat::Data data; }; -shmem_startup_hook_type prev_shmem_startup_hook = NULL; -ProtectedData *data = nullptr; -void +static shmem_startup_hook_type prev_shmem_startup_hook = NULL; + +/* + * prev_shmem_request_hook is only relevant on PostgreSQL 15+, where the + * shmem request phase is separate from the startup phase. + */ +#if PG_VERSION_NUM >= 150000 +static shmem_request_hook_type prev_shmem_request_hook = NULL; + +/* + * gpsc_shmem_request -- request shared memory space. + * + * Installed as shmem_request_hook on PG15+. Chains to the previous hook + * before adding our own request. + */ +static void +gpsc_shmem_request() +{ + if (prev_shmem_request_hook) + prev_shmem_request_hook(); + RequestAddinShmemSpace(sizeof(ProtectedData)); +} +#endif /* PG_VERSION_NUM >= 150000 */ + +static ProtectedData *data = nullptr; + +/* + * gpsc_shmem_startup -- attach to (or initialise) the GpscStat shared segment. + * + * Installed as shmem_startup_hook. On first call (found == false) zeroes the + * counters and initialises the spin lock. Always chains to the previous hook. + */ +static void gpsc_shmem_startup() { if (prev_shmem_startup_hook) prev_shmem_startup_hook(); + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); bool found; data = reinterpret_cast( @@ -65,10 +103,16 @@ gpsc_shmem_startup() LWLockRelease(AddinShmemInitLock); } +/* + * LockGuard -- RAII wrapper around a SpinLock. + * + * Acquires the spin lock on construction and releases it on destruction, + * ensuring the lock is always released even if an exception is thrown. + */ class LockGuard { public: - LockGuard(slock_t *mutex) : mutex_(mutex) + explicit LockGuard(slock_t *mutex) : mutex_(mutex) { SpinLockAcquire(mutex_); } @@ -80,24 +124,51 @@ class LockGuard private: slock_t *mutex_; }; + } // namespace +/* + * GpscStat::init -- install shmem hooks during shared_preload_libraries phase. + * + * Must be called while process_shared_preload_libraries_in_progress is true. + * On PostgreSQL 14 and earlier, shared memory is requested here directly via + * RequestAddinShmemSpace(). On PostgreSQL 15+ a separate shmem_request_hook + * handles the request. + */ void GpscStat::init() { if (!process_shared_preload_libraries_in_progress) return; + +#if PG_VERSION_NUM >= 150000 + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = gpsc_shmem_request; +#else RequestAddinShmemSpace(sizeof(ProtectedData)); +#endif + prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = gpsc_shmem_startup; } +/* + * GpscStat::deinit -- restore shmem hooks to their previous values. + * + * Called from hooks_deinit(). + */ void GpscStat::deinit() { +#if PG_VERSION_NUM >= 150000 + shmem_request_hook = prev_shmem_request_hook; +#endif shmem_startup_hook = prev_shmem_startup_hook; } +/* + * GpscStat::reset -- zero all counters in the shared segment. + */ void GpscStat::reset() { @@ -105,6 +176,12 @@ GpscStat::reset() data->data = GpscStat::Data(); } +/* + * GpscStat::report_send -- record a successful message send. + * + * Parameters: + * msg_size -- size of the sent protobuf message in bytes + */ void GpscStat::report_send(int32_t msg_size) { @@ -114,6 +191,9 @@ GpscStat::report_send(int32_t msg_size) std::max(msg_size, data->data.max_message_size); } +/* + * GpscStat::report_bad_connection -- record a failed UDS connection attempt. + */ void GpscStat::report_bad_connection() { @@ -122,6 +202,12 @@ GpscStat::report_bad_connection() data->data.failed_connects++; } +/* + * GpscStat::report_bad_send -- record a failed send on an established connection. + * + * Parameters: + * msg_size -- size of the message that could not be sent + */ void GpscStat::report_bad_send(int32_t msg_size) { @@ -132,6 +218,9 @@ GpscStat::report_bad_send(int32_t msg_size) std::max(msg_size, data->data.max_message_size); } +/* + * GpscStat::report_error -- record any other error not covered by the above. + */ void GpscStat::report_error() { @@ -140,6 +229,12 @@ GpscStat::report_error() data->data.failed_other++; } +/* + * GpscStat::get_stats -- return a snapshot of all counters. + * + * The snapshot is taken under the spin lock and returned by value, so the + * caller sees a consistent view. + */ GpscStat::Data GpscStat::get_stats() { @@ -147,6 +242,12 @@ GpscStat::get_stats() return data->data; } +/* + * GpscStat::loaded -- return true when the shared segment has been mapped. + * + * Returns false before shmem_startup_hook has run (e.g. if the extension was + * not loaded via shared_preload_libraries). + */ bool GpscStat::loaded() { diff --git a/gpcontrib/gp_stats_collector/src/gp_stats_collector.c b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c index d295e37b396..0a9ef782c89 100644 --- a/gpcontrib/gp_stats_collector/src/gp_stats_collector.c +++ b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c @@ -31,6 +31,7 @@ #include "utils/builtins.h" #include "hook_wrappers.h" +#include "pg_query_state/pg_query_state.h" PG_MODULE_MAGIC; @@ -48,6 +49,12 @@ PG_FUNCTION_INFO_V1(gpsc_test_uds_stop_server); void _PG_init(void) { + /* + * Initialise the pg_query_state signal infrastructure unconditionally. + * It registers custom ProcSignal handlers and shared memory that must be + * set up during shared_preload_libraries processing. + */ + pg_qs_init(); if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) hooks_init(); } diff --git a/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp index 38ea117bda2..7ce9dbcbb68 100644 --- a/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp @@ -52,6 +52,7 @@ extern "C" { #include "GpscStat.h" #include "hook_wrappers.h" #include "memory/gpdbwrappers.h" +#include "pg_query_state/pg_query_state.h" static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; static ExecutorRun_hook_type previous_ExecutorRun_hook = nullptr; @@ -95,16 +96,24 @@ static char *test_sock_path = NULL; static EventSender *sender = nullptr; +/* + * get_sender -- lazily construct the per-backend EventSender instance. + */ static inline EventSender * get_sender() { if (!sender) - { sender = new EventSender(); - } return sender; } +/* + * cpp_call -- invoke a C++ member function, converting exceptions to ereport. + * + * Wraps obj->*func(args...) in a try/catch so that C++ exceptions thrown + * inside EventSender methods are converted to PostgreSQL ERROR instead of + * crashing the backend with an unhandled exception. + */ template R cpp_call(T *obj, R (T::*func)(Args...), Args... args) @@ -115,11 +124,18 @@ cpp_call(T *obj, R (T::*func)(Args...), Args... args) } catch (const std::exception &e) { - ereport(ERROR, (errmsg("Unexpected exception in gpsc %s", e.what()))); + ereport(ERROR, (errmsg("Unexpected exception in gpsc: %s", e.what()))); pg_unreachable(); } } +/* + * hooks_init -- install all executor and utility hooks. + * + * Called from _PG_init() for QD and QE roles. Initialises the GUC registry, + * the GpscStat shared-memory statistics counters, and the stat-statements + * parser, then chains each hook onto the existing hook pointer. + */ void hooks_init() { @@ -148,6 +164,11 @@ hooks_init() ProcessUtility_hook = gpsc_process_utility_hook; } +/* + * hooks_deinit -- restore all hooks to their previous values. + * + * Called from _PG_fini(). Cleans up the EventSender and GpscStat resources. + */ void hooks_deinit() { @@ -166,32 +187,46 @@ hooks_deinit() if (sender) { delete sender; + sender = nullptr; } GpscStat::deinit(); ProcessUtility_hook = previous_ProcessUtility_hook; } +/* + * gpsc_ExecutorStart_hook -- wrapper for ExecutorStart. + * + * Notifies pg_query_state of the new query (enables instrumentation) and + * calls the EventSender before and after the actual ExecutorStart. + */ void gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { + pg_qs_executor_start(query_desc, eflags); + cpp_call(get_sender(), &EventSender::executor_before_start, query_desc, eflags); + if (previous_ExecutorStart_hook) - { (*previous_ExecutorStart_hook)(query_desc, eflags); - } else - { standard_ExecutorStart(query_desc, eflags); - } + cpp_call(get_sender(), &EventSender::executor_after_start, query_desc, eflags); } +/* + * gpsc_ExecutorRun_hook -- wrapper for ExecutorRun. + * + * Pushes the QueryDesc onto the pg_query_state stack so signal handlers can + * find it, then pops it after the run (or on error). + */ void gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, uint64 count, bool execute_once) { + pg_qs_executor_run(query_desc); get_sender()->incr_depth(); PG_TRY(); { @@ -201,18 +236,27 @@ gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, else standard_ExecutorRun(query_desc, direction, count, execute_once); get_sender()->decr_depth(); + pg_qs_pop_query(); } PG_CATCH(); { get_sender()->decr_depth(); + pg_qs_pop_query(); PG_RE_THROW(); } PG_END_TRY(); } +/* + * gpsc_ExecutorFinish_hook -- wrapper for ExecutorFinish. + * + * Same push/pop pattern as ExecutorRun; keeps the QueryDesc visible to + * signal handlers during the finish phase. + */ void gpsc_ExecutorFinish_hook(QueryDesc *query_desc) { + pg_qs_executor_finish(query_desc); get_sender()->incr_depth(); PG_TRY(); { @@ -221,71 +265,88 @@ gpsc_ExecutorFinish_hook(QueryDesc *query_desc) else standard_ExecutorFinish(query_desc); get_sender()->decr_depth(); + pg_qs_pop_query(); } PG_CATCH(); { get_sender()->decr_depth(); + pg_qs_pop_query(); PG_RE_THROW(); } PG_END_TRY(); } +/* + * gpsc_ExecutorEnd_hook -- wrapper for ExecutorEnd. + * + * Notifies pg_query_state that the query is finishing (triggers the final + * plan-tree walk and LOG dump), then calls the EventSender and the standard + * end function. + */ void gpsc_ExecutorEnd_hook(QueryDesc *query_desc) { + pg_qs_executor_end(query_desc); cpp_call(get_sender(), &EventSender::executor_end, query_desc); if (previous_ExecutorEnd_hook) - { (*previous_ExecutorEnd_hook)(query_desc); - } else - { standard_ExecutorEnd(query_desc); - } } +/* + * gpsc_query_info_collect_hook -- wrapper for query_info_collect_hook. + */ void gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg) { cpp_call(get_sender(), &EventSender::query_metrics_collect, status, arg /* queryDesc */, false /* utility */, (ErrorData *) NULL); if (previous_query_info_collect_hook) - { (*previous_query_info_collect_hook)(status, arg); - } } #ifdef IC_TEARDOWN_HOOK +/* + * gpsc_ic_teardown_hook -- wrapper for ic_teardown_hook. + * + * Collects interconnect metrics when the motion layer tears down. + */ void gpsc_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { cpp_call(get_sender(), &EventSender::ic_metrics_collect); if (previous_ic_teardown_hook) - { (*previous_ic_teardown_hook)(transportStates, hasErrors); - } } #endif #ifdef ANALYZE_STATS_COLLECT_HOOK +/* + * gpsc_analyze_stats_collect_hook -- wrapper for analyze_stats_collect_hook. + */ void gpsc_analyze_stats_collect_hook(QueryDesc *query_desc) { cpp_call(get_sender(), &EventSender::analyze_stats_collect, query_desc); if (previous_analyze_stats_collect_hook) - { (*previous_analyze_stats_collect_hook)(query_desc); - } } #endif +/* + * gpsc_process_utility_hook -- wrapper for ProcessUtility_hook. + * + * Constructs a minimal QueryDesc from the utility statement to reuse the + * existing EventSender::query_metrics_collect interface. The QueryDesc is + * freed in both the success and error paths. + */ static void gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc) { - /* Project utility data on QueryDesc to use existing logic */ QueryDesc *query_desc = (QueryDesc *) palloc0(sizeof(QueryDesc)); query_desc->sourceText = queryString; @@ -297,31 +358,26 @@ gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, PG_TRY(); { if (previous_ProcessUtility_hook) - { (*previous_ProcessUtility_hook)(pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); - } else - { standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); - } get_sender()->decr_depth(); cpp_call(get_sender(), &EventSender::query_metrics_collect, METRICS_QUERY_DONE, (void *) query_desc, true /* utility */, (ErrorData *) NULL); - pfree(query_desc); } PG_CATCH(); { - ErrorData *edata; - MemoryContext oldctx; + ErrorData *edata; + MemoryContext oldctx; oldctx = MemoryContextSwitchTo(TopMemoryContext); - edata = CopyErrorData(); + edata = CopyErrorData(); FlushErrorState(); MemoryContextSwitchTo(oldctx); @@ -329,24 +385,31 @@ gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, cpp_call(get_sender(), &EventSender::query_metrics_collect, METRICS_QUERY_ERROR, (void *) query_desc, true /* utility */, edata); - pfree(query_desc); ReThrowError(edata); } PG_END_TRY(); } +/* + * check_stats_loaded -- raise ERROR if GpscStat shared memory is not mapped. + * + * Called by SQL-callable stat functions to guard against use before the + * extension was loaded via shared_preload_libraries. + */ static void check_stats_loaded() { if (!GpscStat::loaded()) - { - ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("gp_stats_collector must be loaded via " - "shared_preload_libraries"))); - } + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("gp_stats_collector must be loaded via " + "shared_preload_libraries"))); } +/* + * gpsc_functions_reset -- reset all GpscStat counters to zero. + */ void gpsc_functions_reset() { @@ -354,6 +417,13 @@ gpsc_functions_reset() GpscStat::reset(); } +/* + * gpsc_functions_get -- return the current GpscStat counters as a tuple. + * + * Returns one row with columns: + * segid, total_messages, send_failures, connection_failures, + * other_errors, max_message_size. + */ Datum gpsc_functions_get(FunctionCallInfo fcinfo) { @@ -361,21 +431,15 @@ gpsc_functions_get(FunctionCallInfo fcinfo) check_stats_loaded(); auto stats = GpscStat::get_stats(); TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "segid", INT4OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber) 2, "total_messages", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "send_failures", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "connection_failures", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber) 5, "other_errors", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber) 6, "max_message_size", INT4OID, - -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "segid", INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "total_messages", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "send_failures", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "connection_failures", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "other_errors", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "max_message_size", INT4OID, -1, 0); tupdesc = BlessTupleDesc(tupdesc); Datum values[ATTNUM]; - bool nulls[ATTNUM]; + bool nulls[ATTNUM]; MemSet(nulls, 0, sizeof(nulls)); values[0] = Int32GetDatum(GpIdentity.segindex); values[1] = Int64GetDatum(stats.total); @@ -384,10 +448,12 @@ gpsc_functions_get(FunctionCallInfo fcinfo) values[4] = Int64GetDatum(stats.failed_other); values[5] = Int32GetDatum(stats.max_message_size); HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); - Datum result = HeapTupleGetDatum(tuple); - PG_RETURN_DATUM(result); + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); } +/* + * test_uds_stop_server -- close and remove the test Unix-domain socket server. + */ void test_uds_stop_server() { @@ -404,13 +470,19 @@ test_uds_stop_server() } } +/* + * test_uds_start_server -- create a listening Unix-domain socket at path. + * + * Intended for integration tests that verify the UDS connector end-to-end. + * Raises ERROR if the socket cannot be created or bound. + */ void test_uds_start_server(const char *path) { struct sockaddr_un addr = {.sun_family = AF_UNIX}; if (strlen(path) >= sizeof(addr.sun_path)) - ereport(ERROR, (errmsg("path too long"))); + ereport(ERROR, (errmsg("Unix socket path too long"))); test_uds_stop_server(); @@ -423,20 +495,27 @@ test_uds_start_server(const char *path) listen(test_server_fd, TEST_MAX_CONNECTIONS) < 0) { test_uds_stop_server(); - ereport(ERROR, (errmsg("socket setup failed: %m"))); + ereport(ERROR, (errmsg("test UDS socket setup failed: %m"))); } } +/* + * test_uds_receive -- accept one connection and count bytes received. + * + * Polls for an incoming connection with a deadline of timeout_ms milliseconds. + * Returns the total number of bytes received, or 0 on timeout. + * Raises ERROR on poll/accept failures. + */ int64 test_uds_receive(int timeout_ms) { - char buf[TEST_RCV_BUF_SIZE]; - int rc; + char buf[TEST_RCV_BUF_SIZE]; + int rc; struct pollfd pfd = {.fd = test_server_fd, .events = POLLIN}; - int64 total = 0; + int64 total = 0; if (test_server_fd < 0) - ereport(ERROR, (errmsg("server not started"))); + ereport(ERROR, (errmsg("test UDS server not started"))); for (;;) { @@ -453,7 +532,7 @@ test_uds_receive(int timeout_ms) if (pfd.revents & POLLIN) { - int client = accept(test_server_fd, NULL, NULL); + int client = accept(test_server_fd, NULL, NULL); ssize_t n; if (client < 0) @@ -466,9 +545,8 @@ test_uds_receive(int timeout_ms) else if (errno != EINTR) break; } - close(client); } return total; -} \ No newline at end of file +} diff --git a/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.c b/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.c new file mode 100644 index 00000000000..f048edc5fcf --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.c @@ -0,0 +1,1019 @@ +/* + * pg_query_state.c + * Core of the pg_query_state signal-dispatch layer. + * + * This module provides: + * - Shared-memory setup (shm_toc segment with params, mq, mq_req_id). + * - Custom ProcSignal registrations for three signals: + * QueryStatePollReason -> SendQueryState() + * UserIdPollReason -> SendCurrentUserId() + * BackendInfoPollReason -> SendCdbComponents() + * - GUC variables: pg_query_state.enable / enable_timing / enable_buffers. + * - Executor lifecycle hooks (start/run/finish/end) that maintain the + * QueryDescStack and enable instrumentation on the top-level query. + * - A requestor-side helper: shm_mq_receive_with_timeout(). + * + * This is the "signal-only" variant: it does NOT push plan-node data to any + * upstream sink (no UDS, no protobuf). The executor_end hook merely walks + * the plan tree and writes a LOG entry for debugging. + * + * + * IDENTIFICATION + * gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.c + */ + +#include "pg_query_state.h" + +#include "access/htup_details.h" +#include "access/xact.h" +#include "catalog/pg_type.h" +#include "cdb/cdbdispatchresult.h" +#include "cdb/cdbdisp_query.h" +#include "cdb/cdbexplain.h" +#include "cdb/cdbvars.h" +#include "executor/execParallel.h" +#include "executor/executor.h" +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "nodes/nodeFuncs.h" +#include "nodes/print.h" +#include "parser/analyze.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "storage/ipc.h" +#include "storage/s_lock.h" +#include "storage/spin.h" +#include "storage/procarray.h" +#include "storage/procsignal.h" +#include "storage/shm_toc.h" +#include "utils/guc.h" +#include "utils/timestamp.h" +#include "utils/lsyscache.h" +#include "utils/portal.h" +#include "utils/typcache.h" + +#define TEXT_CSTR_CMP(text, cstr) \ + (memcmp(VARDATA(text), (cstr), VARSIZE(text) - VARHDRSZ)) + +/* GUC variables */ +/* Master switch: disabling this suppresses all stat collection. */ +bool pg_qs_enable = true; + +/* Collect timing (wall-clock) data in addition to row counts. */ +bool pg_qs_timing = false; + +/* Collect buffer usage via Instrumentation.bufusage. */ +bool pg_qs_buffers = true; + +/* + * Rolling counter incremented for every QueryDesc pushed onto the stack. + * Used to generate synthetic queryId values for statements lacking one. + */ +static int qs_query_count = 0; + +/* Saved hook pointer for chaining shmem_startup callbacks. */ +static shmem_startup_hook_type prev_shmem_startup_hook = NULL; + +/* Whether pg_qs_shmem_startup has completed successfully. */ +static bool module_initialized = false; + +/* + * Monotonically increasing request counter on the requestor side. + * Compared against *mq_req_id in the reply to detect stale responses. + */ +static int reqid = 0; + +/* Shared-memory variables (pointers into the shm_toc segment) */ +/* Table of contents anchoring the whole shared segment. */ +static shm_toc *toc = NULL; + +/* + * Signal parameters written by the requestor and read by the handler. + * Slot 0 in the toc. + */ +pg_qs_params *params = NULL; + +/* + * Raw shared memory queue used to return data from the handler. + * Slot 1 in the toc. + */ +shm_mq *mq = NULL; + +/* + * Shared request-id counter. The requestor increments it before sending a + * signal; the handler echoes it back so the requestor can detect stale + * replies. Slot 2 in the toc. + */ +uint32 *mq_req_id = NULL; + +/* Global signal-reason handles (set during pg_qs_init) */ +List *QueryDescStack = NIL; + +ProcSignalReason UserIdPollReason = INVALID_PROCSIGNAL; +ProcSignalReason QueryStatePollReason = INVALID_PROCSIGNAL; +ProcSignalReason BackendInfoPollReason = INVALID_PROCSIGNAL; + +/* Forward declarations for module-private helpers */ +static Size pg_qs_shmem_size(void); +static void pg_qs_shmem_startup(void); +static void push_query(QueryDesc *queryDesc); + +static List *get_query_backend_info(ArrayType *array); + +static shm_mq_result receive_msg_by_parts(shm_mq_handle *mqh, Size *total, + void **datap, int64 timeout, + int *rc, bool nowait); +static PG_QS_RequestResult GetRemoteBackendInfo(PGPROC *proc, List **result); +static void CollectQEQueryState(List *backendInfo); + +#if PG_VERSION_NUM >= 150000 +static shmem_request_hook_type prev_shmem_request_hook = NULL; +static void pg_qs_shmem_request(void); +#endif + +/* + * pg_qs_shmem_size -- compute the size of the shared memory segment. + * + * The segment holds three objects at fixed toc keys: + * key 0: pg_qs_params + * key 1: message queue of QUEUE_SIZE bytes + * key 2: uint32 request-id counter + */ +static Size +pg_qs_shmem_size(void) +{ + shm_toc_estimator e; + Size size; + int nkeys = 3; + + shm_toc_initialize_estimator(&e); + shm_toc_estimate_chunk(&e, sizeof(pg_qs_params)); + shm_toc_estimate_chunk(&e, (Size) QUEUE_SIZE); + shm_toc_estimate_chunk(&e, sizeof(uint32)); + shm_toc_estimate_keys(&e, nkeys); + size = shm_toc_estimate(&e); + return size; +} + +/* + * pg_qs_shmem_startup -- attach to (or initialize) the shared segment. + * + * Called from the shmem_startup_hook chain after shared memory is mapped. + * On first call (found == false) it initialises all sub-structures. + * On subsequent calls it just re-attaches the toc pointers. + */ +static void +pg_qs_shmem_startup(void) +{ + bool found; + Size shmem_size = pg_qs_shmem_size(); + void *shmem; + int num_toc = 0; + + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + shmem = ShmemInitStruct("pg_query_state", shmem_size, &found); + if (!found) + { + toc = shm_toc_create(PG_QS_MODULE_KEY, shmem, shmem_size); + + params = shm_toc_allocate(toc, sizeof(pg_qs_params)); + shm_toc_insert(toc, num_toc++, params); + + mq = shm_toc_allocate(toc, QUEUE_SIZE); + shm_toc_insert(toc, num_toc++, mq); + + mq_req_id = shm_toc_allocate(toc, sizeof(uint32)); + shm_toc_insert(toc, num_toc++, mq_req_id); + *mq_req_id = 0; + } + else + { + toc = shm_toc_attach(PG_QS_MODULE_KEY, shmem); + params = shm_toc_lookup(toc, num_toc++, false); + mq = shm_toc_lookup(toc, num_toc++, false); + mq_req_id = shm_toc_lookup(toc, num_toc++, false); + } + LWLockRelease(AddinShmemInitLock); + + if (prev_shmem_startup_hook) + prev_shmem_startup_hook(); + + module_initialized = true; +} + +#if PG_VERSION_NUM >= 150000 +/* + * pg_qs_shmem_request -- hook called to request shared memory space. + * + * PostgreSQL 15+ separates the request phase from the startup phase. + * This hook is installed only when building against PG15+. + */ +static void +pg_qs_shmem_request(void) +{ + if (prev_shmem_request_hook) + prev_shmem_request_hook(); + + RequestAddinShmemSpace(pg_qs_shmem_size()); +} +#endif + +/* + * pg_qs_init -- initialise the pg_query_state signal infrastructure. + * + * Must be called from _PG_init() while process_shared_preload_libraries_in_progress + * is true. Registers shared memory, custom ProcSignal handlers and GUC + * variables. Safe to call unconditionally for all roles. + */ +void +pg_qs_init(void) +{ + if (!process_shared_preload_libraries_in_progress) + return; + +#if PG_VERSION_NUM >= 150000 + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = pg_qs_shmem_request; +#else + RequestAddinShmemSpace(pg_qs_shmem_size()); +#endif + + UserIdPollReason = RegisterCustomProcSignalHandler(SendCurrentUserId); + QueryStatePollReason = RegisterCustomProcSignalHandler(SendQueryState); + BackendInfoPollReason = RegisterCustomProcSignalHandler(SendCdbComponents); + + if (QueryStatePollReason == INVALID_PROCSIGNAL || + BackendInfoPollReason == INVALID_PROCSIGNAL || + UserIdPollReason == INVALID_PROCSIGNAL) + { + ereport(WARNING, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("pg_query_state isn't loaded: insufficient custom ProcSignal slots"))); + return; + } + + DefineCustomBoolVariable("pg_query_state.enable", + "Enable module.", + NULL, + &pg_qs_enable, + true, + PGC_SUSET, + 0, + NULL, NULL, NULL); + + DefineCustomBoolVariable("pg_query_state.enable_timing", + "Collect timing data, not just row counts.", + NULL, + &pg_qs_timing, + false, + PGC_SUSET, + 0, + NULL, NULL, NULL); + + DefineCustomBoolVariable("pg_query_state.enable_buffers", + "Collect buffer usage.", + NULL, + &pg_qs_buffers, + true, + PGC_SUSET, + 0, + NULL, NULL, NULL); + + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = pg_qs_shmem_startup; + + elog(LOG, "pg_query_state: signal infrastructure initialised"); +} + +/* Executor lifecycle hooks */ +/* + * pg_qs_executor_start -- called at the start of executor execution. + * + * Enables instrumentation on the QueryDesc when: + * - pg_query_state is enabled + * - this is not an EXPLAIN-only execution + * - we are on a QD or QE role + * - there is no outer query already on the stack (top-level only) + * - the query passes the filter + * - no showstatctx is already attached + * + * Also assigns a synthetic queryId when the planner left it as zero. + * + * Parameters: + * queryDesc -- the QueryDesc being started + * eflags -- executor flags (EXEC_FLAG_EXPLAIN_ONLY etc.) + */ +void +pg_qs_executor_start(QueryDesc *queryDesc, int eflags) +{ + instr_time starttime; + + if (pg_qs_enable + && ((eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) + && (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) + && is_querystack_empty() + && filter_query(queryDesc) + && queryDesc->showstatctx == NULL) + { + queryDesc->instrument_options |= INSTRUMENT_CDB; + queryDesc->instrument_options |= INSTRUMENT_ROWS; + if (pg_qs_timing) + queryDesc->instrument_options |= INSTRUMENT_TIMER; + if (pg_qs_buffers) + queryDesc->instrument_options |= INSTRUMENT_BUFFERS; + + INSTR_TIME_SET_CURRENT(starttime); + queryDesc->showstatctx = + cdbexplain_showExecStatsBegin(queryDesc, starttime); + queryDesc->totaltime = InstrAlloc(1, INSTRUMENT_ALL, false); + } + + if (queryDesc->plannedstmt->queryId == 0) + queryDesc->plannedstmt->queryId = + ((uint64) gp_command_count << 32) + qs_query_count; +} + +/* + * pg_qs_executor_run -- called when the executor begins fetching tuples. + * + * Pushes the QueryDesc onto the stack so signal handlers can find it. + */ +void +pg_qs_executor_run(QueryDesc *queryDesc) +{ + push_query(queryDesc); +} + +/* + * pg_qs_executor_finish -- called after all tuples have been fetched. + * + * Pushes the QueryDesc again to keep the stack consistent during the finish + * phase (needed so signal handlers still see the query during cleanup). + */ +void +pg_qs_executor_finish(QueryDesc *queryDesc) +{ + push_query(queryDesc); +} + +/* + * pg_qs_executor_end -- called when executor resources are released. + * + * Walks the plan tree with instrumentation finalized (InstrEndLoop) and writes + * the collected per-node stats to the server log. + */ +void +pg_qs_executor_end(QueryDesc *queryDesc) +{ + QsWalkerContext *qs_walker_ctx; + + if (!queryDesc) + return; + + qs_walker_ctx = (QsWalkerContext *) palloc0(sizeof(QsWalkerContext)); + qs_walker_ctx->finalize = true; + qs_planstate_walker(queryDesc->planstate, qs_get_node_stats, + qs_walker_ctx, 0); + qs_debug_node_stats(qs_walker_ctx->per_node_stats); +} + +/* + * push_query -- add a QueryDesc to the top of the stack. + * + * Also increments qs_query_count for synthetic queryId generation. + */ +static void +push_query(QueryDesc *queryDesc) +{ + qs_query_count++; + QueryDescStack = lcons(queryDesc, QueryDescStack); +} + +/* + * pg_qs_push_query -- public alias for push_query, called from hook_wrappers. + */ +void +pg_qs_push_query(QueryDesc *queryDesc) +{ + qs_query_count++; + QueryDescStack = lcons(queryDesc, QueryDescStack); +} + +/* + * pg_qs_pop_query -- remove the most-recently-pushed QueryDesc from the stack. + */ +void +pg_qs_pop_query(void) +{ + QueryDescStack = list_delete_first(QueryDescStack); +} + +/* + * is_querystack_empty -- return true when no query is currently executing. + */ +bool +is_querystack_empty(void) +{ + return list_length(QueryDescStack) == 0; +} + +/* + * get_toppest_query -- return the most-recently-pushed QueryDesc, or NULL. + */ +QueryDesc * +get_toppest_query(void) +{ + return (QueryDescStack == NIL) ? NULL : (QueryDesc *) llast(QueryDescStack); +} + +/* + * filter_query -- decide whether to instrument a given QueryDesc. + * + * Returns false for cursor queries with non-default cursor options, and for + * utility statements. Returns true for SELECT, INSERT, UPDATE, DELETE. + */ +bool +filter_query(QueryDesc *queryDesc) +{ + Portal portal; + + if (queryDesc == NULL) + return false; + + if (queryDesc->extended_query && queryDesc->portal_name) + { + portal = GetPortalByName(queryDesc->portal_name); + if (portal->cursorOptions != CURSOR_OPT_NO_SCROLL) + return false; + } + + return (queryDesc->operation == CMD_SELECT || + queryDesc->operation == CMD_DELETE || + queryDesc->operation == CMD_INSERT || + queryDesc->operation == CMD_UPDATE); +} + +/* + * wait_for_mq_detached -- spin until the caller has attached to the mq or + * MAX_SND_TIMEOUT milliseconds elapses. + * + * Returns true if the queue was detached within the timeout (i.e. the other + * end is done), false on timeout. + */ +bool +wait_for_mq_detached(shm_mq_handle *mqh) +{ + instr_time start_time; + instr_time cur_time; + int64 delay = MAX_SND_TIMEOUT; + + INSTR_TIME_SET_CURRENT(start_time); + for (;;) + { + if (shm_mq_wait_for_attach(mqh) == SHM_MQ_DETACHED) + break; + WaitLatch(MyLatch, + WL_LATCH_SET | WL_EXIT_ON_PM_DEATH | WL_TIMEOUT, + delay, PG_WAIT_IPC); + INSTR_TIME_SET_CURRENT(cur_time); + INSTR_TIME_SUBTRACT(cur_time, start_time); + delay = MAX_SND_TIMEOUT - (int64) INSTR_TIME_GET_MILLISEC(cur_time); + if (delay <= 0) + { + elog(WARNING, "pg_query_state: wait_for_mq_detached timed out"); + return false; + } + CHECK_FOR_INTERRUPTS(); + } + return true; +} + +/* + * LockShmem -- acquire an exclusive user-lock keyed by (PG_QS_MODULE_KEY, key). + * + * Used to serialise access to the shared mq between concurrent requestors + * and between requestor and handler. + */ +void +LockShmem(LOCKTAG *tag, uint32 key) +{ + LockAcquireResult result; + + tag->locktag_field1 = PG_QS_MODULE_KEY; + tag->locktag_field2 = key; + tag->locktag_field3 = 0; + tag->locktag_field4 = 0; + tag->locktag_type = LOCKTAG_USERLOCK; + tag->locktag_lockmethodid = USER_LOCKMETHOD; + + result = LockAcquire(tag, ExclusiveLock, false, false); + Assert(result == LOCKACQUIRE_OK); +} + +/* + * UnlockShmem -- release the exclusive user-lock acquired by LockShmem. + */ +void +UnlockShmem(LOCKTAG *tag) +{ + LockRelease(tag, ExclusiveLock, false); +} + +/* + * GetRemoteBackendInfo -- obtain the list of (segid, pid) pairs from QD. + * + * Sends BackendInfoPollReason to proc and waits for the reply. On success, + * *result is populated with gp_segment_pid entries (palloc'd). + * + * Returns the PG_QS_RequestResult code from the reply. + */ +static PG_QS_RequestResult +GetRemoteBackendInfo(PGPROC *proc, List **result) +{ + int sig_result; + shm_mq_handle *mqh; + shm_mq_result mq_receive_result; + Size msg_len; + backend_info *msg; + LOCKTAG tag; + int i; + + LockShmem(&tag, PG_QS_SND_KEY); + params->reason = BackendInfoPollReason; + mq = shm_mq_create(mq, QUEUE_SIZE); + shm_mq_set_sender(mq, proc); + shm_mq_set_receiver(mq, MyProc); + *mq_req_id = reqid; + UnlockShmem(&tag); + + sig_result = SendProcSignal(proc->pid, BackendInfoPollReason, + proc->backendId); + if (sig_result == -1) + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not send BackendInfoPollReason signal"))); + + mqh = shm_mq_attach(mq, NULL, NULL); + mq_receive_result = shm_mq_receive_with_timeout(mqh, &msg_len, + (void **) &msg, + MAX_RCV_TIMEOUT); + + if (mq_receive_result != SHM_MQ_SUCCESS || msg == NULL || + msg->reqid != (uint32) reqid) + { + shm_mq_detach(mqh); + ereport(WARNING, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("GetRemoteBackendInfo: message not received"))); + return QUERY_NOT_RUNNING; + } + + if (msg->result_code != QS_RETURNED) + { + PG_QS_RequestResult result_code = msg->result_code; + shm_mq_detach(mqh); + return result_code; + } + + { + int expected_len = BASE_SIZEOF_GP_BACKEND_INFO + + msg->number * sizeof(gp_segment_pid); + if ((int) msg_len != expected_len) + { + shm_mq_detach(mqh); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("GetRemoteBackendInfo: unexpected message length"))); + } + } + + for (i = 0; i < msg->number; i++) + { + gp_segment_pid *segpid = palloc(sizeof(gp_segment_pid)); + *segpid = msg->pids[i]; + *result = lcons(segpid, *result); + } + + shm_mq_detach(mqh); + return QS_RETURNED; +} + +/* + * CollectQEQueryState -- fan-out query-state signals to all QE backends. + * + * Dispatches a cbdb_mpp_query_state() call to each segment listed in + * backendInfo. Results are returned as raw CdbPgResults. + */ +static void +CollectQEQueryState(List *backendInfo) +{ + ListCell *lc; + int index = 0; + StringInfoData params_buf; + char *sql; + + if (list_length(backendInfo) == 0) + return; + + initStringInfo(¶ms_buf); + + foreach(lc, backendInfo) + { + gp_segment_pid *segpid = (gp_segment_pid *) lfirst(lc); + index++; + appendStringInfo(¶ms_buf, "'(%d,%d)'", segpid->segid, segpid->pid); + if (index != list_length(backendInfo)) + appendStringInfoChar(¶ms_buf, ','); + } + + sql = psprintf("SELECT gpsc.cbdb_mpp_query_state((ARRAY[%s])::gpsc.gp_segment_pid[])", + params_buf.data); + + CdbDispatchCommand(sql, DF_NONE, NULL); + pfree(params_buf.data); + pfree(sql); +} + +/* + * shm_mq_receive_with_timeout -- receive from mqh, blocking up to `timeout` ms. + * + * Calls receive_msg_by_parts() in a loop, sleeping on the latch between + * retries. Returns SHM_MQ_SUCCESS, SHM_MQ_DETACHED, or SHM_MQ_WOULD_BLOCK + * (the last meaning the timeout expired). + * + * On success, *nbytesp is set to the message length and *datap to a palloc'd + * buffer containing the message. + */ +shm_mq_result +shm_mq_receive_with_timeout(shm_mq_handle *mqh, + Size *nbytesp, + void **datap, + int64 timeout) +{ + int rc = 0; + int64 delay = timeout; + instr_time start_time; + instr_time cur_time; + + INSTR_TIME_SET_CURRENT(start_time); + + for (;;) + { + shm_mq_result result; + + result = receive_msg_by_parts(mqh, nbytesp, datap, timeout, &rc, true); + if (result != SHM_MQ_WOULD_BLOCK) + return result; + + if (rc & WL_TIMEOUT || delay <= 0) + return SHM_MQ_WOULD_BLOCK; + + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_EXIT_ON_PM_DEATH | WL_TIMEOUT, + delay, PG_WAIT_EXTENSION); + + INSTR_TIME_SET_CURRENT(cur_time); + INSTR_TIME_SUBTRACT(cur_time, start_time); + delay = timeout - (int64) INSTR_TIME_GET_MILLISEC(cur_time); + if (delay <= 0) + return SHM_MQ_WOULD_BLOCK; + + CHECK_FOR_INTERRUPTS(); + ResetLatch(MyLatch); + } +} + +/* + * receive_msg_by_parts -- reassemble a multi-chunk message from mqh. + * + * The wire protocol prefixes each message with its total byte count (a Size), + * followed by one or more chunks of up to MSG_MAX_SIZE bytes. This function + * reads the prefix, allocates a buffer, and loops until all chunks arrive. + * + * Parameters: + * mqh -- attached message-queue handle + * total -- out: total bytes received + * datap -- out: palloc'd buffer with reassembled message + * timeout -- caller's deadline in ms (used only for PART_RCV_DELAY retries) + * rc -- out: WaitLatch flags (set to WL_TIMEOUT if we give up) + * nowait -- passed through to shm_mq_receive + */ +static shm_mq_result +receive_msg_by_parts(shm_mq_handle *mqh, Size *total, void **datap, + int64 timeout, int *rc, bool nowait) +{ + shm_mq_result mq_receive_result; + shm_mq_msg *buff; + int offset; + Size *expected; + Size expected_data; + Size len; + + /* Read the length prefix. */ + mq_receive_result = shm_mq_receive(mqh, &len, (void **) &expected, nowait); + if (mq_receive_result != SHM_MQ_SUCCESS) + return mq_receive_result; + Assert(len == sizeof(Size)); + + expected_data = *expected; + *datap = palloc0(expected_data); + + /* Reassemble chunks until we have expected_data bytes. */ + for (offset = 0; offset < (int) expected_data; ) + { + int64 delay = timeout; + + for (;;) + { + mq_receive_result = shm_mq_receive(mqh, &len, (void **) &buff, + nowait); + if (mq_receive_result != SHM_MQ_SUCCESS) + { + if (nowait && mq_receive_result == SHM_MQ_WOULD_BLOCK) + { + if (delay > 0) + { + pg_usleep(PART_RCV_DELAY * 1000); + delay -= PART_RCV_DELAY; + continue; + } + if (rc) + *rc |= WL_TIMEOUT; + } + return mq_receive_result; + } + break; + } + memcpy((char *) *datap + offset, buff, len); + offset += len; + } + + *total = offset; + return mq_receive_result; +} + +/* SQL callable functions */ +/* + * pg_query_state -- entry point for the pg_query_state() SQL function. + * + * Obtains the user-id and segment-backend list from the target backend, + * then fans out cbdb_mpp_query_state() to each QE. + */ +PG_FUNCTION_INFO_V1(pg_query_state); +Datum +pg_query_state(PG_FUNCTION_ARGS) +{ + pid_t pid = PG_GETARG_INT32(0); + PGPROC *proc; + LOCKTAG tag; + PG_QS_RequestResult result; + List *backend_info = NIL; + Oid counterpart_user_id; + + if (pid == MyProcPid) + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot extract state of current process"))); + + if (!module_initialized) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("pg_query_state must be loaded via shared_preload_libraries"))); + + proc = BackendPidGetProc(pid); + if (!proc || proc->backendId == InvalidBackendId || + proc->databaseId == InvalidOid || proc->roleId == InvalidOid) + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("backend with pid=%d not found", pid))); + + counterpart_user_id = proc->roleId; + if (!(superuser() || GetUserId() == counterpart_user_id)) + { + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied"))); + } + + LockShmem(&tag, PG_QS_RCV_KEY); + reqid = *mq_req_id + 1; + result = GetRemoteBackendInfo(proc, &backend_info); + UnlockShmem(&tag); + + switch (result) + { + case QUERY_NOT_RUNNING: + elog(DEBUG1, "pg_query_state: pid=%d is not running a query", pid); + break; + + case STAT_DISABLED: + elog(DEBUG1, "pg_query_state: stats collection disabled"); + break; + + case QS_RETURNED: + /* Signal all segment QEs to push their plan-node stats via UDS. */ + CollectQEQueryState(backend_info); + + /* + * Signal the QD backend itself so it pushes coordinator-side plan + * nodes and the plan-doc. SendQueryState() emits directly via UDS. + */ + SendProcSignal(proc->pid, QueryStatePollReason, proc->backendId); + break; + } + + PG_RETURN_VOID(); +} + +/* + * pg_query_state_backends -- list the QE backends participating in the query + * running on backend `pid`. + * + * Returns a set of (segid, pid) rows obtained from the coordinator via + * GetRemoteBackendInfo (the same list the poll path fans out to). A consumer + * can use the row count as the expected number of backends that will report. + * + * Uses the materialize SRF mode: the whole list is built into a tuplestore in + * one call. Returns an empty set when the target query is not running. + */ +PG_FUNCTION_INFO_V1(pg_query_state_backends); +Datum +pg_query_state_backends(PG_FUNCTION_ARGS) +{ + pid_t pid = PG_GETARG_INT32(0); + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + PGPROC *proc; + List *backend_info = NIL; + LOCKTAG tag; + PG_QS_RequestResult info_result; + ListCell *lc; + Oid counterpart_user_id; + + /* Standard set-returning-function materialize-mode preamble. */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not allowed in this context"))); + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("function returning record called in context that cannot accept type record"))); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + MemoryContextSwitchTo(oldcontext); + + if (pid == MyProcPid) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot extract state of current process"))); + + proc = BackendPidGetProc(pid); + if (!proc || proc->backendId == InvalidBackendId || + proc->databaseId == InvalidOid || proc->roleId == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("backend with pid=%d not found", pid))); + + counterpart_user_id = proc->roleId; + if (!(superuser() || GetUserId() == counterpart_user_id)) + { + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied"))); + } + + if (!module_initialized) + { + UnlockShmem(&tag); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("pg_query_state must be loaded via shared_preload_libraries"))); + } + + LockShmem(&tag, PG_QS_RCV_KEY); + reqid = *mq_req_id + 1; + info_result = GetRemoteBackendInfo(proc, &backend_info); + UnlockShmem(&tag); + + /* Not running / disabled: return an empty set rather than erroring. */ + if (info_result != QS_RETURNED) + return (Datum) 0; + + foreach(lc, backend_info) + { + gp_segment_pid *segpid = (gp_segment_pid *) lfirst(lc); + Datum values[2]; + bool nulls[2] = {false, false}; + + values[0] = Int32GetDatum(segpid->segid); + values[1] = Int32GetDatum(segpid->pid); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + + return (Datum) 0; +} + +/* + * cbdb_mpp_query_state -- QE-side entry point dispatched by CollectQEQueryState. + * + * Receives an array of gp_segment_pid, filters those belonging to this + * segment, and fires QueryStatePollReason at each matching backend. + */ +PG_FUNCTION_INFO_V1(cbdb_mpp_query_state); +Datum +cbdb_mpp_query_state(PG_FUNCTION_ARGS) +{ + ListCell *iter; + List *alive_procs = get_query_backend_info(PG_GETARG_ARRAYTYPE_P(0)); + + if (alive_procs == NIL) + PG_RETURN_NULL(); + + /* Set request parameters for signal handler. */ + params->verbose = true; + params->costs = true; + params->timing = true; + params->buffers = true; + params->triggers = false; + params->format = EXPLAIN_FORMAT_JSON; + + foreach(iter, alive_procs) + { + PGPROC *proc = (PGPROC *) lfirst(iter); + int sig_result; + + if (!proc) + continue; + + sig_result = SendProcSignal(proc->pid, QueryStatePollReason, + proc->backendId); + if (sig_result == -1) + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cbdb_mpp_query_state: failed to send signal to pid %d", + proc->pid))); + } + PG_RETURN_VOID(); +} + +/* + * get_query_backend_info -- convert a gp_segment_pid[] SQL array to a list + * of PGPROC pointers for backends running on this segment. + * + * Skips entries for other segments and entries whose backend has exited. + */ +static List * +get_query_backend_info(ArrayType *array) +{ + int16 typlen; + bool typbyval; + char typalign; + Oid element_type = ARR_ELEMTYPE(array); + Datum *data; + bool *nulls; + int nitems; + int len; + List *alive_procs = NIL; + + get_typlenbyvalalign(element_type, &typlen, &typbyval, &typalign); + deconstruct_array(array, element_type, typlen, typbyval, typalign, + &data, &nulls, &nitems); + + len = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); + + for (int i = 0; i < len; i++) + { + HeapTupleHeader td = DatumGetHeapTupleHeader(data[i]); + TupleDesc tupDesc; + HeapTupleData tmptup; + int32 pid; + int32 segid; + bool isnull = false; + PGPROC *proc; + + tupDesc = lookup_rowtype_tupdesc_copy( + HeapTupleHeaderGetTypeId(td), HeapTupleHeaderGetTypMod(td)); + tmptup.t_len = HeapTupleHeaderGetDatumLength(td); + tmptup.t_data = td; + + segid = DatumGetInt32(heap_getattr(&tmptup, 1, tupDesc, &isnull)); + if (isnull || segid != GpIdentity.segindex) + continue; + + pid = DatumGetInt32(heap_getattr(&tmptup, 2, tupDesc, &isnull)); + if (isnull) + continue; + + proc = BackendPidGetProc(pid); + if (proc == NULL) + continue; + + alive_procs = lappend(alive_procs, proc); + } + return alive_procs; +} diff --git a/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.h b/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.h new file mode 100644 index 00000000000..21fce22ea28 --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.h @@ -0,0 +1,231 @@ +/* + * pg_query_state.h + * Public API for the pg_query_state signal-dispatch layer. + * + * This header is included by both the C extension entry point + * (gp_stats_collector.c) and the C++ hook wrappers (hook_wrappers.cpp). + * Keep it C-compatible: no C++ types, wrapped in extern "C". + * + * Copyright (c) 2016-2024, Postgres Professional + * + * IDENTIFICATION + * gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.h + */ +#ifndef __PG_QUERY_STATE_H__ +#define __PG_QUERY_STATE_H__ +#ifdef __cplusplus +extern "C" { +#endif + +#include "postgres.h" + +#include "commands/explain.h" +#include "nodes/pg_list.h" +#include "storage/procarray.h" +#include "storage/shm_mq.h" +#include "cdb/cdbdispatchresult.h" +#include "qs_types.h" + +/* Shared memory queue capacity for passing query-state messages. */ +#define QUEUE_SIZE (64 * 1024) + +/* Maximum single chunk size when splitting a message across shm_mq sends. */ +#define MSG_MAX_SIZE (4 * 1024) + +/* Delay between shm_mq send retries, in microseconds (100 ms). */ +#define WRITING_DELAY (100 * 1000) + +/* Maximum number of send retries before giving up. */ +#define NUM_OF_ATTEMPTS 6 + +/* Bitmask flags for caller-side warnings embedded in shm_mq_msg.warnings. */ +#define TIMINIG_OFF_WARNING 1 +#define BUFFERS_OFF_WARNING 2 + +/* Unique key that identifies our shm_toc segment. */ +#define PG_QS_MODULE_KEY 0xCA94B108 + +/* Table-of-contents slot indices within the shm_toc segment. */ +#define PG_QS_RCV_KEY 0 +#define PG_QS_SND_KEY 1 + +/* + * Timeouts for shm_mq operations. + * The receive timeout must exceed the send timeout so that waiting workers + * always give up before the polling process stops listening. + */ +#define MAX_RCV_TIMEOUT 2000 /* ms */ +#define MAX_SND_TIMEOUT 1000 /* ms */ + +/* + * Sleep between partial-receive retries (SHM_MQ_WOULD_BLOCK case). + * Must be less than MAX_RCV_TIMEOUT. + */ +#define PART_RCV_DELAY 100 /* ms */ + +/* + * Status codes returned by the signal handler to describe the state of the + * queried backend. + */ +typedef enum +{ + QUERY_NOT_RUNNING, /* backend is idle or has no active QueryDesc */ + STAT_DISABLED, /* pg_query_state.enable = false */ + QS_RETURNED /* handler successfully collected and sent stats */ +} PG_QS_RequestResult; + +/* + * Wire format for a query-state reply message transmitted through shm_mq. + * The variable-length `stack` field carries sequentially laid out text frames, + * one per stack depth. + */ +typedef struct +{ + int reqid; + int length; /* total message size including flexible array */ + PGPROC *proc; + PG_QS_RequestResult result_code; + int warnings; /* bitmask of TIMINIG_OFF_WARNING / BUFFERS_OFF_WARNING */ + int stack_depth; + char stack[FLEXIBLE_ARRAY_MEMBER]; +} shm_mq_msg; + +/* + * Wire format for the user-id polling reply. + */ +typedef struct +{ + Oid userid; + uint32 reqid; +} shm_mq_userid_msg; + +#define BASE_SIZEOF_SHM_MQ_MSG (offsetof(shm_mq_msg, stack_depth)) + +/* + * Compact identifier for a backend running on a specific segment. + */ +typedef struct +{ + int32 segid; + int32 pid; +} gp_segment_pid; + +/* + * Wire format for the backend-info (CDB segment PIDs) reply. + */ +typedef struct +{ + int reqid; + int length; + PGPROC *proc; + PG_QS_RequestResult result_code; + int number; + gp_segment_pid pids[FLEXIBLE_ARRAY_MEMBER]; +} backend_info; + +#define BASE_SIZEOF_GP_BACKEND_INFO (offsetof(backend_info, pids)) + +/* + * Parameters passed through shared memory from the requestor to the signal + * handler, controlling what the handler should collect and how. + */ +typedef struct +{ + ProcSignalReason reason; + int reqid; + bool verbose; + bool costs; + bool timing; + bool buffers; + bool triggers; + ExplainFormat format; +} pg_qs_params; + +/* + * Context threaded through the plan-tree walker. + * per_node_stats accumulates one GpscNodeSample per visited node. + */ +typedef struct QsWalkerContext +{ + List *per_node_stats; + int32 parent_plan_node_id; + bool finalize; /* true only in pg_qs_executor end */ +} QsWalkerContext; + +/* + * Result code for the chunked shm_mq send helper. + */ +typedef enum +{ + MSG_BY_PARTS_SUCCEEDED, + MSG_BY_PARTS_FAILED +} msg_by_parts_result; + +extern bool pg_qs_enable; +extern bool pg_qs_timing; +extern bool pg_qs_buffers; +extern List *QueryDescStack; +extern pg_qs_params *params; +extern shm_mq *mq; +extern uint32 *mq_req_id; + +extern ProcSignalReason UserIdPollReason; +extern ProcSignalReason QueryStatePollReason; +extern ProcSignalReason BackendInfoPollReason; + +/* + * pg_qs_init -- register shared memory, custom signals and GUC variables. + * Must be called from _PG_init() during shared_preload_libraries processing. + */ +extern void pg_qs_init(void); + +/* Executor lifecycle hooks -- called from hook_wrappers.cpp. */ +extern void pg_qs_executor_start(QueryDesc *queryDesc, int eflags); +extern void pg_qs_executor_run(QueryDesc *queryDesc); +extern void pg_qs_executor_finish(QueryDesc *queryDesc); +extern void pg_qs_executor_end(QueryDesc *queryDesc); + +/* Shared-memory queue receive helper with millisecond deadline. */ +extern shm_mq_result shm_mq_receive_with_timeout(shm_mq_handle *mqh, + Size *nbytesp, + void **datap, + int64 timeout); + +/* QueryDescStack push/pop helpers. */ +extern void pg_qs_pop_query(void); +extern void pg_qs_push_query(QueryDesc *); + +/* Custom signal handlers registered with RegisterCustomProcSignalHandler. */ +extern void SendQueryState(void); +extern void SendCurrentUserId(void); +extern void SendCdbComponents(void); + +/* Shared-memory lock helpers. */ +extern void UnlockShmem(LOCKTAG *tag); +extern void LockShmem(LOCKTAG *tag, uint32 key); + +/* Chunked shm_mq send. */ +extern msg_by_parts_result send_msg_by_parts(shm_mq_handle *mqh, + Size nbytes, + const void *data); + +/* Plan-tree walker and per-node stat collectors. */ +typedef void (*qs_planstate_walker_callback)(PlanState *, QsWalkerContext *); +extern void qs_planstate_walker(PlanState *, qs_planstate_walker_callback, + QsWalkerContext *, int depth); +extern void qs_get_node_stats(PlanState *, QsWalkerContext *); + +/* Debug logging helpers -- emit collected stats to PostgreSQL LOG. */ +extern void qs_debug_node_stats(List *per_node_stats); +extern void qs_debug_node_sample(GpscNodeSample *sample); + +/* Query filtering and miscellaneous helpers. */ +extern bool filter_query(QueryDesc *queryDesc); +extern bool wait_for_mq_detached(shm_mq_handle *mqh); +extern bool is_querystack_empty(void); +extern QueryDesc *get_toppest_query(void); + +#ifdef __cplusplus +} +#endif +#endif /* __PG_QUERY_STATE_H__ */ diff --git a/gpcontrib/gp_stats_collector/src/pg_query_state/qs_types.h b/gpcontrib/gp_stats_collector/src/pg_query_state/qs_types.h new file mode 100644 index 00000000000..96f28e9602e --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/pg_query_state/qs_types.h @@ -0,0 +1,73 @@ +/* + * qs_types.h + * Per-node sample type collected by the pg_query_state plan-tree walker. + * + * Copyright (c) 2024, Postgres Professional + * + * IDENTIFICATION + * gpcontrib/gp_stats_collector/src/pg_query_state/qs_types.h + */ +#ifndef QS_TYPES_H +#define QS_TYPES_H + +#include + +/* + * Execution phase of a single plan node as observed at signal time. + */ +typedef enum QsNodeStatus +{ + QS_NODE_STATUS_UNSPECIFIED = 0, + QS_NODE_STATUS_INITIALIZED = 1, /* instrumentation allocated but not yet started */ + QS_NODE_STATUS_EXECUTING = 2, /* currently inside a tuple-fetch call */ + QS_NODE_STATUS_FINISHED = 3 /* at least one full loop completed */ +} QsNodeStatus; + +/* + * Per-node snapshot collected by qs_get_node_stats(). + * + * All timing fields mirror the PostgreSQL Instrumentation struct and carry + * the same semantics: startup/total/firsttuple are in seconds, + * ntuples/tuplecount/nloops are raw counters. + * + * relation_oid is populated for scan nodes (SeqScan, IndexScan, + * IndexOnlyScan, BitmapHeapScan, TidScan) by reading the range-table entry + * via EState.es_range_table. It is zero for all other node types. + */ +typedef struct GpscNodeSample +{ + int32_t tmid; /* transaction/time id (gp_gettmid) */ + int32_t ssid; /* gp_session_id */ + int32_t ccnt; /* gp_command_count */ + int32_t plan_node_id; /* Plan.plan_node_id */ + int32_t parent_plan_node_id; /* plan_node_id of logical parent */ + int32_t node_tag; /* nodeTag(plan) */ + int32_t slice_id; /* currentSliceId */ + int32_t segindex; /* GpIdentity.segindex */ + int32_t dbid; /* GpIdentity.dbid */ + int32_t relation_oid; /* OID of scanned relation, or 0 */ + double plan_rows; /* optimizer row estimate */ + double ntuples; /* Instrumentation.ntuples */ + double tuplecount; /* Instrumentation.tuplecount (in-progress loop) */ + double nloops; /* Instrumentation.nloops */ + double startup; /* Instrumentation.startup (seconds) */ + double total; /* Instrumentation.total (seconds) */ + double firsttuple; /* Instrumentation.firsttuple (seconds) */ + uint64_t shared_blks_hit; + uint64_t shared_blks_read; + QsNodeStatus node_status; + bool eof; /* Instrumentation.eof: node exhausted for + * the current cycle (last fetch returned no + * tuple). Lets consumers tell a finished + * node from one still actively producing. */ + /* + * Spill, from the GP-specific Instrumentation fields. Reliable once the node + * is finalized; a mid-run snapshot is a lower bound (Sort/HashJoin populate + * these only at eager-free / explain-end). + */ + bool workfile_created; /* Instrumentation.workfileCreated */ + int64_t workmem_used; /* Instrumentation.workmemused (bytes) */ + int64_t workmem_wanted; /* Instrumentation.workmemwanted (bytes); >0 == spilled */ +} GpscNodeSample; + +#endif /* QS_TYPES_H */ diff --git a/gpcontrib/gp_stats_collector/src/pg_query_state/signal_handler.c b/gpcontrib/gp_stats_collector/src/pg_query_state/signal_handler.c new file mode 100644 index 00000000000..866cca972fc --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/pg_query_state/signal_handler.c @@ -0,0 +1,627 @@ +/* + * signal_handler.c + * Custom signal handlers and plan-tree walker for pg_query_state. + * + * This module implements the three custom ProcSignal handlers registered by + * pg_qs_init(): + * + * SendQueryState() -- fired when QueryStatePollReason is received. + * Walks the active plan tree, collects per-node stats, + * and writes them to the PostgreSQL LOG. + * SendCurrentUserId() -- fired when UserIdPollReason is received. + * Sends the current effective user-id through shm_mq. + * SendCdbComponents() -- fired when BackendInfoPollReason is received (QD only). + * Sends the list of active QE (segid, pid) pairs. + * + * Also contains: + * qs_planstate_walker() -- recursive plan-tree traversal helper. + * qs_get_node_stats() -- per-node stat collection callback. + * qs_debug_node_stats() -- LOG-level dump of a collected stat list. + * qs_debug_node_sample() -- LOG-level dump of a single GpscNodeSample. + * send_msg_by_parts() -- chunked shm_mq send helper. + * + * Copyright (c) 2016-2024, Postgres Professional + * + * IDENTIFICATION + * gpcontrib/gp_stats_collector/src/pg_query_state/signal_handler.c + */ + +#include + +#include "pg_query_state.h" + +#include "cdb/cdbexplain.h" +#include "cdb/cdbutil.h" +#include "cdb/cdbvars.h" +#include "libpq-fe.h" +#include "cdb/cdbconn.h" +#include "commands/explain.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "storage/bufmgr.h" +#include "storage/lock.h" +#include "utils/builtins.h" +#include "utils/memutils.h" +#include "libpq/pqmq.h" + +/* + * shm_mq_send_nonblocking -- attempt to send nbytes through mqh up to + * `attempts` times, sleeping WRITING_DELAY µs between retries. + * + * Returns MSG_BY_PARTS_FAILED immediately on SHM_MQ_DETACHED; retries on + * SHM_MQ_WOULD_BLOCK. + */ +static msg_by_parts_result +shm_mq_send_nonblocking(shm_mq_handle *mqh, Size nbytes, + const void *data, Size attempts) +{ + int i; + shm_mq_result res; + + for (i = 0; i < (int) attempts; i++) + { +#if PG_VERSION_NUM < 150000 + res = shm_mq_send(mqh, nbytes, data, true); +#else + res = shm_mq_send(mqh, nbytes, data, true, true); +#endif + + if (res == SHM_MQ_SUCCESS) + break; + else if (res == SHM_MQ_DETACHED) + return MSG_BY_PARTS_FAILED; + + /* SHM_MQ_WOULD_BLOCK -- back off briefly and retry. */ + pg_usleep(WRITING_DELAY); + } + + if (i == (int) attempts) + return MSG_BY_PARTS_FAILED; + + return MSG_BY_PARTS_SUCCEEDED; +} + +/* + * send_msg_by_parts -- transmit an arbitrarily large buffer through mqh. + * + * The wire protocol is: first send a Size value announcing the total payload + * length, then send the payload itself in chunks of at most MSG_MAX_SIZE + * bytes. The receiver must use receive_msg_by_parts() (in pg_query_state.c) + * to reassemble the chunks. + * + * Parameters: + * mqh -- attached shm_mq handle (sender side) + * nbytes -- total payload size + * data -- pointer to the payload + * + * Returns MSG_BY_PARTS_SUCCEEDED on success, MSG_BY_PARTS_FAILED otherwise. + */ +msg_by_parts_result +send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data) +{ + int offset; + int bytes_left; + int bytes_send; + + /* Announce total length. */ + if (shm_mq_send_nonblocking(mqh, sizeof(Size), &nbytes, + NUM_OF_ATTEMPTS) == MSG_BY_PARTS_FAILED) + return MSG_BY_PARTS_FAILED; + + /* Send payload in chunks. */ + for (offset = 0; offset < (int) nbytes; offset += bytes_send) + { + bytes_left = nbytes - offset; + bytes_send = (bytes_left < MSG_MAX_SIZE) ? bytes_left : MSG_MAX_SIZE; + if (shm_mq_send_nonblocking(mqh, bytes_send, + &(((unsigned char *) data)[offset]), + NUM_OF_ATTEMPTS) == MSG_BY_PARTS_FAILED) + return MSG_BY_PARTS_FAILED; + } + + return MSG_BY_PARTS_SUCCEEDED; +} + +/* + * qs_planstate_walker -- depth-first traversal of a PlanState tree. + * + * Visits every node in the tree rooted at `planstate`, calling `executor` + * on each node before recursing. Handles all node types that have child + * plan states (Append, MergeAppend, BitmapAnd/Or, SubqueryScan, CustomScan, + * init-plans, and sub-plans). + * + * Parameters: + * planstate -- root of the subtree to walk (NULL is a no-op) + * executor -- callback invoked for each node + * qs_walker_ctx -- context threaded through all callbacks + * depth -- current recursion depth (for stack-depth checks) + */ +void +qs_planstate_walker(PlanState *planstate, + qs_planstate_walker_callback executor, + QsWalkerContext *qs_walker_ctx, + int depth) +{ + int32 saved_parent_plan_node_id; + Plan *plan; + ListCell *lc; + + if (planstate == NULL) + return; + + check_stack_depth(); + + plan = planstate->plan; + + executor(planstate, qs_walker_ctx); + saved_parent_plan_node_id = qs_walker_ctx->parent_plan_node_id; + qs_walker_ctx->parent_plan_node_id = plan->plan_node_id; + + /* initPlans */ + foreach(lc, planstate->initPlan) + { + SubPlanState *sps = lfirst_node(SubPlanState, lc); + qs_planstate_walker(sps->planstate, executor, qs_walker_ctx, depth + 1); + } + + /* Left and right children. */ + qs_planstate_walker(outerPlanState(planstate), executor, qs_walker_ctx, + depth + 1); + qs_planstate_walker(innerPlanState(planstate), executor, qs_walker_ctx, + depth + 1); + + /* Type-specific child plans. */ + switch (nodeTag(plan)) + { + case T_Append: + { + AppendState *as = (AppendState *) planstate; + for (int i = 0; i < as->as_nplans; i++) + qs_planstate_walker(as->appendplans[i], executor, + qs_walker_ctx, depth + 1); + break; + } + case T_MergeAppend: + { + MergeAppendState *ms = (MergeAppendState *) planstate; + for (int i = 0; i < ms->ms_nplans; i++) + qs_planstate_walker(ms->mergeplans[i], executor, + qs_walker_ctx, depth + 1); + break; + } + case T_BitmapAnd: + { + BitmapAndState *bas = (BitmapAndState *) planstate; + for (int i = 0; i < bas->nplans; i++) + qs_planstate_walker(bas->bitmapplans[i], executor, + qs_walker_ctx, depth + 1); + break; + } + case T_BitmapOr: + { + BitmapOrState *bos = (BitmapOrState *) planstate; + for (int i = 0; i < bos->nplans; i++) + qs_planstate_walker(bos->bitmapplans[i], executor, + qs_walker_ctx, depth + 1); + break; + } + case T_SubqueryScan: + qs_planstate_walker(((SubqueryScanState *) planstate)->subplan, + executor, qs_walker_ctx, depth + 1); + break; + case T_CustomScan: + foreach(lc, ((CustomScanState *) planstate)->custom_ps) + qs_planstate_walker((PlanState *) lfirst(lc), executor, + qs_walker_ctx, depth + 1); + break; + default: + break; + } + + /* subPlans */ + foreach(lc, planstate->subPlan) + { + SubPlanState *sps = lfirst_node(SubPlanState, lc); + qs_planstate_walker(sps->planstate, executor, qs_walker_ctx, depth + 1); + } + + qs_walker_ctx->parent_plan_node_id = saved_parent_plan_node_id; +} + +/* + * qs_get_node_stats -- walker callback that snapshots one plan node. + * + * Allocates a GpscNodeSample in the current memory context, fills it from + * planstate->instrument (if available), and appends it to + * qs_walker_ctx->per_node_stats. + * + * Parameters: + * planstate -- the plan node being sampled + * qs_walker_ctx -- walker context; per_node_stats is extended in-place + */ +void +qs_get_node_stats(PlanState *planstate, QsWalkerContext *qs_walker_ctx) +{ + GpscNodeSample *nodestat = + (GpscNodeSample *) palloc0(sizeof(GpscNodeSample)); + + /* Identity fields. */ + gp_gettmid(&nodestat->tmid); + nodestat->ssid = gp_session_id; + nodestat->ccnt = gp_command_count; + + /* Plan-tree position. */ + nodestat->plan_node_id = planstate->plan->plan_node_id; + nodestat->parent_plan_node_id = qs_walker_ctx->parent_plan_node_id; + nodestat->node_tag = nodeTag(planstate->plan); + nodestat->slice_id = currentSliceId; + nodestat->segindex = GpIdentity.segindex; + nodestat->dbid = GpIdentity.dbid; + + /* Planner estimate. */ + nodestat->plan_rows = planstate->plan->plan_rows; + + /* Runtime instrumentation (may be NULL for non-instrumented nodes). */ + if (planstate->instrument) + { + Instrumentation *instr = planstate->instrument; + + if (qs_walker_ctx->finalize) + { + InstrEndLoop(instr); + } + + nodestat->ntuples = instr->ntuples + instr->tuplecount; /* include in-progress loop */ + nodestat->tuplecount = instr->tuplecount; + nodestat->nloops = instr->nloops; + nodestat->startup = instr->startup; + nodestat->total = instr->total; + nodestat->firsttuple = instr->firsttuple; + + nodestat->shared_blks_hit = instr->bufusage.shared_blks_hit; + nodestat->shared_blks_read = instr->bufusage.shared_blks_read; + + /* + * eof lets a consumer tell a node that has finished producing (running + * but exhausted for this cycle) from one still actively pulling. Only + * meaningful while running; nloops>0 / not-running already imply done. + */ + nodestat->eof = instr->eof; + + if (instr->running && !instr->eof) + nodestat->node_status = QS_NODE_STATUS_EXECUTING; + else if (instr->nloops > 0) + nodestat->node_status = QS_NODE_STATUS_FINISHED; + else + nodestat->node_status = QS_NODE_STATUS_INITIALIZED; + + /* + * Per-node spill, from the GP-specific Instrumentation fields. These are + * populated by the node executors: workfileCreated live at spill for + * Agg/HashJoin/Hash, at eager-free for Sort; workmemused/workmemwanted at + * batch boundaries (Agg) or explain-end. A running snapshot is therefore a + * lower bound — consumers treat it as such. + */ + nodestat->workfile_created = instr->workfileCreated; + nodestat->workmem_used = (int64_t) instr->workmemused; + nodestat->workmem_wanted = (int64_t) instr->workmemwanted; + } + else + { + nodestat->node_status = QS_NODE_STATUS_INITIALIZED; + } + + /* + * Populate relation_oid for scan nodes by looking up the range-table + * entry using the node's scanrelid. EState.es_range_table is a flat + * List indexed 1-based by scanrelid. + */ + switch (nodeTag(planstate->plan)) + { + case T_SeqScan: + case T_IndexScan: + case T_IndexOnlyScan: + case T_BitmapHeapScan: + case T_TidScan: + { + Index scanrelid = ((Scan *) planstate->plan)->scanrelid; + if (scanrelid > 0 && planstate->state != NULL) + { + List *rtable = planstate->state->es_range_table; + if (scanrelid <= (Index) list_length(rtable)) + { + RangeTblEntry *rte = (RangeTblEntry *) + list_nth(rtable, (int) scanrelid - 1); + if (rte->rtekind == RTE_RELATION) + nodestat->relation_oid = (int32_t) rte->relid; + } + } + break; + } + default: + break; + } + + qs_walker_ctx->per_node_stats = + lappend(qs_walker_ctx->per_node_stats, nodestat); +} + +/* + * qs_debug_node_sample -- emit a single GpscNodeSample to the PostgreSQL LOG. + * + * Intended for development and integration testing. In production deployments + * this will produce a large number of log lines; suppress with log_min_messages. + */ +void +qs_debug_node_sample(GpscNodeSample *s) +{ + elog(DEBUG1, + "GpscNodeSample: " + "plan_node_id=%d parent=%d node_tag=%d " + "slice_id=%d segindex=%d " + "tmid=%d ssid=%d ccnt=%d " + "plan_rows=%.0f " + "ntuples=%.0f tuplecount=%.0f nloops=%.0f " + "startup=%f total=%f firsttuple=%f " + "shared_blks_hit=%lu shared_blks_read=%lu " + "workfile_created=%d workmem_used=%ld workmem_wanted=%ld " + "node_status=%d", + s->plan_node_id, s->parent_plan_node_id, s->node_tag, + s->slice_id, s->segindex, + s->tmid, s->ssid, s->ccnt, + s->plan_rows, + s->ntuples, s->tuplecount, s->nloops, + s->startup, s->total, s->firsttuple, + s->shared_blks_hit, s->shared_blks_read, + (int) s->workfile_created, (long) s->workmem_used, (long) s->workmem_wanted, + (int) s->node_status); +} + +/* + * qs_debug_node_stats -- emit all nodes in per_node_stats to the PostgreSQL LOG. + * + * Logs a summary line followed by one line per node via qs_debug_node_sample(). + */ +void +qs_debug_node_stats(List *per_node_stats) +{ + ListCell *lc; + int i = 0; + + elog(DEBUG1, "GpscNodeSample list: %d nodes", list_length(per_node_stats)); + foreach(lc, per_node_stats) + { + GpscNodeSample *s = (GpscNodeSample *) lfirst(lc); + elog(DEBUG1, "--- node[%d] ---", i++); + qs_debug_node_sample(s); + } +} + +/* + * runtime_explain -- snapshot the active query's plan tree. + * + * Retrieves the top-most QueryDesc from QueryDescStack, walks its planstate + * tree with qs_get_node_stats(), and returns the resulting List of + * GpscNodeSample pointers. + * + * Callers must ensure QueryDescStack is non-empty before calling this. + */ +static List * +runtime_explain(void) +{ + QsWalkerContext *qs_walker_ctx = + (QsWalkerContext *) palloc0(sizeof(QsWalkerContext)); + QueryDesc *queryDesc; + + Assert(list_length(QueryDescStack) > 0); + queryDesc = get_toppest_query(); + qs_planstate_walker(queryDesc->planstate, qs_get_node_stats, + qs_walker_ctx, 0); + return qs_walker_ctx->per_node_stats; +} + +/* + * SendQueryState -- handler for QueryStatePollReason. + * + * Fired asynchronously when another backend (or the monitoring function) + * sends QueryStatePollReason to this process. + * + * Collects a plan-tree snapshot via runtime_explain() and emits it to the + * PostgreSQL LOG via qs_debug_node_stats(). This branch does NOT push data + * to any external sink. + * + * The entire body runs inside a dedicated MemoryContext that is deleted on + * exit, preventing any leaks into the backend's long-lived contexts. Any + * errors are swallowed with FlushErrorState() to avoid crashing the backend. + */ +void +SendQueryState(void) +{ + MemoryContext oldcontext; + MemoryContext qs_context; + List *qs_result = NIL; + + if (!pg_qs_enable) + return; /* STAT_DISABLED */ + + if (!list_length(QueryDescStack)) + return; /* QUERY_NOT_RUNNING */ + + if (stack_is_too_deep()) + { + elog(DEBUG1, "pg_query_state: skipping poll, call stack too deep"); + return; + } + + qs_context = AllocSetContextCreate(TopMemoryContext, + "pg_query_state signal context", + ALLOCSET_DEFAULT_SIZES); + oldcontext = MemoryContextSwitchTo(qs_context); + + PG_TRY(); + { + qs_result = runtime_explain(); + qs_debug_node_stats(qs_result); + } + PG_CATCH(); + { + FlushErrorState(); + } + PG_END_TRY(); + + MemoryContextSwitchTo(oldcontext); + MemoryContextDelete(qs_context); +} + +/* + * SendCurrentUserId -- handler for UserIdPollReason. + * + * Sends a shm_mq_userid_msg containing the current effective user-id through + * the shared mq so the requestor can verify the target backend's identity. + */ +void +SendCurrentUserId(void) +{ + shm_mq_handle *mqh; + shm_mq_userid_msg msg; + LOCKTAG tag; + + msg.userid = GetUserId(); + + LockShmem(&tag, PG_QS_SND_KEY); + mqh = shm_mq_attach(mq, NULL, NULL); + msg.reqid = *mq_req_id; + + if (shm_mq_get_sender(mq) != MyProc || + params->reason != UserIdPollReason) + { + elog(WARNING, "pg_query_state: SendCurrentUserId: stale or mismatched request"); + } + else if (send_msg_by_parts(mqh, sizeof(msg), &msg) != MSG_BY_PARTS_SUCCEEDED) + { + elog(WARNING, "pg_query_state: SendCurrentUserId: failed to send reply"); + } + +#if PG_VERSION_NUM < 100000 + shm_mq_detach(mq); +#else + shm_mq_detach(mqh); +#endif + UnlockShmem(&tag); +} + +/* + * fill_segpid -- populate consecutive gp_segment_pid slots from one CDB segment. + * + * Iterates the activelist of segInfo and fills msg->pids starting at *index, + * incrementing *index for each entry. + */ +static void +fill_segpid(CdbComponentDatabaseInfo *segInfo, backend_info *msg, int *index) +{ + ListCell *lc; + + foreach(lc, segInfo->activelist) + { + SegmentDatabaseDescriptor *dbdesc = + (SegmentDatabaseDescriptor *) lfirst(lc); + gp_segment_pid *segpid = &msg->pids[(*index)++]; + segpid->pid = dbdesc->backendPid; + segpid->segid = dbdesc->segindex; + } +} + +/* + * SendCdbComponents -- handler for BackendInfoPollReason (QD only). + * + * Collects the list of active QE (segid, pid) pairs from the CDB component + * database and sends them back to the requestor through shm_mq as a + * backend_info message. + * + * Side effects: + * - Calls cdbcomponent_getCdbComponents(), which may allocate memory. + * - All allocations are in a short-lived MemoryContext deleted on exit. + */ +void +SendCdbComponents(void) +{ + shm_mq_handle *mqh = NULL; + CdbComponentDatabases *cdbs; + msg_by_parts_result send_result; + MemoryContext oldctx; + int index = 0; + volatile int32 savedInterruptHoldoffCount; + MemoryContext query_state_ctx = + AllocSetContextCreate(TopMemoryContext, + "pg_query_state SendCdbComponents", + ALLOCSET_DEFAULT_SIZES); + + oldctx = MemoryContextSwitchTo(query_state_ctx); + savedInterruptHoldoffCount = InterruptHoldoffCount; + + PG_TRY(); + { + mqh = shm_mq_attach(mq, NULL, NULL); + + if (shm_mq_get_sender(mq) != MyProc || + params->reason != BackendInfoPollReason) + { + elog(DEBUG1, "pg_query_state: SendCdbComponents: stale request, discarding"); + shm_mq_detach(mqh); + } + else if (!pg_qs_enable) + { + elog(DEBUG1, "pg_query_state: SendCdbComponents: module disabled"); + shm_mq_msg disabled_msg = {*mq_req_id, BASE_SIZEOF_SHM_MQ_MSG, + MyProc, STAT_DISABLED}; + if (send_msg_by_parts(mqh, disabled_msg.length, + &disabled_msg) != MSG_BY_PARTS_SUCCEEDED) + shm_mq_detach(mqh); + } + else if (list_length(QueryDescStack) == 0) + { + elog(DEBUG1, "pg_query_state: SendCdbComponents: no active query"); + shm_mq_msg not_running_msg = {*mq_req_id, BASE_SIZEOF_SHM_MQ_MSG, + MyProc, QUERY_NOT_RUNNING}; + if (send_msg_by_parts(mqh, not_running_msg.length, + ¬_running_msg) != MSG_BY_PARTS_SUCCEEDED) + shm_mq_detach(mqh); + } + else + { + cdbs = cdbcomponent_getCdbComponents(); + int msglen = BASE_SIZEOF_GP_BACKEND_INFO + + sizeof(gp_segment_pid) * cdbs->numActiveQEs; + backend_info *msg = (backend_info *) palloc0(msglen); + + msg->reqid = *mq_req_id; + msg->length = msglen; + msg->result_code = QS_RETURNED; + + for (int i = 0; i < cdbs->total_segment_dbs; i++) + { + CdbComponentDatabaseInfo *segInfo = + &cdbs->segment_db_info[i]; + fill_segpid(segInfo, msg, &index); + } + Assert(index == cdbs->numActiveQEs); + msg->number = index; + + send_result = send_msg_by_parts(mqh, msglen, msg); + if (send_result != MSG_BY_PARTS_SUCCEEDED) + shm_mq_detach(mqh); + } + } + PG_CATCH(); + { + elog(WARNING, "pg_query_state: SendCdbComponents: error during send"); + elog_dismiss(WARNING); + if (mqh) + shm_mq_detach(mqh); + InterruptHoldoffCount = savedInterruptHoldoffCount; + } + PG_END_TRY(); + + MemoryContextSwitchTo(oldctx); + MemoryContextDelete(query_state_ctx); +} From 27d11b56860b483d2ef5fc9702015a8d7b47d37c Mon Sep 17 00:00:00 2001 From: roaldm153 Date: Mon, 3 Aug 2026 18:09:27 +0500 Subject: [PATCH 03/17] ci(gpsc): add gp_stats_collector build-and-test workflow Builds Cloudberry with the extension across ubuntu22.04/rocky8/rocky9, stands up a demo cluster with gp_stats_collector preloaded, and runs the pg_regress and multi-session isolation2 suites. --- .github/workflows/gpsc-ci.yaml | 316 +++++++++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 .github/workflows/gpsc-ci.yaml diff --git a/.github/workflows/gpsc-ci.yaml b/.github/workflows/gpsc-ci.yaml new file mode 100644 index 00000000000..87e2df5081b --- /dev/null +++ b/.github/workflows/gpsc-ci.yaml @@ -0,0 +1,316 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# gp_stats_collector CI Workflow +# +# Builds Cloudberry with --with-gp-stats-collector (the default in +# configure-cloudberry.sh), stands up a demo cluster with the extension +# preloaded, and runs the gp_stats_collector regression suites. +# +# Scoped to changes that can affect the extension or the core patches it +# depends on, so it does not run on every unrelated push. +# -------------------------------------------------------------------- +name: GPSC CI Pipeline + +on: + push: + branches: + - 'pgqs-**' + pull_request: + paths: + - 'gpcontrib/gp_stats_collector/**' + - '.github/workflows/gpsc-ci.yaml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + test-gpsc: + name: Build and Test gp_stats_collector (${{ matrix.os }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu22.04 + image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + - os: rocky8 + image: apache/incubator-cloudberry:cbdb-build-rocky8-latest + - os: rocky9 + image: apache/incubator-cloudberry:cbdb-build-rocky9-latest + container: + image: ${{ matrix.image }} + options: >- + --user root + -h cdw + + steps: + - name: Checkout Cloudberry source + uses: actions/checkout@v4 + with: + # Init/build scripts hardcode a "cloudberry" source dir name + # (e.g. create-cloudberry-demo-cluster.sh uses ${SRC_DIR}/../cloudberry), + # so the checkout path must be "cloudberry", not the repo name. + path: cloudberry + submodules: recursive + + - name: Cloudberry Environment Initialization + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + mkdir -p "${SRC_DIR}/build-logs" + chown -R gpadmin:gpadmin . + chmod -R 755 . + + - name: Configure + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure failed" + exit 1 + fi + + - name: Build + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build failed" + exit 1 + fi + + - name: Create demo cluster (gp_stats_collector preloaded) + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then + echo "::error::Demo cluster creation failed" + exit 1 + fi + # pg_query_state requires the module in shared_preload_libraries. + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ra" + + - name: Run gp_stats_collector regression suite + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/test-cloudberry.sh + # Capture make output to an artifact file: the raw job log gets + # truncated by the huge Build step, so tail it here on failure. + if ! time su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + SRC_DIR=${SRC_DIR} \ + PGOPTIONS='' \ + MAKE_NAME='GPSC Regress' \ + MAKE_TARGET=installcheck \ + MAKE_DIRECTORY=--directory=${SRC_DIR}/gpcontrib/gp_stats_collector \ + ${SRC_DIR}/devops/build/automation/cloudberry/scripts/test-cloudberry.sh \ + > ${SRC_DIR}/build-logs/gpsc-regress-make.log 2>&1"; then + echo "::error::gp_stats_collector installcheck failed" + echo "===== gpsc-regress-make.log (tail) =====" + tail -120 ${SRC_DIR}/build-logs/gpsc-regress-make.log 2>/dev/null || true + echo "===== regression.diffs =====" + cat ${SRC_DIR}/gpcontrib/gp_stats_collector/regression.diffs 2>/dev/null || true + exit 1 + fi + + - name: Run pg_query_state suite + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! time su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + SRC_DIR=${SRC_DIR} \ + PGOPTIONS='' \ + MAKE_NAME='GPSC pg_query_state' \ + MAKE_TARGET=installcheck \ + MAKE_DIRECTORY=--directory=${SRC_DIR}/gpcontrib/gp_stats_collector/test \ + ${SRC_DIR}/devops/build/automation/cloudberry/scripts/test-cloudberry.sh \ + > ${SRC_DIR}/build-logs/gpsc-pqs-make.log 2>&1"; then + echo "::error::pg_query_state suite failed" + echo "===== gpsc-pqs-make.log (tail) =====" + tail -120 ${SRC_DIR}/build-logs/gpsc-pqs-make.log 2>/dev/null || true + echo "===== test/regression.diffs =====" + cat ${SRC_DIR}/gpcontrib/gp_stats_collector/test/regression.diffs 2>/dev/null || true + exit 1 + fi + + - name: Upload regression artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: gpsc-results-${{ matrix.os }} + path: | + cloudberry/gpcontrib/gp_stats_collector/regression.out + cloudberry/gpcontrib/gp_stats_collector/regression.diffs + cloudberry/gpcontrib/gp_stats_collector/results/ + cloudberry/gpcontrib/gp_stats_collector/test/results/ + cloudberry/gpcontrib/gp_stats_collector/test/regression.diffs + cloudberry/build-logs/ + retention-days: 7 + + # Runs the pg_query_state isolation2 suite (multi-session / happy-path checks + # that plain pg_regress cannot express). The fault injector (gp_inject_fault) + # is enabled by default (--enable-faultinjector=yes), so no debug build is + # needed. + test-gpsc-isolation2: + name: pg_query_state multi-session (gp_stats_collector) + runs-on: ubuntu-latest + container: + image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + options: >- + --user root + -h cdw + + steps: + - name: Checkout Cloudberry source + uses: actions/checkout@v4 + with: + path: cloudberry + submodules: recursive + + - name: Cloudberry Environment Initialization + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + mkdir -p "${SRC_DIR}/build-logs" + chown -R gpadmin:gpadmin . + chmod -R 755 . + + - name: Configure + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure failed" + exit 1 + fi + + - name: Build + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build failed" + exit 1 + fi + + - name: Build isolation2 harness + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! time su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + make -C src/test/isolation2 install"; then + echo "::error::isolation2 harness build failed" + exit 1 + fi + + - name: Create demo cluster (gp_stats_collector preloaded) + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then + echo "::error::Demo cluster creation failed" + exit 1 + fi + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ra" + + - name: Run pg_query_state isolation2 suite + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + make -C gpcontrib/gp_stats_collector/test/isolation2 installcheck \ + > ${SRC_DIR}/build-logs/gpsc-iso2.log 2>&1"; then + echo "::error::pg_query_state isolation2 suite failed" + echo "===== gpsc-iso2.log (tail) =====" + tail -100 ${SRC_DIR}/build-logs/gpsc-iso2.log 2>/dev/null || true + echo "===== isolation2 regression.diffs =====" + cat ${SRC_DIR}/gpcontrib/gp_stats_collector/test/isolation2/regression.diffs 2>/dev/null || true + exit 1 + fi + + - name: Upload isolation2 artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: gpsc-iso2-results + path: | + cloudberry/gpcontrib/gp_stats_collector/test/isolation2/results/ + cloudberry/gpcontrib/gp_stats_collector/test/isolation2/regression.diffs + cloudberry/build-logs/ + retention-days: 7 From 2700d20f9d0eb8c2eed7ba8bb1b2a446ab15fd8b Mon Sep 17 00:00:00 2001 From: roaldm153 Date: Mon, 3 Aug 2026 18:09:27 +0500 Subject: [PATCH 04/17] test(gpsc): add regression and multi-session isolation2 suites pg_regress contract: catalog registration and input-validation errors. isolation2 (pg_query_state multi-session): idle backend, happy-path poll of a suspended query, the permission gate, STAT_DISABLED, and a strict one-backend-per-primary-segment count. --- gpcontrib/gp_stats_collector/test/Makefile | 22 +++++ .../test/expected/gpsc_pg_query_state.out | 57 ++++++++++++ .../test/isolation2/.gitignore | 4 + .../test/isolation2/Makefile | 33 +++++++ .../isolation2/expected/gpsc_pqs_backends.out | 27 ++++++ .../isolation2/expected/gpsc_pqs_disabled.out | 61 ++++++++++++ .../isolation2/expected/gpsc_pqs_perms.out | 92 +++++++++++++++++++ .../isolation2/expected/gpsc_pqs_running.out | 69 ++++++++++++++ .../expected/gpsc_pqs_seg_count.out | 58 ++++++++++++ .../test/isolation2/expected/setup.out | 6 ++ .../test/isolation2/isolation2_schedule | 20 ++++ .../test/isolation2/sql/gpsc_pqs_backends.sql | 21 +++++ .../test/isolation2/sql/gpsc_pqs_disabled.sql | 36 ++++++++ .../test/isolation2/sql/gpsc_pqs_perms.sql | 57 ++++++++++++ .../test/isolation2/sql/gpsc_pqs_running.sql | 44 +++++++++ .../isolation2/sql/gpsc_pqs_seg_count.sql | 36 ++++++++ .../test/isolation2/sql/setup.sql | 4 + .../test/sql/gpsc_pg_query_state.sql | 46 ++++++++++ pom.xml | 3 + 19 files changed, 696 insertions(+) create mode 100644 gpcontrib/gp_stats_collector/test/Makefile create mode 100644 gpcontrib/gp_stats_collector/test/expected/gpsc_pg_query_state.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/.gitignore create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/Makefile create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_backends.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_disabled.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_perms.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_running.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_seg_count.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/setup.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/isolation2_schedule create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_backends.sql create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_disabled.sql create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_perms.sql create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_running.sql create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_seg_count.sql create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/setup.sql create mode 100644 gpcontrib/gp_stats_collector/test/sql/gpsc_pg_query_state.sql diff --git a/gpcontrib/gp_stats_collector/test/Makefile b/gpcontrib/gp_stats_collector/test/Makefile new file mode 100644 index 00000000000..3931f4d9592 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/Makefile @@ -0,0 +1,22 @@ +# Regression tests for the gp_stats_collector pg_query_state signal API. +# +# Self-contained installcheck suite: the extension must already be built and +# installed (make -C .. install) and loaded via shared_preload_libraries in the +# target cluster. Run with: +# +# make -C gpcontrib/gp_stats_collector/test installcheck +# +# pg_regress defaults to ./sql/.sql and ./expected/.out. + +REGRESS = gpsc_pg_query_state + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = gpcontrib/gp_stats_collector/test +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/gpcontrib/gp_stats_collector/test/expected/gpsc_pg_query_state.out b/gpcontrib/gp_stats_collector/test/expected/gpsc_pg_query_state.out new file mode 100644 index 00000000000..fffe14158c0 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/expected/gpsc_pg_query_state.out @@ -0,0 +1,57 @@ +-- pg_query_state signal API (extension v1.2): catalog contract + negative paths. +-- +-- Deterministic coverage only: SQL-visible function/type registration and the +-- input-validation error branches. The asynchronous happy path (poll a live +-- query and observe per-node stats) is exercised separately under isolation2, +-- since it depends on a second running backend and timing. +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +-- end_ignore +-- +-- Catalog contract: the three SQL-visible functions are registered in the gpsc +-- schema with the expected return type and dispatch (exec) location. +-- proexeclocation: c = coordinator, a = any (QE-local), s = all segments. +-- +SELECT proname, + pronargs, + prorettype::regtype AS returns, + proexeclocation +FROM pg_proc +WHERE pronamespace = 'gpsc'::regnamespace + AND proname IN ('pg_query_state', 'pg_query_state_backends', 'cbdb_mpp_query_state') +ORDER BY proname; + proname | pronargs | returns | proexeclocation +-------------------------+----------+---------+----------------- + cbdb_mpp_query_state | 1 | void | a + pg_query_state | 1 | void | c + pg_query_state_backends | 1 | record | c +(3 rows) + +-- Composite identifier type used by the signal layer is present. +SELECT typname +FROM pg_type +WHERE typnamespace = 'gpsc'::regnamespace + AND typname = 'gp_segment_pid'; + typname +---------------- + gp_segment_pid +(1 row) + +-- +-- Negative: a backend cannot poll its own state. +-- +SELECT gpsc.pg_query_state(pg_backend_pid()); +ERROR: cannot extract state of current process +SELECT * FROM gpsc.pg_query_state_backends(pg_backend_pid()); +ERROR: cannot extract state of current process +-- +-- Negative: a pid that maps to no live backend is rejected. +-- +SELECT gpsc.pg_query_state(-1); +ERROR: backend with pid=-1 not found +SELECT * FROM gpsc.pg_query_state_backends(-1); +ERROR: backend with pid=-1 not found +-- Cleanup +-- start_ignore +DROP EXTENSION gp_stats_collector; +-- end_ignore diff --git a/gpcontrib/gp_stats_collector/test/isolation2/.gitignore b/gpcontrib/gp_stats_collector/test/isolation2/.gitignore new file mode 100644 index 00000000000..0d2848e26fb --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/.gitignore @@ -0,0 +1,4 @@ +/sql_isolation_testcase.py +/results/ +/regression.diffs +/regression.out diff --git a/gpcontrib/gp_stats_collector/test/isolation2/Makefile b/gpcontrib/gp_stats_collector/test/isolation2/Makefile new file mode 100644 index 00000000000..2a835922c65 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/Makefile @@ -0,0 +1,33 @@ +# isolation2 suite for the gp_stats_collector pg_query_state signal API. +# +# Multi-session tests that a plain pg_regress run cannot express (one backend +# polling another). Reuses the core pg_isolation2_regress harness rather than +# rebuilding it. +# +# Prerequisites (handled by the CI isolation2 job): +# - gp_inject_fault available (--enable-faultinjector, on by default) for the +# happy-path spec. +# - Extension installed and gp_stats_collector in shared_preload_libraries. +# - Harness built: make -C $(top_builddir)/src/test/isolation2 install +# +# Run: +# make -C gpcontrib/gp_stats_collector/test/isolation2 installcheck + +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +ISO2 = $(top_builddir)/src/test/isolation2 + +# isolation2_main.c hardcodes "python3 ./sql_isolation_testcase.py", resolved +# from the current directory, so symlink the core driver here before running. +installcheck: + @ln -sf $(ISO2)/sql_isolation_testcase.py ./sql_isolation_testcase.py + $(ISO2)/pg_isolation2_regress \ + --init-file=$(top_builddir)/src/test/regress/init_file \ + --init-file=$(ISO2)/init_file_isolation2 \ + --inputdir=. --outputdir=. \ + --bindir='$(bindir)' \ + --schedule=./isolation2_schedule + +clean: + rm -rf results/ regression.diffs regression.out sql_isolation_testcase.py diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_backends.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_backends.out new file mode 100644 index 00000000000..a19477a411b --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_backends.out @@ -0,0 +1,27 @@ +-- pg_query_state_backends against an *idle* backend returns an empty set. +-- +-- Deterministic multi-session check with no async race: session 1 tags itself +-- and sits idle; session 2 looks up its pid and polls it. An idle backend is +-- "not running a query", so GetRemoteBackendInfo returns QUERY_NOT_RUNNING and +-- the function yields an empty set (not an error). +-- +-- Extensions are created by setup.sql. + +-- Session 1: tag connection so session 2 can find its pid, then go idle. +1: SET application_name TO 'qs_idle_target'; +SET +1: SELECT 1; + ?column? +---------- + 1 +(1 row) + +-- Session 2: idle target -> zero participating backends. +2: SELECT count(*) AS n_backends FROM gpsc.pg_query_state_backends( (SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_idle_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1)); + n_backends +------------ + 0 +(1 row) + +1q: ... +2q: ... diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_disabled.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_disabled.out new file mode 100644 index 00000000000..792610bb011 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_disabled.out @@ -0,0 +1,61 @@ +-- STAT_DISABLED: when the target backend has pg_query_state.enable = off, its +-- SendCdbComponents reply is STAT_DISABLED, so polling reports an empty backend +-- list even though a query is actively running on the segments. +-- +-- Distinguishes "disabled" from "idle": here the query really is executing +-- (suspended on a fault), yet the disabled module yields nothing. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_disabled_t (id int) DISTRIBUTED BY (id); +CREATE +INSERT INTO qs_disabled_t SELECT generate_series(1, 100); +INSERT 100 + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- Target disables the module for its own session, then runs a query that hangs. +1: SET application_name TO 'qs_disabled_target'; +SET +1: SET pg_query_state.enable TO off; +SET +1&: SELECT count(*) FROM qs_disabled_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- Running, but module disabled on the target -> empty backend list. +2: SELECT count(*) AS n_backends FROM gpsc.pg_query_state_backends( (SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_disabled_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1)); + n_backends +------------ + 0 +(1 row) + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> + count +------- + 100 +(1 row) +1q: ... +2q: ... + +DROP TABLE qs_disabled_t; +DROP diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_perms.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_perms.out new file mode 100644 index 00000000000..f6bc6a8ca7e --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_perms.out @@ -0,0 +1,92 @@ +-- Permission gate: a non-superuser that does not own the target query is +-- denied; a superuser is allowed. +-- +-- The gate is superuser() || GetUserId() == proc->roleId. isolation2 runs all +-- sessions under the same session role (the one that launched the harness), so +-- the "non-super owner is allowed" branch cannot be expressed here and is not +-- covered; the deny and superuser-allow branches are. +-- +-- A non-superuser cannot see another backend's application_name in +-- pg_stat_activity, so the target pid is captured (as superuser) into a table +-- before SET ROLE. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_perm_t (id int) DISTRIBUTED BY (id); +CREATE +INSERT INTO qs_perm_t SELECT generate_series(1, 100); +INSERT 100 +CREATE TABLE qs_perm_pid (pid int); +CREATE +CREATE ROLE qs_unpriv; +CREATE +GRANT SELECT ON qs_perm_pid TO qs_unpriv; +GRANT +-- No gpsc grants here on purpose: the extension grants USAGE/EXECUTE to PUBLIC +-- in its migration, so an ordinary role reaches the roleId gate exactly as it +-- would in production. This test verifies the gate, not the schema grants. + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +1: SET application_name TO 'qs_perm_target'; +SET +1&: SELECT count(*) FROM qs_perm_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- Capture the target pid as superuser (sees application_name). +2: INSERT INTO qs_perm_pid SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_perm_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1; +INSERT 1 + +-- Non-superuser, non-owner: both entry points are denied. +2: SET ROLE qs_unpriv; +SET +2: SELECT gpsc.pg_query_state((SELECT pid FROM qs_perm_pid)); +ERROR: permission denied +2: SELECT * FROM gpsc.pg_query_state_backends((SELECT pid FROM qs_perm_pid)); +ERROR: permission denied +2: RESET ROLE; +RESET + +-- Superuser: allowed (non-empty backend list). +2: SELECT count(*) > 0 AS has_backends FROM gpsc.pg_query_state_backends((SELECT pid FROM qs_perm_pid)); + has_backends +-------------- + t +(1 row) + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> + count +------- + 100 +(1 row) +1q: ... +2q: ... + +DROP OWNED BY qs_unpriv; +DROP +DROP ROLE qs_unpriv; +DROP +DROP TABLE qs_perm_pid; +DROP +DROP TABLE qs_perm_t; +DROP diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_running.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_running.out new file mode 100644 index 00000000000..de10078ca8d --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_running.out @@ -0,0 +1,69 @@ +-- Happy path: a query suspended mid-execution on the QEs is observed live. +-- +-- Session 1 launches a query that hits an 'executor_pre_tuple_processed' +-- suspend fault on every primary segment, so its QE backends sit inside the +-- executor with a live plan tree. Session 2 then: +-- * pg_query_state_backends(pid) -> at least one participating backend, +-- * pg_query_state(pid) -> succeeds (fire-and-forget, returns void). +-- The fault is reset and the suspended query is reaped. +-- +-- Extensions (gp_stats_collector, gp_inject_fault) come from setup.sql. + +CREATE TABLE qs_running_t (id int) DISTRIBUTED BY (id); +CREATE +INSERT INTO qs_running_t SELECT generate_series(1, 100); +INSERT 100 + +-- Suspend execution on all primary segments. +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- Session 1: tag the connection, then launch a query that hangs on the QEs. +1: SET application_name TO 'qs_running_target'; +SET +1&: SELECT count(*) FROM qs_running_t; + +-- Wait until the fault has been hit on the segments. +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- Session 2: the running query has live QE backends, and polling succeeds. +2: SELECT count(*) > 0 AS has_backends FROM gpsc.pg_query_state_backends( (SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_running_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1)); + has_backends +-------------- + t +(1 row) +2: SELECT gpsc.pg_query_state( (SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_running_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1)); + pg_query_state +---------------- + +(1 row) + +-- Release the fault and reap the suspended query. +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> + count +------- + 100 +(1 row) +1q: ... +2q: ... + +DROP TABLE qs_running_t; +DROP diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_seg_count.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_seg_count.out new file mode 100644 index 00000000000..b4ad8a9a84d --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_seg_count.out @@ -0,0 +1,58 @@ +-- backends reports exactly one participating backend per primary segment for a +-- single-gang query -- a strict count rather than the has_backends>0 smoke +-- check in gpsc_pqs_running. +-- +-- A plain scan+count is one gang, so the QE list must match the number of +-- primary segments. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_segcount_t (id int) DISTRIBUTED BY (id); +CREATE +INSERT INTO qs_segcount_t SELECT generate_series(1, 100); +INSERT 100 + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +1: SET application_name TO 'qs_segcount_target'; +SET +1&: SELECT count(*) FROM qs_segcount_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- One backend per primary segment. +2: SELECT count(*) = (SELECT count(*) FROM gp_segment_configuration WHERE role = 'p' AND content > -1) AS matches_primaries FROM gpsc.pg_query_state_backends( (SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_segcount_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1)); + matches_primaries +------------------- + t +(1 row) + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> + count +------- + 100 +(1 row) +1q: ... +2q: ... + +DROP TABLE qs_segcount_t; +DROP diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/setup.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/setup.out new file mode 100644 index 00000000000..f7ab4e44725 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/setup.out @@ -0,0 +1,6 @@ +-- Shared setup for the pg_query_state isolation2 suite. +-- pg_isolation2_regress always runs a "setup" test before the schedule. +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +CREATE +CREATE EXTENSION IF NOT EXISTS gp_inject_fault; +CREATE diff --git a/gpcontrib/gp_stats_collector/test/isolation2/isolation2_schedule b/gpcontrib/gp_stats_collector/test/isolation2/isolation2_schedule new file mode 100644 index 00000000000..5adb0d9cc0a --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/isolation2_schedule @@ -0,0 +1,20 @@ +# pg_query_state isolation2 schedule. +# +# gpsc_pqs_backends -- deterministic: an idle backend yields an empty backend +# list (no fault injector required). +# gpsc_pqs_running -- happy path: a query suspended mid-execution on the QEs +# is observed via pg_query_state_backends/pg_query_state. +# gpsc_pqs_perms -- permission gate: non-superuser non-owner is denied, +# superuser is allowed. +# gpsc_pqs_disabled -- STAT_DISABLED: target with pg_query_state.enable=off +# reports no backends despite a running query. +# gpsc_pqs_seg_count -- strict count: one participating backend per primary +# segment for a single-gang query. +# +# The gpsc_pqs_* specs after gpsc_pqs_backends use gp_inject_fault (enabled by +# default) to suspend a running query while it is polled. +test: gpsc_pqs_backends +test: gpsc_pqs_running +test: gpsc_pqs_perms +test: gpsc_pqs_disabled +test: gpsc_pqs_seg_count diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_backends.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_backends.sql new file mode 100644 index 00000000000..4df90380c7a --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_backends.sql @@ -0,0 +1,21 @@ +-- pg_query_state_backends against an *idle* backend returns an empty set. +-- +-- Deterministic multi-session check with no async race: session 1 tags itself +-- and sits idle; session 2 looks up its pid and polls it. An idle backend is +-- "not running a query", so GetRemoteBackendInfo returns QUERY_NOT_RUNNING and +-- the function yields an empty set (not an error). +-- +-- Extensions are created by setup.sql. + +-- Session 1: tag connection so session 2 can find its pid, then go idle. +1: SET application_name TO 'qs_idle_target'; +1: SELECT 1; + +-- Session 2: idle target -> zero participating backends. +2: SELECT count(*) AS n_backends FROM gpsc.pg_query_state_backends( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_idle_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1)); + +1q: +2q: diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_disabled.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_disabled.sql new file mode 100644 index 00000000000..10bf81d188a --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_disabled.sql @@ -0,0 +1,36 @@ +-- STAT_DISABLED: when the target backend has pg_query_state.enable = off, its +-- SendCdbComponents reply is STAT_DISABLED, so polling reports an empty backend +-- list even though a query is actively running on the segments. +-- +-- Distinguishes "disabled" from "idle": here the query really is executing +-- (suspended on a fault), yet the disabled module yields nothing. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_disabled_t (id int) DISTRIBUTED BY (id); +INSERT INTO qs_disabled_t SELECT generate_series(1, 100); + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- Target disables the module for its own session, then runs a query that hangs. +1: SET application_name TO 'qs_disabled_target'; +1: SET pg_query_state.enable TO off; +1&: SELECT count(*) FROM qs_disabled_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- Running, but module disabled on the target -> empty backend list. +2: SELECT count(*) AS n_backends FROM gpsc.pg_query_state_backends( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_disabled_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1)); + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; +1<: +1q: +2q: + +DROP TABLE qs_disabled_t; diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_perms.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_perms.sql new file mode 100644 index 00000000000..bbed5cefce2 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_perms.sql @@ -0,0 +1,57 @@ +-- Permission gate: a non-superuser that does not own the target query is +-- denied; a superuser is allowed. +-- +-- The gate is superuser() || GetUserId() == proc->roleId. isolation2 runs all +-- sessions under the same session role (the one that launched the harness), so +-- the "non-super owner is allowed" branch cannot be expressed here and is not +-- covered; the deny and superuser-allow branches are. +-- +-- A non-superuser cannot see another backend's application_name in +-- pg_stat_activity, so the target pid is captured (as superuser) into a table +-- before SET ROLE. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_perm_t (id int) DISTRIBUTED BY (id); +INSERT INTO qs_perm_t SELECT generate_series(1, 100); +CREATE TABLE qs_perm_pid (pid int); +CREATE ROLE qs_unpriv; +GRANT SELECT ON qs_perm_pid TO qs_unpriv; +-- No gpsc grants here on purpose: the extension grants USAGE/EXECUTE to PUBLIC +-- in its migration, so an ordinary role reaches the roleId gate exactly as it +-- would in production. This test verifies the gate, not the schema grants. + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +1: SET application_name TO 'qs_perm_target'; +1&: SELECT count(*) FROM qs_perm_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- Capture the target pid as superuser (sees application_name). +2: INSERT INTO qs_perm_pid SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_perm_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1; + +-- Non-superuser, non-owner: both entry points are denied. +2: SET ROLE qs_unpriv; +2: SELECT gpsc.pg_query_state((SELECT pid FROM qs_perm_pid)); +2: SELECT * FROM gpsc.pg_query_state_backends((SELECT pid FROM qs_perm_pid)); +2: RESET ROLE; + +-- Superuser: allowed (non-empty backend list). +2: SELECT count(*) > 0 AS has_backends + FROM gpsc.pg_query_state_backends((SELECT pid FROM qs_perm_pid)); + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; +1<: +1q: +2q: + +DROP OWNED BY qs_unpriv; +DROP ROLE qs_unpriv; +DROP TABLE qs_perm_pid; +DROP TABLE qs_perm_t; diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_running.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_running.sql new file mode 100644 index 00000000000..81c1359ce0d --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_running.sql @@ -0,0 +1,44 @@ +-- Happy path: a query suspended mid-execution on the QEs is observed live. +-- +-- Session 1 launches a query that hits an 'executor_pre_tuple_processed' +-- suspend fault on every primary segment, so its QE backends sit inside the +-- executor with a live plan tree. Session 2 then: +-- * pg_query_state_backends(pid) -> at least one participating backend, +-- * pg_query_state(pid) -> succeeds (fire-and-forget, returns void). +-- The fault is reset and the suspended query is reaped. +-- +-- Extensions (gp_stats_collector, gp_inject_fault) come from setup.sql. + +CREATE TABLE qs_running_t (id int) DISTRIBUTED BY (id); +INSERT INTO qs_running_t SELECT generate_series(1, 100); + +-- Suspend execution on all primary segments. +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- Session 1: tag the connection, then launch a query that hangs on the QEs. +1: SET application_name TO 'qs_running_target'; +1&: SELECT count(*) FROM qs_running_t; + +-- Wait until the fault has been hit on the segments. +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- Session 2: the running query has live QE backends, and polling succeeds. +2: SELECT count(*) > 0 AS has_backends FROM gpsc.pg_query_state_backends( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_running_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1)); +2: SELECT gpsc.pg_query_state( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_running_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1)); + +-- Release the fault and reap the suspended query. +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; +1<: +1q: +2q: + +DROP TABLE qs_running_t; diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_seg_count.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_seg_count.sql new file mode 100644 index 00000000000..d887f2e71c1 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_seg_count.sql @@ -0,0 +1,36 @@ +-- backends reports exactly one participating backend per primary segment for a +-- single-gang query -- a strict count rather than the has_backends>0 smoke +-- check in gpsc_pqs_running. +-- +-- A plain scan+count is one gang, so the QE list must match the number of +-- primary segments. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_segcount_t (id int) DISTRIBUTED BY (id); +INSERT INTO qs_segcount_t SELECT generate_series(1, 100); + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +1: SET application_name TO 'qs_segcount_target'; +1&: SELECT count(*) FROM qs_segcount_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- One backend per primary segment. +2: SELECT count(*) = (SELECT count(*) FROM gp_segment_configuration + WHERE role = 'p' AND content > -1) AS matches_primaries + FROM gpsc.pg_query_state_backends( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_segcount_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1)); + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; +1<: +1q: +2q: + +DROP TABLE qs_segcount_t; diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/setup.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/setup.sql new file mode 100644 index 00000000000..faec1135517 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/setup.sql @@ -0,0 +1,4 @@ +-- Shared setup for the pg_query_state isolation2 suite. +-- pg_isolation2_regress always runs a "setup" test before the schedule. +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +CREATE EXTENSION IF NOT EXISTS gp_inject_fault; diff --git a/gpcontrib/gp_stats_collector/test/sql/gpsc_pg_query_state.sql b/gpcontrib/gp_stats_collector/test/sql/gpsc_pg_query_state.sql new file mode 100644 index 00000000000..daa79235332 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/sql/gpsc_pg_query_state.sql @@ -0,0 +1,46 @@ +-- pg_query_state signal API (extension v1.2): catalog contract + negative paths. +-- +-- Deterministic coverage only: SQL-visible function/type registration and the +-- input-validation error branches. The asynchronous happy path (poll a live +-- query and observe per-node stats) is exercised separately under isolation2, +-- since it depends on a second running backend and timing. +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +-- end_ignore + +-- +-- Catalog contract: the three SQL-visible functions are registered in the gpsc +-- schema with the expected return type and dispatch (exec) location. +-- proexeclocation: c = coordinator, a = any (QE-local), s = all segments. +-- +SELECT proname, + pronargs, + prorettype::regtype AS returns, + proexeclocation +FROM pg_proc +WHERE pronamespace = 'gpsc'::regnamespace + AND proname IN ('pg_query_state', 'pg_query_state_backends', 'cbdb_mpp_query_state') +ORDER BY proname; + +-- Composite identifier type used by the signal layer is present. +SELECT typname +FROM pg_type +WHERE typnamespace = 'gpsc'::regnamespace + AND typname = 'gp_segment_pid'; + +-- +-- Negative: a backend cannot poll its own state. +-- +SELECT gpsc.pg_query_state(pg_backend_pid()); +SELECT * FROM gpsc.pg_query_state_backends(pg_backend_pid()); + +-- +-- Negative: a pid that maps to no live backend is rejected. +-- +SELECT gpsc.pg_query_state(-1); +SELECT * FROM gpsc.pg_query_state_backends(-1); + +-- Cleanup +-- start_ignore +DROP EXTENSION gp_stats_collector; +-- end_ignore diff --git a/pom.xml b/pom.xml index 51a7830d5ae..a029c92d271 100644 --- a/pom.xml +++ b/pom.xml @@ -1280,6 +1280,9 @@ code or new licensing patterns. gpcontrib/gp_stats_collector/gp_stats_collector.control gpcontrib/gp_stats_collector/.clang-format gpcontrib/gp_stats_collector/Makefile + gpcontrib/gp_stats_collector/test/Makefile + gpcontrib/gp_stats_collector/test/isolation2/Makefile + gpcontrib/gp_stats_collector/test/isolation2/isolation2_schedule