Skip to content

fix(ci): fall back to Observer when live WorkflowRun logs are empty #733

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
artemiyokulov wants to merge 2 commits into
base: main
Choose a base branch
from
+163 −67
Open

fix(ci): fall back to Observer when live WorkflowRun logs are empty #733

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7527941
fix(ci): fall back to Observer when live WorkflowRun logs are empty
aokulovx5 Aug 4, 2026
004ae1b
chore: add changeset for CI WorkflowRun Observer log fallback
aokulovx5 Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 153 additions & 67 deletions plugins/openchoreo-ci-backend/src/services/WorkflowService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,11 @@ export class WorkflowService {

try {
if (hasLiveObservability) {
// Use OpenChoreo API directly for recent workflow runs
// Live path while the Argo Workflow CR exists. After
// podGC.strategy=OnWorkflowSuccess, pods are deleted but the CR (and
// hasLiveObservability=true) remains — live /logs then returns [].
// Align with GenericWorkflowService: fall back to Observer for
// terminal runs when live logs are empty.
const client = createOpenChoreoApiClient({
baseUrl: this.baseUrl,
token,
Expand Down Expand Up @@ -477,83 +481,54 @@ export class WorkflowService {
throw new Error('Failed to fetch workflow run logs: invalid payload');
}

const entries: LogEntry[] = data.map(entry => ({
const liveEntries: LogEntry[] = data.map(entry => ({
timestamp: entry.timestamp ?? '',
log: entry.log,
}));

this.logger.debug(
entries.length > 0
? `Successfully fetched ${entries.length} workflow run logs from openchoreo-api`
: `No live logs yet for run ${runName}`,
if (liveEntries.length > 0) {
this.logger.debug(
`Successfully fetched ${liveEntries.length} workflow run logs from openchoreo-api`,
);
return liveEntries;
}

// Incremental poll while running: empty means no new lines yet.
if (
typeof options.sinceSeconds === 'number' &&
options.sinceSeconds > 0
) {
this.logger.debug(
`No live logs yet for run ${runName} (sinceSeconds=${options.sinceSeconds})`,
);
return liveEntries;
}

const terminal = await this.isWorkflowRunTerminal(
namespaceName,
runName,
token,
);
return entries;
if (!terminal) {
this.logger.debug(`No live logs yet for run ${runName}`);
return liveEntries;
}

this.logger.info(
`Live logs empty for terminal run ${runName}; falling back to observer-api`,
);
// Fall through to Observer below.
}

// Use observer API for older workflow runs
const { observerUrl } = await this.resolver.resolveForBuild(
// Observer path: older runs, or terminal runs after podGC cleared live pods.
return await this.fetchWorkflowRunLogsFromObserver(
namespaceName,
projectName,
componentName,
runName,
options,
token,
);

if (!observerUrl) {
throw new ObservabilityNotConfiguredError(componentName);
}

const obsClient = createObservabilityClientWithUrl(
observerUrl,
token,
this.logger,
);

this.logger.debug(
`Sending workflow run logs request for component ${componentName} with run: ${runName}`,
);

// The observer rejects queries where the time range exceeds 30 days.
// Cap the window at 29 days to stay safely within that limit regardless
// of clock skew or boundary-comparison behaviour on the observer side.
const maxAllowedMs = 29 * 24 * 60 * 60 * 1000;
const requestedMs =
typeof options.sinceSeconds === 'number' && options.sinceSeconds > 0
? options.sinceSeconds * 1000
: maxAllowedMs;
const sinceMs = Math.min(requestedMs, maxAllowedMs);
const startTime = new Date(Date.now() - sinceMs).toISOString();
const endTime = new Date().toISOString();

const { data, error, response } = await obsClient.POST(
'/api/v1/logs/query',
{
body: {
startTime,
endTime,
limit: 1000,
sortOrder: 'asc',
searchScope: {
namespace: namespaceName,
workflowRunName: runName,
...(options.step ? { taskName: options.step } : {}),
},
},
},
);

assertApiResponse({ data, error, response }, 'fetch workflow run logs');

const entries: LogEntry[] = ((data?.logs || []) as any[]).map(
(entry: any) => ({
timestamp: entry.timestamp ?? '',
log: entry.log,
}),
);

this.logger.debug(
`Successfully fetched ${entries.length} workflow run logs from observer-api`,
);

return entries;
} catch (error: unknown) {
if (error instanceof ObservabilityNotConfiguredError) {
this.logger.info(
Expand All @@ -570,6 +545,117 @@ export class WorkflowService {
}
}

/**
* True when WorkflowRun has reached a terminal phase.
* Used to decide Observer fallback after live pod logs are gone (podGC).
*/
private async isWorkflowRunTerminal(
namespaceName: string,
runName: string,
token?: string,
): Promise<boolean> {
const terminal = new Set([
'Succeeded',
'Failed',
'Completed',
'Error',
'Cancelled',
]);
try {
const client = createOpenChoreoApiClient({
baseUrl: this.baseUrl,
token,
logger: this.logger,
});
const { data, error, response } = await client.GET(
'/api/v1/namespaces/{namespaceName}/workflowruns/{runName}',
{ params: { path: { namespaceName, runName } } },
);
if (error || !response.ok || !data) {
return false;
}
return terminal.has(deriveWorkflowRunStatus(data));
Comment on lines +552 to +577

@coderabbitai coderabbitai Bot Aug 4, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Description: Find all WorkflowRun Ready-condition reasons and task phases used in the repo.
set -uo pipefail

# Locate other terminal-status checks to compare the accepted value sets.
rg -nP -C4 "'Succeeded'|\"Succeeded\"" --type=ts -g '!**/node_modules/**' | head -100

# Locate reason/phase constants and unions.
rg -nP -C3 "(reason|phase)\s*[:=]\s*['\"]" --type=ts -g '!**/node_modules/**' | head -80

# Show every caller/definition of deriveWorkflowRunStatus.
rg -nP -C6 '\bderiveWorkflowRunStatus\b' --type=ts -g '!**/node_modules/**'

Repository: openchoreo/backstage-plugins

Length of output: 166

🏁 Script executed:

#!/bin/bash
set -u

echo "Repo files around workflow service:"
fd -a 'WorkflowService.ts' . || true

echo
echo "Git status/stat:"
git status --short || true
git diff --stat || true

echo
echo "Search deriveWorkflowRunStatus in TS files:"
rg -n -C 8 '\bderiveWorkflowRunStatus\b' . --glob '*.ts' || true

echo
echo "Search terminal status strings in TS files:"
rg -n -C 4 "Succeeded|Failed|Completed|Error|Cancelled|DeadlineExceeded|Terminated|Timeout|Running|Pending" . --glob '*.ts' || true

Repository: openchoreo/backstage-plugins

Length of output: 50385

🏁 Script executed:

#!/bin/bash
set -u

echo "Focus WorkspaceService.ts file:"
wc -l plugins/openchoreo-ci-backend/src/services/WorkflowService.ts
sed -n '1,90p' plugins/openchoreo-ci-runtime/src/services/WorkflowService.ts 2>/dev/null || true
sed -n '1,90p' plugins/openchoreo-ci-backend/src/services/WorkflowService.ts
sed -n '520,610p' plugins/openchoreo-ci-backend/src/services/WorkflowService.ts

echo
echo "targeted deriveWorkflowRunStatus occurrences:"
rg -n -C 8 '\bderiveWorkflowRunStatus\b' plugins packages --glob '*.ts' | sed -n '1,220p'

echo
echo "WorkflowRun status type definitions:"
rg -n -C 8 "WorkflowRun|workflowruns|completedAt|readyCondition" packages --glob '*.ts' plugins --glob '*.ts' | sed -n '1,260p'

echo
echo "terminal status occurrences outside generated:"
rg -n -C 4 "Succeeded|Failed|Completed|Error|Cancelled|DeadlineExceeded|Terminated|Timeout|Running|Pending" plugins packages --glob '*.ts' --glob '!**/packages/openchoreo-client-node/src/generated/**/*.ts' | sed -n '1,240p'

Repository: openchoreo/backstage-plugins

Length of output: 50384

🏁 Script executed:

#!/bin/bash
set -u

echo "WorkflowService surrounding fetchWorkflowRunLogs:"
sed -n '410,590p' plugins/openchoreo-ci-backend/src/services/WorkflowService.ts

echo
echo "Programmatic behavior probe for deriveWorkflowRunStatus / isWorkflowRunTerminal terminal allowlist"
node - <<'JS'
function deriveWorkflowRunStatus(run) {
  const readyCondition = (run.status?.conditions ?? []).find(
    (c) => c.type === 'Ready',
  );
  const tasks = run.status?.tasks ?? [];

  if (run.status?.completedAt) {
    if (tasks.some((t) => t.phase === 'Failed' || t.phase === 'Error')) {
      return 'Failed';
    }
    const reason = readyCondition?.reason;
    if (reason && reason !== 'Running' && reason !== 'Pending') {
      return reason;
    }
    return 'Succeeded';
  }

  if (readyCondition) {
    return readyCondition.reason ||
      (readyCondition.status === 'True' ? 'Succeeded' : 'Running');
  }

  if (tasks.some((t) => t.phase === 'Failed' || t.phase === 'Error')) {
    return 'Failed';
  }
  if (tasks.every((t) => t.phase === 'Succeeded') && tasks.length > 0) {
    return 'Succeeded';
  }
  if (tasks.some((t) => t.phase === 'Running')) {
    return 'Running';
  }
  if (run.status?.startedAt) return 'Running';
  return 'Pending';
}

const terminal = new Set([
  'Succeeded',
  'Failed',
  'Completed',
  'Error',
  'Cancelled',
]);

for (const name of [
  'DeadlineExceeded',
  'Terminated',
  'Timeout',
  'Cancelled',
  'Succeeded',
  'Running',
  'Pending',
]) {
  const run = {
    status: { completedAt: '2026-08-01T00:00:00Z', conditions: [{ type: 'Ready', reason: name }] },
  };
  const derived = deriveWorkflowRunStatus(run);
  console.log(JSON.stringify({ reason: name, derived, terminal, isTerminal: terminal.has(derived) }));
}
JS

Repository: openchoreo/backstage-plugins

Length of output: 6277

Treat completed workflow runs as terminal.

isWorkflowRunTerminal only checks a fixed set of status names, while deriveWorkflowRunStatus returns the Ready reason any time status.completedAt is set unless the reason is Running or Pending. Completed runs with reasons such as DeadlineExceeded, Terminated, or Timeout are seen as non-terminal, so fetchWorkflowRunLogs returns the empty live [] result instead of falling back to Observer. Use completedAt as the terminal signal here.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@plugins/openchoreo-ci-backend/src/services/WorkflowService.ts` around lines
552 - 577, Update isWorkflowRunTerminal to treat a workflow run as terminal
whenever the API data has completedAt set, regardless of the derived Ready
reason. Preserve the existing terminal-status handling for runs without
completedAt, so statuses such as Succeeded, Failed, and Cancelled remain
terminal.

} catch (err) {
this.logger.debug(
`Could not determine terminal status for run ${runName}: ${err}`,
);
return false;
}
}

private async fetchWorkflowRunLogsFromObserver(
namespaceName: string,
projectName: string,
componentName: string,
runName: string,
options: { step?: string; sinceSeconds?: number } = {},
token?: string,
): Promise<LogEntry[]> {
const { observerUrl } = await this.resolver.resolveForBuild(
namespaceName,
projectName,
token,
);

if (!observerUrl) {
throw new ObservabilityNotConfiguredError(componentName);
}

const obsClient = createObservabilityClientWithUrl(
observerUrl,
token,
this.logger,
);

this.logger.debug(
`Sending workflow run logs request for component ${componentName} with run: ${runName}`,
);

// The observer rejects queries where the time range exceeds 30 days.
// Cap the window at 29 days to stay safely within that limit regardless
// of clock skew or boundary-comparison behaviour on the observer side.
const maxAllowedMs = 29 * 24 * 60 * 60 * 1000;
const requestedMs =
typeof options.sinceSeconds === 'number' && options.sinceSeconds > 0
? options.sinceSeconds * 1000
: maxAllowedMs;
const sinceMs = Math.min(requestedMs, maxAllowedMs);
const startTime = new Date(Date.now() - sinceMs).toISOString();
const endTime = new Date().toISOString();

const { data, error, response } = await obsClient.POST(
'/api/v1/logs/query',
{
body: {
startTime,
endTime,
limit: 1000,
sortOrder: 'asc',
searchScope: {
namespace: namespaceName,
workflowRunName: runName,
...(options.step ? { taskName: options.step } : {}),
},
},
},
);

assertApiResponse({ data, error, response }, 'fetch workflow run logs');

const entries: LogEntry[] = ((data?.logs || []) as any[]).map(
(entry: any) => ({
timestamp: entry.timestamp ?? '',
log: entry.log,
}),
);

this.logger.debug(
`Successfully fetched ${entries.length} workflow run logs from observer-api`,
);

return entries;
}

async fetchWorkflowRunEvents(
namespaceName: string,
projectName: string,
Expand Down
Loading