Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions packages/dd-trace/src/config/helper.js
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,39 @@ function validateAccess (name) {
}
}

/**
* In PM2 cluster mode, per-app env vars (DD_SERVICE, DD_ENV, etc.) are not
* passed as individual environment variables. Instead PM2 serializes the entire
* process config into a single `pm2_env` JSON string and passes only that to

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😢

Should we potentially open a PR against PM2 to stop this and instead just sets the envs when starting the child accordingly?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

* `cluster.fork()`. pm2 later unpacks it into process.env, but that happens
* inside the main module, after --require dd-trace/init` has already run.
*
* This function copies only DD_* and OTEL_* keys out of the pm2_env blob into
* process.env before config is read, so the tracer sees them at init time.
* Keys already present in process.env are not overwritten, so any host-level
* or explicit env var still takes precedence.
*/
Comment thread
tlhunter marked this conversation as resolved.
function applyPm2ClusterEnv () {
if (typeof process.env.pm2_env !== 'string') return

let pm2Config
try {
pm2Config = JSON.parse(process.env.pm2_env)
} catch {
return
}

for (const [key, value] of Object.entries(pm2Config)) {
if ((key.startsWith('DD_') || key.startsWith('OTEL_')) && !(key in process.env) && value != null) {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Honor PM2 per-app env overrides

In PM2 cluster mode the worker later unpacks pm2_env by assigning its keys into process.env, so per-app values in the PM2 config override any inherited daemon/shell environment. With this guard, if the PM2 daemon was started with DD_SERVICE=host but the app's ecosystem config sets DD_SERVICE=app, --require dd-trace/init keeps the inherited host value and caches the tracer config before PM2 performs its normal overwrite, so traces are reported under the wrong service for that app.

Useful? React with 👍 / 👎.

@BridgeAR BridgeAR Jun 11, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems a legitimate issue

Suggested change
if ((key.startsWith('DD_') || key.startsWith('OTEL_')) && !(key in process.env) && value != null) {
if (key.startsWith('DD_') || key.startsWith('OTEL_') || aliasToCanonical.has(key)) {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Copy supported env aliases from pm2_env

This only extracts DD_/OTEL_ names, but DATADOG_API_KEY is a supported alias for DD_API_KEY in supported-configurations.json. In PM2 cluster mode, users who put DATADOG_API_KEY in the app's PM2 env and preload the tracer will still get config.apiKey === undefined until PM2 unpacks the blob later, so agentless/LLMObs/log-submission paths can be disabled or drop data even though the app was configured with a valid supported key.

Useful? React with 👍 / 👎.

process.env[key] = String(value)
}
}
}

// Run at module load so that early reads in index.js (DD_TRACE_ENABLED,
// OTEL_TRACES_EXPORTER) see the correct values before getConfig() is called.
applyPm2ClusterEnv()

module.exports = {
/**
* Expose raw stable config maps and warnings for consumers that need
Expand Down Expand Up @@ -264,4 +297,6 @@ module.exports = {
}
}
},

applyPm2ClusterEnv,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please do not expose the internal helper and test it with proxyquire instead. That way we prevent changing production code for tests.

}
88 changes: 88 additions & 0 deletions packages/dd-trace/test/config/helper.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,94 @@ const proxyquire = require('proxyquire')

require('../setup/core')

const { applyPm2ClusterEnv } = require('../../src/config/helper')

describe('applyPm2ClusterEnv', () => {
let originalEnv

beforeEach(() => {
originalEnv = process.env
process.env = {}
})

afterEach(() => {
process.env = originalEnv
})

it('copies DD_* keys from pm2_env blob into process.env', () => {
process.env.pm2_env = JSON.stringify({ DD_SERVICE: 'pm2-service', DD_ENV: 'pm2-env' })
applyPm2ClusterEnv()
assert.strictEqual(process.env.DD_SERVICE, 'pm2-service')
assert.strictEqual(process.env.DD_ENV, 'pm2-env')
})

it('copies OTEL_* keys from pm2_env blob into process.env', () => {
process.env.pm2_env = JSON.stringify({ OTEL_SERVICE_NAME: 'pm2-otel-service' })
applyPm2ClusterEnv()
assert.strictEqual(process.env.OTEL_SERVICE_NAME, 'pm2-otel-service')
})

it('does not overwrite DD_* keys already present in process.env', () => {
process.env.DD_SERVICE = 'host-service'
process.env.pm2_env = JSON.stringify({ DD_SERVICE: 'pm2-service' })
applyPm2ClusterEnv()
assert.strictEqual(process.env.DD_SERVICE, 'host-service')
})

it('does not overwrite OTEL_* keys already present in process.env', () => {
process.env.OTEL_SERVICE_NAME = 'host-otel'
process.env.pm2_env = JSON.stringify({ OTEL_SERVICE_NAME: 'pm2-otel' })
applyPm2ClusterEnv()
assert.strictEqual(process.env.OTEL_SERVICE_NAME, 'host-otel')
})

it('does not copy non-DD/OTEL keys', () => {
process.env.pm2_env = JSON.stringify({ NODE_ENV: 'production', pm_id: 0 })
applyPm2ClusterEnv()
assert.strictEqual(process.env.NODE_ENV, undefined)
assert.strictEqual(process.env.pm_id, undefined)
})

it('skips keys with null values', () => {
process.env.pm2_env = JSON.stringify({ DD_SERVICE: null })
applyPm2ClusterEnv()
assert.strictEqual(process.env.DD_SERVICE, undefined)
})

it('coerces non-string values to string', () => {
process.env.pm2_env = JSON.stringify({ DD_TRACE_SAMPLE_RATE: 0.5 })
applyPm2ClusterEnv()
assert.strictEqual(process.env.DD_TRACE_SAMPLE_RATE, '0.5')
})

it('does nothing when pm2_env is absent', () => {
applyPm2ClusterEnv()
assert.deepStrictEqual(Object.keys(process.env), [])
})

it('does nothing when pm2_env is not a string', () => {
process.env.pm2_env = 42
applyPm2ClusterEnv()
assert.deepStrictEqual(Object.keys(process.env), ['pm2_env'])
})

it('does nothing when pm2_env is malformed JSON', () => {
process.env.pm2_env = 'not-valid-json'
applyPm2ClusterEnv()
assert.deepStrictEqual(Object.keys(process.env), ['pm2_env'])
})

it('populates process.env before module exports are first accessed (early-read scenario)', () => {
// Simulates the PM2 cluster mode timing: pm2_env is present in process.env
// when helper.js is first required, so the module-level call runs before
// index.js reads DD_TRACE_ENABLED / OTEL_TRACES_EXPORTER.
process.env.pm2_env = JSON.stringify({ DD_TRACE_ENABLED: 'false', DD_SERVICE: 'early-service' })
const { getValueFromEnvSources } = proxyquire.noPreserveCache()('../../src/config/helper', {})
assert.strictEqual(getValueFromEnvSources('DD_TRACE_ENABLED'), 'false')
assert.strictEqual(getValueFromEnvSources('DD_SERVICE'), 'early-service')
})
})

describe('config-helper stable config sources', () => {
let StableConfigStub

Expand Down
65 changes: 65 additions & 0 deletions packages/dd-trace/test/config/index.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -4629,4 +4629,69 @@ rules:
assert.strictEqual(config.service, 'node')
})
})

describe('PM2 cluster mode', () => {
it('should apply DD_* vars from pm2_env blob when not already set', () => {
process.env.pm2_env = JSON.stringify({ DD_SERVICE: 'pm2-service', DD_ENV: 'pm2-env' })
const config = getConfig()
assert.strictEqual(config.service, 'pm2-service')
assert.strictEqual(config.env, 'pm2-env')
})

it('should apply OTEL_* vars from pm2_env blob when not already set', () => {
process.env.pm2_env = JSON.stringify({ OTEL_SERVICE_NAME: 'pm2-otel-service' })
const config = getConfig()
assert.strictEqual(config.service, 'pm2-otel-service')
})

it('should not overwrite DD_* vars already present in process.env', () => {
process.env.DD_SERVICE = 'host-service'
process.env.pm2_env = JSON.stringify({ DD_SERVICE: 'pm2-service' })
const config = getConfig()
assert.strictEqual(config.service, 'host-service')
})

it('should not overwrite OTEL_* vars already present in process.env', () => {
process.env.OTEL_SERVICE_NAME = 'host-otel-service'
process.env.pm2_env = JSON.stringify({ OTEL_SERVICE_NAME: 'pm2-otel-service' })
const config = getConfig()
assert.strictEqual(config.service, 'host-otel-service')
})

it('should not copy non-DD/OTEL keys from pm2_env into process.env', () => {
process.env.pm2_env = JSON.stringify({ NODE_ENV: 'production', pm_id: 0, name: 'my-app' })
getConfig()
assert.strictEqual(process.env.NODE_ENV, undefined)
assert.strictEqual(process.env.pm_id, undefined)
assert.strictEqual(process.env.name, undefined)
})

it('should silently skip malformed pm2_env JSON', () => {
process.env.pm2_env = 'not-valid-json'
getConfig()
})

it('should skip keys with null values in pm2_env', () => {
process.env.pm2_env = JSON.stringify({ DD_SERVICE: null })
const config = getConfig()
assert.strictEqual(process.env.DD_SERVICE, undefined)
assert.strictEqual(config.service, 'node')
})

it('should do nothing when pm2_env is absent', () => {
const config = getConfig()
assert.strictEqual(config.service, 'node')
})

it('should do nothing when pm2_env is not a string', () => {
process.env.pm2_env = 42
getConfig()
})

it('should coerce non-string values to strings when copying into process.env', () => {
process.env.pm2_env = JSON.stringify({ DD_TRACE_SAMPLE_RATE: 0.5 })
getConfig()
assert.strictEqual(process.env.DD_TRACE_SAMPLE_RATE, '0.5')
})
})
})
Loading