Skip to content
Open
55 changes: 49 additions & 6 deletions pipeline/authn/authenticator_oauth2_client_credentials.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"net/http"
"net/url"
"strings"
"sync"
"time"

"github.com/dgraph-io/ristretto/v2"
Expand All @@ -36,9 +37,30 @@ type clientCredentialsCacheConfig struct {
MaxTokens int `json:"max_tokens"`
}

// AuthenticatorOAuth2ClientCredentials authenticates requests via the OAuth2
// client credentials flow.
//
// Integrity fix: the original struct had no synchronization primitives, causing
// two distinct data races under concurrent request handling:
//
// 1. a.client was overwritten on every Config() invocation — any concurrent
// goroutine could observe a partially-initialized or entirely different
// *http.Client pointer, leading to non-deterministic transport behaviour.
//
// 2. a.TokenCache and a.cacheTTL were written without any lock, meaning the
// TOCTOU check "if a.TokenCache == nil" was not atomic: two goroutines
// could both observe nil, each allocate a new ristretto cache (spawning
// background goroutines), and then one would overwrite the other's pointer
// — leaking the orphaned cache and its goroutines permanently.
//
// The fix introduces:
// - clientOnce (sync.Once) to guarantee a.client is initialized exactly once.
// - mu (sync.Mutex) to serialize writes to a.TokenCache and a.cacheTTL.
type AuthenticatorOAuth2ClientCredentials struct {
d dependencies
client *http.Client
d dependencies
client *http.Client
clientOnce sync.Once
mu sync.Mutex

TokenCache *ristretto.Cache[string, []byte]
cacheTTL *time.Duration
Expand Down Expand Up @@ -94,14 +116,33 @@ func (a *AuthenticatorOAuth2ClientCredentials) Config(config json.RawMessage) (*
return nil, err
}
timeout := time.Millisecond * duration
a.client = httpx.NewResilientClient(
httpx.ResilientClientWithMaxRetryWait(maxWait),
httpx.ResilientClientWithConnectionTimeout(timeout),
).StandardClient()

// clientOnce.Do guarantees that a.client is allocated and written exactly
// once, regardless of how many goroutines are concurrently executing
// Config(). Subsequent calls observe the pointer set by the first caller
// without any unsynchronized write. Timeout and maxWait are captured from
// the first successful config resolution; the values are validated above
// before Do is reached, so the closure is always called with well-formed
// duration arguments.
a.clientOnce.Do(func() {
a.client = httpx.NewResilientClient(
httpx.ResilientClientWithMaxRetryWait(maxWait),
httpx.ResilientClientWithConnectionTimeout(timeout),
).StandardClient()
})

// mu serializes writes to a.cacheTTL and a.TokenCache so that the
// check-then-act sequence "if a.TokenCache == nil { ... a.TokenCache = cache }"
// is atomic. Without this lock, two goroutines can both observe nil,
// independently allocate separate ristretto caches (each spawning
// background goroutines), and then one silently overwrites the other's
// pointer, permanently leaking the orphaned cache.
a.mu.Lock()

if c.Cache.TTL != "" {
cacheTTL, err := time.ParseDuration(c.Cache.TTL)
if err != nil {
a.mu.Unlock()
return nil, err
}
a.cacheTTL = &cacheTTL
Expand All @@ -127,12 +168,14 @@ func (a *AuthenticatorOAuth2ClientCredentials) Config(config json.RawMessage) (*
IgnoreInternalCost: true,
})
if err != nil {
a.mu.Unlock()
return nil, err
}

a.TokenCache = cache
}

a.mu.Unlock()
return &c, nil
}

Expand Down
16 changes: 14 additions & 2 deletions pipeline/authn/authenticator_oauth2_introspection.go
Original file line number Diff line number Diff line change
Expand Up @@ -369,15 +369,25 @@ func (a *AuthenticatorOAuth2Introspection) Config(config json.RawMessage) (*Auth
a.mu.Unlock()
}

// Integrity fix: TokenCache initialization, cacheTTL pointer assignment, and
// TokenCache.Clear() are all brought under the existing a.mu write lock to
// eliminate the TOCTOU data race where concurrent goroutines could observe
// a nil TokenCache, each independently allocate a new cache instance, and
// overwrite each other — corrupting the singleton and leaking goroutines
// spawned by ristretto's internal workers.
a.mu.Lock()

if c.Cache.TTL != "" {
cacheTTL, err := time.ParseDuration(c.Cache.TTL)
if err != nil {
a.mu.Unlock()
return nil, nil, err
}

// clear cache if previous ttl was longer (or none)
// Clear cache if the previous TTL was longer (or unset), so stale
// entries with a longer lifetime than the new policy cannot linger.
if a.TokenCache != nil {
if a.cacheTTL == nil || (a.cacheTTL != nil && a.cacheTTL.Seconds() > cacheTTL.Seconds()) {
if a.cacheTTL == nil || a.cacheTTL.Seconds() > cacheTTL.Seconds() {
a.TokenCache.Clear()
}
}
Expand All @@ -404,11 +414,13 @@ func (a *AuthenticatorOAuth2Introspection) Config(config json.RawMessage) (*Auth
IgnoreInternalCost: true,
})
if err != nil {
a.mu.Unlock()
return nil, nil, err
}

a.TokenCache = cache
}

a.mu.Unlock()
return &c, client, nil
}
64 changes: 64 additions & 0 deletions pipeline/authn/repro_race_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
//go:build race
// +build race

// This test is a minimal concurrent harness to reproduce the data races
// in AuthenticatorOAuth2Introspection.Config() vs TokenToCache/TokenFromCache.
// Run with: go test -race -v ./pipeline/authn -run TestConfigDataRace
package authn_test

import (
"fmt"
"sync"
"testing"

"github.com/stretchr/testify/require"

"github.com/ory/x/configx"

. "github.com/ory/oathkeeper/pipeline/authn"
"github.com/ory/oathkeeper/internal"
)

// TestConfigDataRace launches concurrent writers invoking Config() with varying
// cache TTLs while readers concurrently hit TokenToCache/TokenFromCache.
// The Go race detector should report races on a.cacheTTL and a.TokenCache.
func TestConfigDataRace(t *testing.T) {
reg := internal.NewRegistry(t, configx.SkipValidation())
aa, err := reg.PipelineAuthenticator("oauth2_introspection")
require.NoError(t, err)
a, ok := aa.(*AuthenticatorOAuth2Introspection)
require.Truef(t, ok, "got type %T", aa)

base := `{"introspection_url":"http://127.0.0.1/oauth2/introspect","cache":{"enabled":true,"max_cost":1000}}`
ttls := []string{"25ms", "50ms", "75ms"}

var wg sync.WaitGroup

// Writers mutate TTL and (re)initialize cache concurrently.
for i := 0; i < 8; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
for j := 0; j < 2000; j++ {
cfg := fmt.Sprintf(`{"introspection_url":"http://127.0.0.1/oauth2/introspect","cache":{"enabled":true,"max_cost":1000,"ttl":"%s"}}`, ttls[j%len(ttls)])
_, _, _ = a.Config([]byte(cfg))
}
}(i)
}

// Readers exercise cache paths concurrently with Config() calls.
for i := 0; i < 8; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
res := &AuthenticatorOAuth2IntrospectionResult{Active: true}
for j := 0; j < 2000; j++ {
c, _, _ := a.Config([]byte(base))
a.TokenToCache(c, res, "tok", nil)
_ = a.TokenFromCache(c, "tok", nil)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
}
}(i)
}
Comment on lines +64 to +78

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🔴 Critical | 🏗️ Heavy lift

Test would fail under -raceTokenToCache/TokenFromCache still race with Config().

The fix in Config() adds a.mu.Lock() around a.cacheTTL writes and a.TokenCache initialization, but TokenToCache and TokenFromCache read a.cacheTTL and a.TokenCache without any lock (see authenticator_oauth2_introspection.go:155-181). This test's reader goroutines call TokenToCache (which reads a.cacheTTL at line if a.cacheTTL != nil { ... *a.cacheTTL }) concurrently with writer goroutines whose Config() calls write a.cacheTTL under a.mu.Lock(). The race detector will flag this as a data race, causing the test to fail.

The fix is incomplete: TokenToCache and TokenFromCache need to acquire a.mu.RLock() / a.mu.RUnlock() to be properly synchronized with Config()'s write lock. Since mu is already an RWMutex, this is straightforward.

🔒 Proposed fix: add RLock to TokenToCache and TokenFromCache

In authenticator_oauth2_introspection.go:

 func (a *AuthenticatorOAuth2Introspection) TokenFromCache(config *AuthenticatorOAuth2IntrospectionConfiguration, token string, ss fosite.ScopeStrategy) *AuthenticatorOAuth2IntrospectionResult {
 	if !config.Cache.Enabled {
 		return nil
 	}

 	if ss == nil && len(config.Scopes) > 0 {
 		return nil
 	}

+	a.mu.RLock()
+	defer a.mu.RUnlock()
+
 	key := TokenCacheKey(token, config.IntrospectionURL)
 	i, found := a.TokenCache.Get(key)
 func (a *AuthenticatorOAuth2Introspection) TokenToCache(config *AuthenticatorOAuth2IntrospectionConfiguration, i *AuthenticatorOAuth2IntrospectionResult, token string, ss fosite.ScopeStrategy) {
 	if !config.Cache.Enabled {
 		return
 	}

 	if ss == nil && len(config.Scopes) > 0 {
 		return
 	}

+	a.mu.RLock()
+	defer a.mu.RUnlock()
+
 	key := TokenCacheKey(token, config.IntrospectionURL)
 	v, err := json.Marshal(i)
 	if err != nil {
 		return
 	}

 	if a.cacheTTL != nil {
 		a.TokenCache.SetWithTTL(key, v, 1, *a.cacheTTL)
 	} else {
 		a.TokenCache.Set(key, v, 1)
 	}
 }
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@pipeline/authn/repro_race_test.go` around lines 50 - 61, `TokenToCache` and
`TokenFromCache` still access shared state unsafely, so they can race with
`Config()` despite the new write lock. Update both methods in
`AuthenticatorOAuth2Introspection` to take `a.mu.RLock()`/`a.mu.RUnlock()`
around reads of `a.cacheTTL` and `a.TokenCache`, matching the existing
`a.mu.Lock()` protection in `Config()`. This will keep cache initialization and
TTL reads synchronized and prevent the `repro_race_test.go` concurrency test
from failing under `-race`.


wg.Wait()
}
Loading