[Credential Cache Pr 1/3] CachedSupplier ALLOW static stability with uniform backoff

core/sdk-core/src/main/java/software/amazon/awssdk/core/exception/CacheInvalidatingException.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.core.exception;
+
+import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.utils.cache.CacheInvalidatingError;
+
+/**
+ * An exception that signals a non-recoverable credential refresh failure.
+ * When thrown by a credential provider's refresh function, the caching layer
+ * will propagate this exception immediately to the caller without applying
+ * refresh backoff or extending cached credential expiration.
+ *
+ * <p>This is used for errors where the credential source has definitively
+ * indicated that the current authentication state is invalid and requires
+ * user intervention (e.g., expired SSO tokens, changed user credentials).</p>
+ */
+@SdkPublicApi
+public final class CacheInvalidatingException extends SdkClientException implements CacheInvalidatingError {
+
+    private CacheInvalidatingException(Builder builder) {
+        super(builder);
+    }
+
+    public static CacheInvalidatingException create(String message) {
+        return builder().message(message).build();
+    }
+
+    public static CacheInvalidatingException create(String message, Throwable cause) {
+        return builder().message(message).cause(cause).build();
+    }
+
+    @Override
+    public Builder toBuilder() {
+        return new BuilderImpl(this);
+    }
+
+    public static Builder builder() {
+        return new BuilderImpl();
+    }
+
+    public interface Builder extends SdkClientException.Builder {
+        @Override
+        Builder message(String message);
+
+        @Override
+        Builder cause(Throwable cause);
+
+        @Override
+        Builder writableStackTrace(Boolean writableStackTrace);
+
+        @Override
+        Builder numAttempts(Integer numAttempts);
+
+        @Override
+        CacheInvalidatingException build();
+    }
+
+    protected static final class BuilderImpl extends SdkClientException.BuilderImpl implements Builder {
+
+        protected BuilderImpl() {
+        }
+
+        protected BuilderImpl(CacheInvalidatingException ex) {
+            super(ex);
+        }
+
+        @Override
+        public Builder message(String message) {
+            this.message = message;
+            return this;
+        }
+
+        @Override
+        public Builder cause(Throwable cause) {
+            this.cause = cause;
+            return this;
+        }
+
+        @Override
+        public Builder writableStackTrace(Boolean writableStackTrace) {
+            this.writableStackTrace = writableStackTrace;
+            return this;
+        }
+
+        @Override
+        public Builder numAttempts(Integer numAttempts) {
+            this.numAttempts = numAttempts;
+            return this;
+        }
+
+        @Override
+        public CacheInvalidatingException build() {
+            return new CacheInvalidatingException(this);
+        }
+    }
+}

utils/src/main/java/software/amazon/awssdk/utils/cache/CacheInvalidatingError.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.utils.cache;
+
+import software.amazon.awssdk.annotations.SdkProtectedApi;
+
+/**
+ * Marker interface for exceptions that indicate a non-recoverable refresh failure.
+ * When thrown during a cache refresh, the caching layer will propagate the exception
+ * immediately without applying backoff or extending expiration.
+ *
+ * <p>Exceptions implementing this interface bypass cache static stability behavior,
+ * ensuring that actionable errors (such as expired tokens or changed credentials)
+ * are never suppressed by the caching layer.</p>
+ */
+@SdkProtectedApi
+public interface CacheInvalidatingError {
+}

utils/src/main/java/software/amazon/awssdk/utils/cache/CachedSupplier.java

@@ -23,7 +23,6 @@
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Supplier;
@@ -54,6 +53,16 @@ public class CachedSupplier<T> implements Supplier<T>, SdkAutoCloseable {
      */
     private static final Duration BLOCKING_REFRESH_MAX_WAIT = Duration.ofSeconds(5);
 
+    /**
+     * Minimum backoff duration in seconds when a refresh fails (inclusive).
+     */
+    private static final int STATIC_STABILITY_BACKOFF_MIN_SECONDS = 300;
+
+    /**
+     * Maximum backoff duration in seconds when a refresh fails (inclusive).
+     */
+    private static final int STATIC_STABILITY_BACKOFF_MAX_SECONDS = 600;
+
 
     /**
      * Used as a primitive form of rate limiting for the speed of our refreshes. This will make sure that the backing supplier has
@@ -83,11 +92,6 @@ public class CachedSupplier<T> implements Supplier<T>, SdkAutoCloseable {
      */
     private final Clock clock;
 
-    /**
-     * The number of consecutive failures encountered when updating a stale value.
-     */
-    private final AtomicInteger consecutiveStaleRetrievalFailures = new AtomicInteger(0);
-
     /**
      * The name to include with each log message, to differentiate caches.
      */
@@ -229,8 +233,6 @@ private void refreshCache() {
      * Perform necessary transformations of the successfully-fetched value based on the stale value behavior of this supplier.
      */
     private RefreshResult<T> handleFetchedSuccess(RefreshResult<T> fetch) {
-        consecutiveStaleRetrievalFailures.set(0);
-
         Instant now = clock.instant();
 
         if (now.isBefore(fetch.staleTime())) {
@@ -269,25 +271,55 @@ private RefreshResult<T> handleFetchFailure(RuntimeException e) {
 
         Instant now = clock.instant();
         if (!now.isBefore(currentCachedValue.staleTime())) {
-            int numFailures = consecutiveStaleRetrievalFailures.incrementAndGet();
-
             switch (staleValueBehavior) {
                 case STRICT:
                     throw e;
                 case ALLOW:
-                    Instant newStaleTime = jitterTime(now, Duration.ofMillis(1), maxStaleFailureJitter(numFailures));
-                    log.warn(() -> "(" + cachedValueName + ") Cached value expiration has been extended to " +
-                                   newStaleTime + " because calling the downstream service failed (consecutive failures: " +
-                                   numFailures + ").", e);
+                    // Cache-invalidating errors bypass static stability
+                    if (e instanceof CacheInvalidatingError) {
+                        throw e;
+                    }
+
+                    // Uniform random backoff: 5-10 minutes
+                    long backoffSeconds = STATIC_STABILITY_BACKOFF_MIN_SECONDS
+                        + jitterRandom.nextInt(
+                            STATIC_STABILITY_BACKOFF_MAX_SECONDS - STATIC_STABILITY_BACKOFF_MIN_SECONDS + 1);
+                    Instant extendedStaleTime = now.plusSeconds(backoffSeconds);
+
+                    log.warn(() -> "(" + cachedValueName + ") Credential refresh failed: " + e.getMessage()
+                                   + ". Extending cached credential expiration. A refresh of these credentials"
+                                   + " will be attempted again after " + backoffSeconds + " seconds.", e);
 
                     return currentCachedValue.toBuilder()
-                                             .staleTime(newStaleTime)
+                                             .staleTime(extendedStaleTime)
+                                             .prefetchTime(extendedStaleTime)
                                              .build();
                 default:
                     throw new IllegalStateException("Unknown stale-value-behavior: " + staleValueBehavior);
             }
         }
 
+        // Not yet stale — we're in the prefetch window. Handle failure based on mode.
+        if (staleValueBehavior == StaleValueBehavior.ALLOW) {
+            if (e instanceof CacheInvalidatingError) {
+                throw e;
+            }
+            // During prefetch window failure: extend prefetchTime to suppress further attempts
+            long backoffSeconds = STATIC_STABILITY_BACKOFF_MIN_SECONDS
+                + jitterRandom.nextInt(
+                    STATIC_STABILITY_BACKOFF_MAX_SECONDS - STATIC_STABILITY_BACKOFF_MIN_SECONDS + 1);
+            Instant extendedPrefetchTime = now.plusSeconds(backoffSeconds);
+
+            log.warn(() -> "(" + cachedValueName + ") Credential refresh failed: " + e.getMessage()
+                           + ". Extending cached credential expiration. A refresh of these credentials"
+                           + " will be attempted again after " + backoffSeconds + " seconds.", e);
+
+            return currentCachedValue.toBuilder()
+                                     .staleTime(extendedPrefetchTime)
+                                     .prefetchTime(extendedPrefetchTime)
+                                     .build();
+        }
+
         return currentCachedValue;
     }
 
@@ -333,6 +365,12 @@ private Duration maxPrefetchJitter(RefreshResult<T> result) {
         return timeBetweenPrefetchAndStale;
     }
 
+    private Instant jitterTime(Instant time, Duration jitterStart, Duration jitterEnd) {
+        long jitterRange = jitterEnd.minus(jitterStart).toMillis();
+        long jitterAmount = Math.abs(jitterRandom.nextLong() % jitterRange);
+        return time.plus(jitterStart).plusMillis(jitterAmount);
+    }
+
     private Duration maxStaleFailureJitter(int numFailures) {
         // prevent cycling back through low values
         if (numFailures > 63) {
@@ -350,12 +388,6 @@ protected Duration maxStaleFailureJitterTest(int numFailures) {
         return maxStaleFailureJitter(numFailures);
     }
 
-    private Instant jitterTime(Instant time, Duration jitterStart, Duration jitterEnd) {
-        long jitterRange = jitterEnd.minus(jitterStart).toMillis();
-        long jitterAmount = Math.abs(jitterRandom.nextLong() % jitterRange);
-        return time.plus(jitterStart).plusMillis(jitterAmount);
-    }
-
     /**
      * Free any resources consumed by the prefetch strategy this supplier is using.
      */
@@ -488,8 +520,14 @@ public enum StaleValueBehavior {
         STRICT,
 
         /**
-         * Allow stale values to be returned from the cache. Value retrieval will never fail, as long as the cache has
-         * succeeded when calling the underlying supplier at least once.
+         * Allow stale values to be returned from the cache with static stability semantics. On refresh failure,
+         * extends the stale time by a uniformly random backoff between 5 and 10 minutes (300-600 seconds).
+         *
+         * <p>If the failure is a {@link CacheInvalidatingError}, the exception is re-thrown immediately
+         * without extending the stale time.</p>
+         *
+         * <p>Value retrieval will never fail as long as the cache has succeeded at least once,
+         * unless the error is cache-invalidating.</p>
          */
         ALLOW
     }