From aa2c19e32d016d925a784acc18ef852e51329979 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 25 Jun 2025 13:44:11 +0200 Subject: [PATCH 01/42] refactor: tidy up RetryingClient - bundle parameters of private methods into a `RetryingContext` - extract attempt execution - improve naming of variables --- .../client/retry/AbstractRetryingClient.java | 16 + .../armeria/client/retry/RetryingClient.java | 388 ++++++++++-------- .../client/retry/RetryingRpcClient.java | 2 +- 3 files changed, 242 insertions(+), 164 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index e8e5cc277e3..00c516be514 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -255,11 +255,27 @@ protected static int getTotalAttempts(ClientRequestContext ctx) { /** * Creates a new derived {@link ClientRequestContext}, replacing the requests. * If {@link ClientRequestContext#endpointGroup()} exists, a new {@link Endpoint} will be selected. + * + * @deprecated Use {@link #newAttemptContext(ClientRequestContext, HttpRequest, RpcRequest, boolean)} + * instead. */ + @Deprecated protected static ClientRequestContext newDerivedContext(ClientRequestContext ctx, @Nullable HttpRequest req, @Nullable RpcRequest rpcReq, boolean initialAttempt) { + return newAttemptContext(ctx, req, rpcReq, initialAttempt); + } + + /** + * Creates a new {@link ClientRequestContext} for a retry attempt by replacing the request in + * {@link ClientRequestContext} with {@code req} or {@code rpcReq}. + * If {@link ClientRequestContext#endpointGroup()} exists, a new {@link Endpoint} will be selected. + */ + protected static ClientRequestContext newAttemptContext(ClientRequestContext ctx, + @Nullable HttpRequest req, + @Nullable RpcRequest rpcReq, + boolean initialAttempt) { return ClientUtil.newDerivedContext(ctx, req, rpcReq, initialAttempt); } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index d914bfece68..9c63ae5b5e9 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -242,19 +242,25 @@ public static Function newDecorator(RetryRul @Override protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) throws Exception { - final CompletableFuture responseFuture = new CompletableFuture<>(); - final HttpResponse res = HttpResponse.of(responseFuture, ctx.eventLoop()); + final CompletableFuture resFuture = new CompletableFuture<>(); + final HttpResponse res = HttpResponse.of(resFuture, ctx.eventLoop()); if (ctx.exchangeType().isRequestStreaming()) { - final HttpRequestDuplicator reqDuplicator = req.toDuplicator(ctx.eventLoop().withoutContext(), 0); - doExecute0(ctx, reqDuplicator, req, res, responseFuture); + final HttpRequestDuplicator reqDuplicator = + req.toDuplicator(ctx.eventLoop().withoutContext(), 0); + final RetryingContext retryingContext = new RetryingContext( + ctx, mappedRetryConfig(ctx), req, reqDuplicator, res, resFuture); + doExecute0(retryingContext); } else { req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) .handle((agg, cause) -> { if (cause != null) { - handleException(ctx, null, responseFuture, cause, true); + handleException(ctx, null, resFuture, cause, true); } else { final HttpRequestDuplicator reqDuplicator = new AggregatedHttpRequestDuplicator(agg); - doExecute0(ctx, reqDuplicator, req, res, responseFuture); + final RetryingContext retryingContext = new RetryingContext(ctx, mappedRetryConfig(ctx), + req, reqDuplicator, res, + resFuture); + doExecute0(retryingContext); } return null; }); @@ -262,234 +268,238 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro return res; } - private void doExecute0(ClientRequestContext ctx, HttpRequestDuplicator rootReqDuplicator, - HttpRequest originalReq, HttpResponse returnedRes, - CompletableFuture future) { - final int totalAttempts = getTotalAttempts(ctx); - final boolean initialAttempt = totalAttempts <= 1; + private void doExecute0(RetryingContext retryingContext) { + final RetryConfig config = retryingContext.config(); + final ClientRequestContext ctx = retryingContext.ctx(); + final HttpRequestDuplicator reqDuplicator = retryingContext.reqDuplicator(); + final HttpRequest req = retryingContext.req(); + final HttpResponse res = retryingContext.res(); + + final int attemptNo = getTotalAttempts(ctx); + final boolean isInitialAttempt = attemptNo <= 1; // The request or response has been aborted by the client before it receives a response, // so stop retrying. - if (originalReq.whenComplete().isCompletedExceptionally()) { - originalReq.whenComplete().handle((unused, cause) -> { - handleException(ctx, rootReqDuplicator, future, cause, initialAttempt); + if (req.whenComplete().isCompletedExceptionally()) { + req.whenComplete().handle((unused, cause) -> { + handleException(retryingContext, cause, isInitialAttempt); return null; }); return; } - if (returnedRes.isComplete()) { - returnedRes.whenComplete().handle((result, cause) -> { + if (res.isComplete()) { + res.whenComplete().handle((result, cause) -> { final Throwable abortCause; if (cause != null) { abortCause = cause; } else { abortCause = AbortedStreamException.get(); } - handleException(ctx, rootReqDuplicator, future, abortCause, initialAttempt); + handleException(retryingContext, abortCause, isInitialAttempt); return null; }); return; } if (!setResponseTimeout(ctx)) { - handleException(ctx, rootReqDuplicator, future, ResponseTimeoutException.get(), initialAttempt); + handleException(retryingContext, ResponseTimeoutException.get(), isInitialAttempt); return; } - final HttpRequest duplicateReq; - if (initialAttempt) { - duplicateReq = rootReqDuplicator.duplicate(); + final HttpRequest attemptReq; + if (isInitialAttempt) { + attemptReq = reqDuplicator.duplicate(); } else { - final RequestHeadersBuilder newHeaders = originalReq.headers().toBuilder(); - newHeaders.setInt(ARMERIA_RETRY_COUNT, totalAttempts - 1); - duplicateReq = rootReqDuplicator.duplicate(newHeaders.build()); + final RequestHeadersBuilder attemptHeadersBuilder = req.headers().toBuilder(); + attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, attemptNo - 1); + attemptReq = reqDuplicator.duplicate(attemptHeadersBuilder.build()); } - final ClientRequestContext derivedCtx; + final ClientRequestContext attemptCtx; try { - derivedCtx = newDerivedContext(ctx, duplicateReq, ctx.rpcRequest(), initialAttempt); + attemptCtx = newAttemptContext(ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); } catch (Throwable t) { - handleException(ctx, rootReqDuplicator, future, t, initialAttempt); + handleException(retryingContext, t, isInitialAttempt); return; } - final HttpRequest ctxReq = derivedCtx.request(); - assert ctxReq != null; - final HttpResponse response; - final ClientRequestContextExtension ctxExtension = derivedCtx.as(ClientRequestContextExtension.class); - if (!initialAttempt && ctxExtension != null && derivedCtx.endpoint() == null) { - // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(derivedCtx); - // if the endpoint hasn't been selected, try to initialize the ctx with a new endpoint/event loop - response = initContextAndExecuteWithFallback( - unwrap(), ctxExtension, HttpResponse::of, - (context, cause) -> HttpResponse.ofFailure(cause), ctxReq, false); - } else { - response = executeWithFallback(unwrap(), derivedCtx, - (context, cause) -> HttpResponse.ofFailure(cause), ctxReq, false); - } + final HttpResponse attemptRes = executeAttempt(attemptCtx, isInitialAttempt); - final RetryConfig config = mappedRetryConfig(ctx); if (!ctx.exchangeType().isResponseStreaming() || config.requiresResponseTrailers()) { - response.aggregate().handle((aggregated, cause) -> { + attemptRes.aggregate().handle((aggregatedAttemptRes, cause) -> { if (cause != null) { - derivedCtx.logBuilder().endRequest(cause); - derivedCtx.logBuilder().endResponse(cause); - handleResponseWithoutContent(config, ctx, rootReqDuplicator, originalReq, returnedRes, - future, derivedCtx, HttpResponse.ofFailure(cause), cause); + attemptCtx.logBuilder().endRequest(cause); + attemptCtx.logBuilder().endResponse(cause); + handleResponseWithoutContent( + retryingContext, attemptCtx, HttpResponse.ofFailure(cause), cause); } else { - completeLogIfBytesNotTransferred(aggregated, derivedCtx); - derivedCtx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { - handleAggregatedResponse(config, ctx, rootReqDuplicator, originalReq, returnedRes, - future, derivedCtx, aggregated); + completeLogIfBytesNotTransferred(aggregatedAttemptRes, attemptCtx); + attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { + handleAggregatedResponse(retryingContext, attemptCtx, aggregatedAttemptRes); }); } return null; }); } else { - handleStreamingResponse(config, ctx, rootReqDuplicator, originalReq, returnedRes, - future, derivedCtx, response); + handleStreamingResponse(retryingContext, attemptCtx, attemptRes); + } + } + + private HttpResponse executeAttempt(ClientRequestContext attemptCtx, boolean isInitialAttempt) { + final HttpRequest attemptReq = attemptCtx.request(); + assert attemptReq != null; + final HttpResponse attemptRes; + final ClientRequestContextExtension attemptCtxExtension = + attemptCtx.as(ClientRequestContextExtension.class); + if (!isInitialAttempt && attemptCtxExtension != null && attemptCtx.endpoint() == null) { + // clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); + // if the endpoint hasn't been selected, try to initialize the ctx with a new endpoint/event loop + attemptRes = initContextAndExecuteWithFallback( + unwrap(), attemptCtxExtension, HttpResponse::of, + (context, cause) -> + HttpResponse.ofFailure(cause), attemptReq, false); + } else { + attemptRes = executeWithFallback(unwrap(), attemptCtx, + (unused, cause) -> + HttpResponse.ofFailure(cause), attemptReq, false); } + return attemptRes; } // TODO(ikhoon): Add a request-scope class such as RetryRequestContext to avoid passing too many parameters. - private void handleResponseWithoutContent(RetryConfig config, ClientRequestContext ctx, - HttpRequestDuplicator rootReqDuplicator, HttpRequest originalReq, - HttpResponse returnedRes, CompletableFuture future, - ClientRequestContext derivedCtx, HttpResponse response, - @Nullable Throwable responseCause) { - if (responseCause != null) { - responseCause = Exceptions.peel(responseCause); + private void handleResponseWithoutContent(RetryingContext retryingContext, + ClientRequestContext attemptCtx, + HttpResponse attemptResWithoutContent, + @Nullable Throwable attemptResCause) { + if (attemptResCause != null) { + attemptResCause = Exceptions.peel(attemptResCause); } try { - final RetryRule retryRule = retryRule(config); - final CompletionStage f = retryRule.shouldRetry(derivedCtx, responseCause); + final RetryRule retryRule = retryRule(retryingContext.config()); + final CompletionStage f = retryRule.shouldRetry(attemptCtx, attemptResCause); f.handle((decision, shouldRetryCause) -> { warnIfExceptionIsRaised(retryRule, shouldRetryCause); - handleRetryDecision(decision, ctx, derivedCtx, rootReqDuplicator, - originalReq, returnedRes, future, response); + handleRetryDecision(retryingContext, decision, attemptCtx, attemptResWithoutContent); return null; }); } catch (Throwable cause) { - response.abort(); - handleException(ctx, rootReqDuplicator, future, cause, false); + attemptResWithoutContent.abort(); + handleException(retryingContext, cause, false); } } - private void handleStreamingResponse(RetryConfig retryConfig, ClientRequestContext ctx, - HttpRequestDuplicator rootReqDuplicator, - HttpRequest originalReq, HttpResponse returnedRes, - CompletableFuture future, - ClientRequestContext derivedCtx, - HttpResponse response) { - final SplitHttpResponse splitResponse = response.split(); - splitResponse.headers().handle((headers, headersCause) -> { - final Throwable responseCause; + private void handleStreamingResponse(RetryingContext retryingContext, + ClientRequestContext attemptCtx, + HttpResponse attemptRes) { + final RetryConfig retryConfig = retryingContext.config(); + + final SplitHttpResponse splitAttemptRes = attemptRes.split(); + splitAttemptRes.headers().handle((attemptResHeaders, headersCause) -> { + final Throwable attemptResCause; if (headersCause == null) { - final RequestLog log = derivedCtx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); - responseCause = log != null ? log.responseCause() : null; + final RequestLog log = attemptCtx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); + attemptResCause = log != null ? log.responseCause() : null; } else { - responseCause = Exceptions.peel(headersCause); + attemptResCause = Exceptions.peel(headersCause); } - completeLogIfBytesNotTransferred(response, headers, derivedCtx, responseCause); - - derivedCtx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { - if (retryConfig.needsContentInRule() && responseCause == null) { - final HttpResponse response0 = splitResponse.unsplit(); - final HttpResponseDuplicator duplicator = - response0.toDuplicator(derivedCtx.eventLoop().withoutContext(), - derivedCtx.maxResponseLength()); + completeLogIfBytesNotTransferred(attemptCtx, attemptRes, attemptResHeaders, attemptResCause); + + attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { + if (retryConfig.needsContentInRule() && attemptResCause == null) { + final HttpResponse unsplitAttemptRes = splitAttemptRes.unsplit(); + final HttpResponseDuplicator attemptResDuplicator = + unsplitAttemptRes.toDuplicator(attemptCtx.eventLoop().withoutContext(), + attemptCtx.maxResponseLength()); try { - final TruncatingHttpResponse truncatingHttpResponse = - new TruncatingHttpResponse(duplicator.duplicate(), + final TruncatingHttpResponse truncatingAttemptRes = + new TruncatingHttpResponse(attemptResDuplicator.duplicate(), retryConfig.maxContentLength()); - final HttpResponse duplicated = duplicator.duplicate(); - duplicator.close(); + final HttpResponse duplicatedAttemptRes = attemptResDuplicator.duplicate(); + attemptResDuplicator.close(); final RetryRuleWithContent ruleWithContent = retryConfig.retryRuleWithContent(); assert ruleWithContent != null; - ruleWithContent.shouldRetry(derivedCtx, truncatingHttpResponse, null) + ruleWithContent.shouldRetry(attemptCtx, truncatingAttemptRes, null) .handle((decision, cause) -> { warnIfExceptionIsRaised(ruleWithContent, cause); - truncatingHttpResponse.abort(); - handleRetryDecision(decision, ctx, derivedCtx, rootReqDuplicator, - originalReq, returnedRes, future, duplicated); + truncatingAttemptRes.abort(); + handleRetryDecision( + retryingContext, + decision, attemptCtx, duplicatedAttemptRes); return null; }); } catch (Throwable cause) { - duplicator.abort(cause); - handleException(ctx, rootReqDuplicator, future, cause, false); + attemptResDuplicator.abort(cause); + handleException(retryingContext, cause, false); } } else { - final HttpResponse response0; - if (responseCause != null) { - splitResponse.body().abort(responseCause); - response0 = HttpResponse.ofFailure(responseCause); + final HttpResponse unsplitAttemptRes; + if (attemptResCause != null) { + splitAttemptRes.body().abort(attemptResCause); + unsplitAttemptRes = HttpResponse.ofFailure(attemptResCause); } else { - response0 = splitResponse.unsplit(); + unsplitAttemptRes = splitAttemptRes.unsplit(); } - handleResponseWithoutContent(retryConfig, ctx, rootReqDuplicator, originalReq, returnedRes, - future, derivedCtx, response0, responseCause); + handleResponseWithoutContent(retryingContext, + attemptCtx, unsplitAttemptRes, attemptResCause); } }); return null; }); } - private void handleAggregatedResponse(RetryConfig retryConfig, ClientRequestContext ctx, - HttpRequestDuplicator rootReqDuplicator, - HttpRequest originalReq, HttpResponse returnedRes, - CompletableFuture future, - ClientRequestContext derivedCtx, - AggregatedHttpResponse aggregatedRes) { + private void handleAggregatedResponse(RetryingContext retryingContext, + ClientRequestContext attemptCtx, + AggregatedHttpResponse aggregatedAttemptRes) { + final RetryConfig retryConfig = retryingContext.config(); + if (retryConfig.needsContentInRule()) { final RetryRuleWithContent ruleWithContent = retryConfig.retryRuleWithContent(); assert ruleWithContent != null; try { - ruleWithContent.shouldRetry(derivedCtx, aggregatedRes.toHttpResponse(), null) + ruleWithContent.shouldRetry(attemptCtx, aggregatedAttemptRes.toHttpResponse(), null) .handle((decision, cause) -> { warnIfExceptionIsRaised(ruleWithContent, cause); - handleRetryDecision( - decision, ctx, derivedCtx, rootReqDuplicator, originalReq, - returnedRes, future, aggregatedRes.toHttpResponse()); + handleRetryDecision(retryingContext, + decision, attemptCtx, aggregatedAttemptRes.toHttpResponse()); return null; }); } catch (Throwable cause) { - handleException(ctx, rootReqDuplicator, future, cause, false); + handleException(retryingContext, cause, false); } return; } - handleResponseWithoutContent(retryConfig, ctx, rootReqDuplicator, originalReq, returnedRes, - future, derivedCtx, aggregatedRes.toHttpResponse(), null); + handleResponseWithoutContent(retryingContext, attemptCtx, aggregatedAttemptRes.toHttpResponse(), null); } - private static void completeLogIfBytesNotTransferred(AggregatedHttpResponse response, - ClientRequestContext ctx) { - if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { - final RequestLogBuilder logBuilder = ctx.logBuilder(); - logBuilder.endRequest(); - logBuilder.responseHeaders(response.headers()); - if (!response.trailers().isEmpty()) { - logBuilder.responseTrailers(response.trailers()); + private static void completeLogIfBytesNotTransferred(AggregatedHttpResponse aggregatedAttemptRes, + ClientRequestContext attemptCtx) { + if (!attemptCtx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { + final RequestLogBuilder attemptLogBuilder = attemptCtx.logBuilder(); + attemptLogBuilder.endRequest(); + attemptLogBuilder.responseHeaders(aggregatedAttemptRes.headers()); + if (!aggregatedAttemptRes.trailers().isEmpty()) { + attemptLogBuilder.responseTrailers(aggregatedAttemptRes.trailers()); } - logBuilder.endResponse(); + attemptLogBuilder.endResponse(); } } private static void completeLogIfBytesNotTransferred( - HttpResponse response, @Nullable ResponseHeaders headers, ClientRequestContext ctx, + ClientRequestContext attemptCtx, HttpResponse attemptRes, @Nullable ResponseHeaders attemptHeaders, @Nullable Throwable responseCause) { - if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { - final RequestLogBuilder logBuilder = ctx.logBuilder(); + if (!attemptCtx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { + final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); if (responseCause != null) { logBuilder.endRequest(responseCause); logBuilder.endResponse(responseCause); } else { logBuilder.endRequest(); - if (headers != null) { - logBuilder.responseHeaders(headers); + if (attemptHeaders != null) { + logBuilder.responseHeaders(attemptHeaders); } - response.whenComplete().handle((unused, cause) -> { + attemptRes.whenComplete().handle((unused, cause) -> { if (cause != null) { logBuilder.endResponse(cause); } else { @@ -507,13 +517,21 @@ private static void warnIfExceptionIsRaised(Object retryRule, @Nullable Throwabl } } + private static void handleException(RetryingContext retryingContext, + Throwable cause, + boolean endRequestLog) { + handleException( + retryingContext.ctx(), retryingContext.reqDuplicator(), retryingContext.resFuture(), cause, + endRequestLog); + } + private static void handleException(ClientRequestContext ctx, - @Nullable HttpRequestDuplicator rootReqDuplicator, - CompletableFuture future, Throwable cause, + @Nullable HttpRequestDuplicator reqDuplicator, + CompletableFuture resFuture, Throwable cause, boolean endRequestLog) { - future.completeExceptionally(cause); - if (rootReqDuplicator != null) { - rootReqDuplicator.abort(cause); + resFuture.completeExceptionally(cause); + if (reqDuplicator != null) { + reqDuplicator.abort(cause); } if (endRequestLog) { ctx.logBuilder().endRequest(cause); @@ -521,54 +539,53 @@ private static void handleException(ClientRequestContext ctx, ctx.logBuilder().endResponse(cause); } - private void handleRetryDecision(@Nullable RetryDecision decision, ClientRequestContext ctx, - ClientRequestContext derivedCtx, HttpRequestDuplicator rootReqDuplicator, - HttpRequest originalReq, HttpResponse returnedRes, - CompletableFuture future, HttpResponse originalRes) { + private void handleRetryDecision(RetryingContext retryingContext, @Nullable RetryDecision decision, + ClientRequestContext attemptCtx, HttpResponse attemptRes) { final Backoff backoff = decision != null ? decision.backoff() : null; if (backoff != null) { - final long millisAfter = useRetryAfter ? getRetryAfterMillis(derivedCtx) : -1; - final long nextDelay = getNextDelay(ctx, backoff, millisAfter); + final long millisAfter = useRetryAfter ? getRetryAfterMillis(attemptCtx) : -1; + final long nextDelay = getNextDelay(retryingContext.ctx(), backoff, millisAfter); if (nextDelay >= 0) { - abortResponse(originalRes, derivedCtx); + abortResponse(attemptRes, attemptCtx); scheduleNextRetry( - ctx, cause -> handleException(ctx, rootReqDuplicator, future, cause, false), - () -> doExecute0(ctx, rootReqDuplicator, originalReq, returnedRes, future), + retryingContext.ctx(), cause -> handleException(retryingContext, cause, false), + () -> doExecute0(retryingContext), nextDelay); return; } } - onRetryingComplete(ctx); - future.complete(originalRes); - rootReqDuplicator.close(); + onRetryingComplete(retryingContext.ctx()); + retryingContext.resFuture().complete(attemptRes); + retryingContext.reqDuplicator().close(); } - private static void abortResponse(HttpResponse originalRes, ClientRequestContext derivedCtx) { + private static void abortResponse(HttpResponse attemptRes, ClientRequestContext attemptCtx) { // Set response content with null to make sure that the log is complete. - final RequestLogBuilder logBuilder = derivedCtx.logBuilder(); + final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); logBuilder.responseContent(null, null); logBuilder.responseContentPreview(null); - originalRes.abort(); + attemptRes.abort(); } - private static long getRetryAfterMillis(ClientRequestContext ctx) { - final RequestLogAccess log = ctx.log(); - final String value; - final RequestLog requestLog = log.getIfAvailable(RequestLogProperty.RESPONSE_HEADERS); - value = requestLog != null ? requestLog.responseHeaders().get(HttpHeaderNames.RETRY_AFTER) : null; + private static long getRetryAfterMillis(ClientRequestContext attemptCtx) { + final RequestLogAccess attemptLog = attemptCtx.log(); + final String retryAfterValue; + final RequestLog requestLog = attemptLog.getIfAvailable(RequestLogProperty.RESPONSE_HEADERS); + retryAfterValue = requestLog != null ? + requestLog.responseHeaders().get(HttpHeaderNames.RETRY_AFTER) : null; - if (value != null) { + if (retryAfterValue != null) { try { - return Duration.ofSeconds(Integer.parseInt(value)).toMillis(); + return Duration.ofSeconds(Integer.parseInt(retryAfterValue)).toMillis(); } catch (Exception ignored) { // Not a second value. } try { @SuppressWarnings("UseOfObsoleteDateTimeApi") - final Date date = DateFormatter.parseHttpDate(value); - if (date != null) { - return date.getTime() - System.currentTimeMillis(); + final Date retryAfterDate = DateFormatter.parseHttpDate(retryAfterValue); + if (retryAfterDate != null) { + return retryAfterDate.getTime() - System.currentTimeMillis(); } } catch (Exception ignored) { // `parseHttpDate()` can raise an exception rather than returning `null` @@ -576,7 +593,7 @@ private static long getRetryAfterMillis(ClientRequestContext ctx) { } logger.debug("The retryAfter: {}, from the server is neither an HTTP date nor a second.", - value); + retryAfterValue); } return -1; @@ -591,4 +608,49 @@ private static RetryRule retryRule(RetryConfig retryConfig) { return rule; } } + + private static class RetryingContext { + private final ClientRequestContext ctx; + private final RetryConfig retryConfig; + private final HttpRequest req; + private final HttpRequestDuplicator reqDuplicator; + private final HttpResponse res; + private final CompletableFuture resFuture; + + RetryingContext(ClientRequestContext ctx, RetryConfig retryConfig, + HttpRequest req, HttpRequestDuplicator reqDuplicator, + HttpResponse res, + CompletableFuture resFuture) { + this.ctx = ctx; + this.retryConfig = retryConfig; + this.req = req; + this.reqDuplicator = reqDuplicator; + this.res = res; + this.resFuture = resFuture; + } + + RetryConfig config() { + return retryConfig; + } + + ClientRequestContext ctx() { + return ctx; + } + + HttpRequestDuplicator reqDuplicator() { + return reqDuplicator; + } + + HttpRequest req() { + return req; + } + + HttpResponse res() { + return res; + } + + CompletableFuture resFuture() { + return resFuture; + } + } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 32146db4c68..e0066267f60 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -163,7 +163,7 @@ private void doExecute0(ClientRequestContext ctx, RpcRequest req, return; } - final ClientRequestContext derivedCtx = newDerivedContext(ctx, null, req, initialAttempt); + final ClientRequestContext derivedCtx = newAttemptContext(ctx, null, req, initialAttempt); if (!initialAttempt) { derivedCtx.mutateAdditionalRequestHeaders( From 4aaa6781efc2f15bdd4e77629a5610884dfa4015 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 25 Jun 2025 13:58:22 +0200 Subject: [PATCH 02/42] refactor: tidy up RetryingRpcClient - extract attempt execution - improve naming of variables --- .../client/retry/RetryingRpcClient.java | 100 ++++++++++-------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index e0066267f60..dd44bce2d95 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -142,95 +142,101 @@ public static RetryingRpcClientBuilder builder(RetryConfigMapping m @Override protected RpcResponse doExecute(ClientRequestContext ctx, RpcRequest req) throws Exception { - final CompletableFuture future = new CompletableFuture<>(); - final RpcResponse res = RpcResponse.from(future); - doExecute0(ctx, req, res, future); + final CompletableFuture resFuture = new CompletableFuture<>(); + final RpcResponse res = RpcResponse.from(resFuture); + doExecute0(ctx, req, res, resFuture); return res; } private void doExecute0(ClientRequestContext ctx, RpcRequest req, - RpcResponse returnedRes, CompletableFuture future) { - final int totalAttempts = getTotalAttempts(ctx); - final boolean initialAttempt = totalAttempts <= 1; - if (returnedRes.isDone()) { + RpcResponse res, CompletableFuture resFuture) { + final int attemptNo = getTotalAttempts(ctx); + final boolean isInitialAttempt = attemptNo <= 1; + if (res.isDone()) { // The response has been cancelled by the client before it receives a response, so stop retrying. - handleException(ctx, future, new CancellationException( - "the response returned to the client has been cancelled"), initialAttempt); + handleException(ctx, resFuture, new CancellationException( + "the response returned to the client has been cancelled"), isInitialAttempt); return; } if (!setResponseTimeout(ctx)) { - handleException(ctx, future, ResponseTimeoutException.get(), initialAttempt); + handleException(ctx, resFuture, ResponseTimeoutException.get(), isInitialAttempt); return; } - final ClientRequestContext derivedCtx = newAttemptContext(ctx, null, req, initialAttempt); + final ClientRequestContext attemptCtx = newAttemptContext(ctx, null, req, isInitialAttempt); - if (!initialAttempt) { - derivedCtx.mutateAdditionalRequestHeaders( - mutator -> mutator.add(ARMERIA_RETRY_COUNT, StringUtil.toString(totalAttempts - 1))); + if (!isInitialAttempt) { + attemptCtx.mutateAdditionalRequestHeaders( + mutator -> mutator.add(ARMERIA_RETRY_COUNT, StringUtil.toString(attemptNo - 1))); } - final RpcResponse res; - - final ClientRequestContextExtension ctxExtension = derivedCtx.as(ClientRequestContextExtension.class); - final EndpointGroup endpointGroup = derivedCtx.endpointGroup(); - if (!initialAttempt && ctxExtension != null && - endpointGroup != null && derivedCtx.endpoint() == null) { - // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(derivedCtx); - // if the endpoint hasn't been selected, try to initialize the ctx with a new endpoint/event loop - res = initContextAndExecuteWithFallback(unwrap(), ctxExtension, RpcResponse::from, - (context, cause) -> RpcResponse.ofFailure(cause), - req, true); - } else { - res = executeWithFallback(unwrap(), derivedCtx, - (context, cause) -> RpcResponse.ofFailure(cause), - req, true); - } + final RpcResponse attemptRes = executeAttempt(req, attemptCtx, isInitialAttempt); final RetryConfig retryConfig = mappedRetryConfig(ctx); final RetryRuleWithContent retryRule = retryConfig.needsContentInRule() ? retryConfig.retryRuleWithContent() : retryConfig.fromRetryRule(); - res.handle((unused1, cause) -> { + attemptRes.handle((unused1, cause) -> { try { assert retryRule != null; - retryRule.shouldRetry(derivedCtx, res, cause).handle((decision, unused3) -> { + retryRule.shouldRetry(attemptCtx, attemptRes, cause).handle((decision, unused3) -> { final Backoff backoff = decision != null ? decision.backoff() : null; if (backoff != null) { - final long nextDelay = getNextDelay(derivedCtx, backoff); + final long nextDelay = getNextDelay(attemptCtx, backoff); if (nextDelay < 0) { - onRetryComplete(ctx, derivedCtx, res, future); + onRetryComplete(ctx, resFuture, attemptCtx, attemptRes); return null; } - scheduleNextRetry(ctx, cause0 -> handleException(ctx, future, cause0, false), - () -> doExecute0(ctx, req, returnedRes, future), nextDelay); + scheduleNextRetry(ctx, cause0 -> handleException(ctx, resFuture, cause0, false), + () -> doExecute0(ctx, req, res, resFuture), nextDelay); } else { - onRetryComplete(ctx, derivedCtx, res, future); + onRetryComplete(ctx, resFuture, attemptCtx, attemptRes); } return null; }); } catch (Throwable t) { - handleException(ctx, future, t, false); + handleException(ctx, resFuture, t, false); } return null; }); } - private static void onRetryComplete(ClientRequestContext ctx, ClientRequestContext derivedCtx, - RpcResponse res, CompletableFuture future) { + private RpcResponse executeAttempt(RpcRequest req, ClientRequestContext attemptCtx, + boolean isInitialAttempt) { + final RpcResponse attemptRes; + final ClientRequestContextExtension attemptCtxExtension = + attemptCtx.as(ClientRequestContextExtension.class); + final EndpointGroup endpointGroup = attemptCtx.endpointGroup(); + if (!isInitialAttempt && attemptCtxExtension != null && + endpointGroup != null && attemptCtx.endpoint() == null) { + // clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); + // if the endpoint hasn't been selected, try to initialize the ctx with a new endpoint/event loop + attemptRes = initContextAndExecuteWithFallback(unwrap(), attemptCtxExtension, RpcResponse::from, + (unused, cause) -> RpcResponse.ofFailure(cause), + req, true); + } else { + attemptRes = executeWithFallback(unwrap(), attemptCtx, + (unused, cause) -> RpcResponse.ofFailure(cause), + req, true); + } + return attemptRes; + } + + private static void onRetryComplete(ClientRequestContext ctx, CompletableFuture resFuture, + ClientRequestContext attemptCtx, RpcResponse attemptRes) { onRetryingComplete(ctx); - final HttpRequest actualHttpReq = derivedCtx.request(); - if (actualHttpReq != null) { - ctx.updateRequest(actualHttpReq); + final HttpRequest attemptReq = attemptCtx.request(); + if (attemptReq != null) { + ctx.updateRequest(attemptReq); } - future.complete(res); + resFuture.complete(attemptRes); } - private static void handleException(ClientRequestContext ctx, CompletableFuture future, + private static void handleException(ClientRequestContext ctx, CompletableFuture resFuture, Throwable cause, boolean endRequestLog) { - future.completeExceptionally(cause); + resFuture.completeExceptionally(cause); if (endRequestLog) { ctx.logBuilder().endRequest(cause); } From 3739f0c43590f1675012217110c7e4fc1d86e211 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 26 Jun 2025 18:17:11 +0200 Subject: [PATCH 03/42] [WIP] refactor: encapsulate an attempt into a separate class - RetryingClient is broken, just for discussion purposes --- .../armeria/client/retry/RetryingClient.java | 708 +++++++++++------- 1 file changed, 434 insertions(+), 274 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 9c63ae5b5e9..5cb6905116e 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -29,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.HttpClient; import com.linecorp.armeria.client.ResponseTimeoutException; @@ -247,20 +248,23 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro if (ctx.exchangeType().isRequestStreaming()) { final HttpRequestDuplicator reqDuplicator = req.toDuplicator(ctx.eventLoop().withoutContext(), 0); - final RetryingContext retryingContext = new RetryingContext( + final RetryingContext rctx = new RetryingContext( ctx, mappedRetryConfig(ctx), req, reqDuplicator, res, resFuture); - doExecute0(retryingContext); + final Attempt attempt = new Attempt(rctx, unwrap(), 1); + doExecuteAttempt(rctx, attempt); } else { req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) - .handle((agg, cause) -> { - if (cause != null) { - handleException(ctx, null, resFuture, cause, true); + .handle((agg, reqCause) -> { + if (reqCause != null) { + resFuture.completeExceptionally(reqCause); + ctx.logBuilder().endRequest(reqCause); + ctx.logBuilder().endResponse(reqCause); } else { final HttpRequestDuplicator reqDuplicator = new AggregatedHttpRequestDuplicator(agg); - final RetryingContext retryingContext = new RetryingContext(ctx, mappedRetryConfig(ctx), - req, reqDuplicator, res, - resFuture); - doExecute0(retryingContext); + final RetryingContext rctx = new RetryingContext( + ctx, mappedRetryConfig(ctx), req, reqDuplicator, res, resFuture); + final Attempt attempt = new Attempt(rctx, unwrap(), 1); + doExecuteAttempt(rctx, attempt); } return null; }); @@ -268,303 +272,142 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro return res; } - private void doExecute0(RetryingContext retryingContext) { - final RetryConfig config = retryingContext.config(); - final ClientRequestContext ctx = retryingContext.ctx(); - final HttpRequestDuplicator reqDuplicator = retryingContext.reqDuplicator(); - final HttpRequest req = retryingContext.req(); - final HttpResponse res = retryingContext.res(); - - final int attemptNo = getTotalAttempts(ctx); - final boolean isInitialAttempt = attemptNo <= 1; + private void doExecuteAttempt(RetryingContext rctx, Attempt attempt) { + assert attempt.state() == Attempt.State.INITIALIZED; + final boolean isInitialAttempt = attempt.number() <= 1; // The request or response has been aborted by the client before it receives a response, // so stop retrying. - if (req.whenComplete().isCompletedExceptionally()) { - req.whenComplete().handle((unused, cause) -> { - handleException(retryingContext, cause, isInitialAttempt); + if (rctx.req().whenComplete().isCompletedExceptionally()) { + rctx.req().whenComplete().handle((unused, cause) -> { + abort(rctx, cause, isInitialAttempt); return null; }); return; } - if (res.isComplete()) { - res.whenComplete().handle((result, cause) -> { + if (rctx.res().isComplete()) { + rctx.res().whenComplete().handle((result, cause) -> { final Throwable abortCause; if (cause != null) { abortCause = cause; } else { abortCause = AbortedStreamException.get(); } - handleException(retryingContext, abortCause, isInitialAttempt); + abort(rctx, abortCause, isInitialAttempt); return null; }); return; } - if (!setResponseTimeout(ctx)) { - handleException(retryingContext, ResponseTimeoutException.get(), isInitialAttempt); + if (!setResponseTimeout(rctx.ctx())) { + abort(rctx, ResponseTimeoutException.get(), isInitialAttempt); return; } - final HttpRequest attemptReq; - if (isInitialAttempt) { - attemptReq = reqDuplicator.duplicate(); - } else { - final RequestHeadersBuilder attemptHeadersBuilder = req.headers().toBuilder(); - attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, attemptNo - 1); - attemptReq = reqDuplicator.duplicate(attemptHeadersBuilder.build()); - } - - final ClientRequestContext attemptCtx; - try { - attemptCtx = newAttemptContext(ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); - } catch (Throwable t) { - handleException(retryingContext, t, isInitialAttempt); - return; - } + attempt.execute().handle((attemptProposal, unexpectedAttemptCause) -> { + if (unexpectedAttemptCause != null) { + assert attempt.state() == Attempt.State.ABORTED; + abort(rctx, unexpectedAttemptCause); + return null; + } - final HttpResponse attemptRes = executeAttempt(attemptCtx, isInitialAttempt); + assert attempt.state() == Attempt.State.PROPOSED; - if (!ctx.exchangeType().isResponseStreaming() || config.requiresResponseTrailers()) { - attemptRes.aggregate().handle((aggregatedAttemptRes, cause) -> { - if (cause != null) { - attemptCtx.logBuilder().endRequest(cause); - attemptCtx.logBuilder().endResponse(cause); - handleResponseWithoutContent( - retryingContext, attemptCtx, HttpResponse.ofFailure(cause), cause); - } else { - completeLogIfBytesNotTransferred(aggregatedAttemptRes, attemptCtx); - attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { - handleAggregatedResponse(retryingContext, attemptCtx, aggregatedAttemptRes); - }); + decideOnAttempt(rctx, attempt).handle((nextDelayMillis, unexpectedDecisionCause) -> { + if (unexpectedDecisionCause != null) { + attempt.abort(unexpectedDecisionCause); + assert attempt.state() == Attempt.State.ABORTED; + abort(rctx, unexpectedDecisionCause); + return null; } - return null; - }); - } else { - handleStreamingResponse(retryingContext, attemptCtx, attemptRes); - } - } - private HttpResponse executeAttempt(ClientRequestContext attemptCtx, boolean isInitialAttempt) { - final HttpRequest attemptReq = attemptCtx.request(); - assert attemptReq != null; - final HttpResponse attemptRes; - final ClientRequestContextExtension attemptCtxExtension = - attemptCtx.as(ClientRequestContextExtension.class); - if (!isInitialAttempt && attemptCtxExtension != null && attemptCtx.endpoint() == null) { - // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); - // if the endpoint hasn't been selected, try to initialize the ctx with a new endpoint/event loop - attemptRes = initContextAndExecuteWithFallback( - unwrap(), attemptCtxExtension, HttpResponse::of, - (context, cause) -> - HttpResponse.ofFailure(cause), attemptReq, false); - } else { - attemptRes = executeWithFallback(unwrap(), attemptCtx, - (unused, cause) -> - HttpResponse.ofFailure(cause), attemptReq, false); - } - return attemptRes; - } + assert attempt.state() == Attempt.State.PROPOSED; - // TODO(ikhoon): Add a request-scope class such as RetryRequestContext to avoid passing too many parameters. - private void handleResponseWithoutContent(RetryingContext retryingContext, - ClientRequestContext attemptCtx, - HttpResponse attemptResWithoutContent, - @Nullable Throwable attemptResCause) { - if (attemptResCause != null) { - attemptResCause = Exceptions.peel(attemptResCause); - } - try { - final RetryRule retryRule = retryRule(retryingContext.config()); - final CompletionStage f = retryRule.shouldRetry(attemptCtx, attemptResCause); - f.handle((decision, shouldRetryCause) -> { - warnIfExceptionIsRaised(retryRule, shouldRetryCause); - handleRetryDecision(retryingContext, decision, attemptCtx, attemptResWithoutContent); - return null; - }); - } catch (Throwable cause) { - attemptResWithoutContent.abort(); - handleException(retryingContext, cause, false); - } - } + if (nextDelayMillis >= 0) { + attempt.abort(); + assert attempt.state() == Attempt.State.ABORTED; - private void handleStreamingResponse(RetryingContext retryingContext, - ClientRequestContext attemptCtx, - HttpResponse attemptRes) { - final RetryConfig retryConfig = retryingContext.config(); - - final SplitHttpResponse splitAttemptRes = attemptRes.split(); - splitAttemptRes.headers().handle((attemptResHeaders, headersCause) -> { - final Throwable attemptResCause; - if (headersCause == null) { - final RequestLog log = attemptCtx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); - attemptResCause = log != null ? log.responseCause() : null; - } else { - attemptResCause = Exceptions.peel(headersCause); - } - completeLogIfBytesNotTransferred(attemptCtx, attemptRes, attemptResHeaders, attemptResCause); - - attemptCtx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { - if (retryConfig.needsContentInRule() && attemptResCause == null) { - final HttpResponse unsplitAttemptRes = splitAttemptRes.unsplit(); - final HttpResponseDuplicator attemptResDuplicator = - unsplitAttemptRes.toDuplicator(attemptCtx.eventLoop().withoutContext(), - attemptCtx.maxResponseLength()); - try { - final TruncatingHttpResponse truncatingAttemptRes = - new TruncatingHttpResponse(attemptResDuplicator.duplicate(), - retryConfig.maxContentLength()); - final HttpResponse duplicatedAttemptRes = attemptResDuplicator.duplicate(); - attemptResDuplicator.close(); - - final RetryRuleWithContent ruleWithContent = - retryConfig.retryRuleWithContent(); - assert ruleWithContent != null; - ruleWithContent.shouldRetry(attemptCtx, truncatingAttemptRes, null) - .handle((decision, cause) -> { - warnIfExceptionIsRaised(ruleWithContent, cause); - truncatingAttemptRes.abort(); - handleRetryDecision( - retryingContext, - decision, attemptCtx, duplicatedAttemptRes); - return null; - }); - } catch (Throwable cause) { - attemptResDuplicator.abort(cause); - handleException(retryingContext, cause, false); - } - } else { - final HttpResponse unsplitAttemptRes; - if (attemptResCause != null) { - splitAttemptRes.body().abort(attemptResCause); - unsplitAttemptRes = HttpResponse.ofFailure(attemptResCause); - } else { - unsplitAttemptRes = splitAttemptRes.unsplit(); - } - handleResponseWithoutContent(retryingContext, - attemptCtx, unsplitAttemptRes, attemptResCause); + scheduleNextRetry( + rctx.ctx(), schedulingCause -> abort(rctx, schedulingCause), + () -> { + final Attempt nextAttempt = new Attempt(rctx, unwrap(), attempt.number() + 1); + doExecuteAttempt(rctx, nextAttempt); + }, + nextDelayMillis); + return null; } + + commit(rctx, attempt); + assert attempt.state() == Attempt.State.COMMITTED; + return null; }); + return null; }); } - private void handleAggregatedResponse(RetryingContext retryingContext, - ClientRequestContext attemptCtx, - AggregatedHttpResponse aggregatedAttemptRes) { - final RetryConfig retryConfig = retryingContext.config(); - - if (retryConfig.needsContentInRule()) { - final RetryRuleWithContent ruleWithContent = retryConfig.retryRuleWithContent(); - assert ruleWithContent != null; - try { - ruleWithContent.shouldRetry(attemptCtx, aggregatedAttemptRes.toHttpResponse(), null) - .handle((decision, cause) -> { - warnIfExceptionIsRaised(ruleWithContent, cause); - handleRetryDecision(retryingContext, - decision, attemptCtx, aggregatedAttemptRes.toHttpResponse()); - return null; - }); - } catch (Throwable cause) { - handleException(retryingContext, cause, false); - } - return; - } - handleResponseWithoutContent(retryingContext, attemptCtx, aggregatedAttemptRes.toHttpResponse(), null); - } - - private static void completeLogIfBytesNotTransferred(AggregatedHttpResponse aggregatedAttemptRes, - ClientRequestContext attemptCtx) { - if (!attemptCtx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { - final RequestLogBuilder attemptLogBuilder = attemptCtx.logBuilder(); - attemptLogBuilder.endRequest(); - attemptLogBuilder.responseHeaders(aggregatedAttemptRes.headers()); - if (!aggregatedAttemptRes.trailers().isEmpty()) { - attemptLogBuilder.responseTrailers(aggregatedAttemptRes.trailers()); - } - attemptLogBuilder.endResponse(); + private CompletionStage decideOnAttempt(RetryingContext rctx, Attempt attempt) { + assert attempt.state() == Attempt.State.PROPOSED; + final Attempt.Result result = attempt.result(); + final CompletionStage<@Nullable RetryDecision> retryDecisionFuture; + + if (rctx.config().needsContentInRule()) { + final RetryRuleWithContent retryRuleWithContent = + rctx.config().retryRuleWithContent(); + assert retryRuleWithContent != null; + retryDecisionFuture = retryRuleWithContent + .shouldRetry(result.ctx(), result.res(), result.cause()) + .exceptionally(cause -> { + logger.warn("Unexpected exception is raised from {}.", retryRuleWithContent, cause); + return null; + }); + } else { + final RetryRule retryRule = rctx.config().retryRule(); + assert retryRule != null; + retryDecisionFuture = retryRule.shouldRetry(result.ctx(), result.cause()) + .exceptionally(cause -> { + logger.warn("Unexpected exception is raised from {}.", retryRule, + cause); + return null; + }); } - } - private static void completeLogIfBytesNotTransferred( - ClientRequestContext attemptCtx, HttpResponse attemptRes, @Nullable ResponseHeaders attemptHeaders, - @Nullable Throwable responseCause) { - if (!attemptCtx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { - final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); - if (responseCause != null) { - logBuilder.endRequest(responseCause); - logBuilder.endResponse(responseCause); - } else { - logBuilder.endRequest(); - if (attemptHeaders != null) { - logBuilder.responseHeaders(attemptHeaders); + return retryDecisionFuture.handle((decision, cause) -> { + assert cause == null; + assert attempt.state() == Attempt.State.PROPOSED; + final Backoff backoff = decision != null ? decision.backoff() : null; + if (backoff != null) { + final long millisAfter = useRetryAfter ? getRetryAfterMillis(result.ctx()) : -1; + final long nextDelay = getNextDelay(result.ctx(), backoff, millisAfter); + if (nextDelay >= 0) { + return nextDelay; } - attemptRes.whenComplete().handle((unused, cause) -> { - if (cause != null) { - logBuilder.endResponse(cause); - } else { - logBuilder.endResponse(); - } - return null; - }); } - } - } - private static void warnIfExceptionIsRaised(Object retryRule, @Nullable Throwable cause) { - if (cause != null) { - logger.warn("Unexpected exception is raised from {}.", retryRule, cause); - } + return -1L; + }); } - private static void handleException(RetryingContext retryingContext, - Throwable cause, - boolean endRequestLog) { - handleException( - retryingContext.ctx(), retryingContext.reqDuplicator(), retryingContext.resFuture(), cause, - endRequestLog); + private static void commit(RetryingContext rctx, Attempt attempt) { + assert attempt.state() == Attempt.State.PROPOSED; + attempt.commit(); + assert attempt.state() == Attempt.State.COMMITTED; + final Attempt.Result attemptResultToCommit = attempt.result(); + onRetryingComplete(rctx.ctx()); + rctx.resFuture().complete(attemptResultToCommit.res()); + rctx.reqDuplicator().close(); } - private static void handleException(ClientRequestContext ctx, - @Nullable HttpRequestDuplicator reqDuplicator, - CompletableFuture resFuture, Throwable cause, - boolean endRequestLog) { - resFuture.completeExceptionally(cause); - if (reqDuplicator != null) { - reqDuplicator.abort(cause); - } - if (endRequestLog) { - ctx.logBuilder().endRequest(cause); - } - ctx.logBuilder().endResponse(cause); + private static void abort(RetryingContext rctx, Throwable cause) { + abort(rctx, cause, false); } - private void handleRetryDecision(RetryingContext retryingContext, @Nullable RetryDecision decision, - ClientRequestContext attemptCtx, HttpResponse attemptRes) { - final Backoff backoff = decision != null ? decision.backoff() : null; - if (backoff != null) { - final long millisAfter = useRetryAfter ? getRetryAfterMillis(attemptCtx) : -1; - final long nextDelay = getNextDelay(retryingContext.ctx(), backoff, millisAfter); - if (nextDelay >= 0) { - abortResponse(attemptRes, attemptCtx); - scheduleNextRetry( - retryingContext.ctx(), cause -> handleException(retryingContext, cause, false), - () -> doExecute0(retryingContext), - nextDelay); - return; - } + private static void abort(RetryingContext rctx, Throwable cause, boolean endRequestLog) { + rctx.resFuture().completeExceptionally(cause); + rctx.reqDuplicator().abort(cause); + if (endRequestLog) { + rctx.ctx().logBuilder().endRequest(cause); } - onRetryingComplete(retryingContext.ctx()); - retryingContext.resFuture().complete(attemptRes); - retryingContext.reqDuplicator().close(); - } - - private static void abortResponse(HttpResponse attemptRes, ClientRequestContext attemptCtx) { - // Set response content with null to make sure that the log is complete. - final RequestLogBuilder logBuilder = attemptCtx.logBuilder(); - logBuilder.responseContent(null, null); - logBuilder.responseContentPreview(null); - attemptRes.abort(); + rctx.ctx().logBuilder().endResponse(cause); } private static long getRetryAfterMillis(ClientRequestContext attemptCtx) { @@ -599,16 +442,6 @@ private static long getRetryAfterMillis(ClientRequestContext attemptCtx) { return -1; } - private static RetryRule retryRule(RetryConfig retryConfig) { - if (retryConfig.needsContentInRule()) { - return retryConfig.fromRetryRuleWithContent(); - } else { - final RetryRule rule = retryConfig.retryRule(); - assert rule != null; - return rule; - } - } - private static class RetryingContext { private final ClientRequestContext ctx; private final RetryConfig retryConfig; @@ -617,10 +450,13 @@ private static class RetryingContext { private final HttpResponse res; private final CompletableFuture resFuture; - RetryingContext(ClientRequestContext ctx, RetryConfig retryConfig, - HttpRequest req, HttpRequestDuplicator reqDuplicator, + RetryingContext(ClientRequestContext ctx, + RetryConfig retryConfig, + HttpRequest req, + HttpRequestDuplicator reqDuplicator, HttpResponse res, CompletableFuture resFuture) { + this.ctx = ctx; this.retryConfig = retryConfig; this.req = req; @@ -653,4 +489,328 @@ CompletableFuture resFuture() { return resFuture; } } + + private static class Attempt { + private enum State { + // State of the attempt after construction and before the call to execute(). + INITIALIZED, + // State of the attempt before the outer call to execute() and before the inner call to propose(). + EXECUTING, + // State of the attempt before the call to commit() and before the outer call to commit() or abort() + // It means that this attempt ended up with a response, with or without content depending on the + // RetryRule, and it is up to the caller to decide on whether to commit on this response (i.e., + // returning this response from this client) or to abort it we are continue retrying). + PROPOSED, + // After the outer call to commit(). Terminal state. + COMMITTED, + ABORTED + // After the outer call to abort() or after an unexpected exception. Terminal state. + // After setting this state, all attempt-related resources are freed. + } + + static class Result { + private final ClientRequestContext ctx; + @Nullable + private final HttpResponse res; + @Nullable + private final Throwable cause; + + Result(ClientRequestContext ctx, @Nullable HttpResponse res, @Nullable Throwable cause) { + this.ctx = ctx; + this.res = res; + this.cause = cause; + } + + ClientRequestContext ctx() { + return ctx; + } + + @Nullable + HttpResponse res() { + return res; + } + + @Nullable + Throwable cause() { + return cause; + } + + void close() { + if (res != null) { + // abort() is idempotent + res.abort(); + } + } + } + + private State state; + + @Nullable + private Result result; + private final CompletableFuture proposalFuture; + + RetryingContext rctx; + + final int number; + + // Available only in Attempt.State.EXECUTING. + @Nullable + private ClientRequestContext ctx; + @Nullable + private HttpResponse res; + + Client delegate; + + Attempt(RetryingContext rctx, Client delegate, int number) { + this.rctx = rctx; + this.delegate = delegate; + this.number = number; + proposalFuture = new CompletableFuture<>(); + state = State.INITIALIZED; + } + + int number() { + return number; + } + + State state() { + return state; + } + + CompletableFuture execute() { + assert state == State.INITIALIZED; + final boolean isInitialAttempt = number <= 1; + + final HttpRequest attemptReq; + if (isInitialAttempt) { + attemptReq = rctx.reqDuplicator().duplicate(); + } else { + final RequestHeadersBuilder attemptHeadersBuilder = rctx.req().headers().toBuilder(); + attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, number - 1); + attemptReq = rctx.reqDuplicator().duplicate(attemptHeadersBuilder.build()); + } + + try { + ctx = newAttemptContext( + rctx.ctx(), attemptReq, rctx.ctx().rpcRequest(), isInitialAttempt); + } catch (Throwable t) { + abort(t); + return proposalFuture; + } + + executeAttemptRequest(); + assert state == State.EXECUTING && res != null && ctx != null; + + if (!rctx.ctx().exchangeType().isResponseStreaming() || rctx.config().requiresResponseTrailers()) { + handleAggRes(); + } else { + handleStreamingRes(); + } + + return proposalFuture; + } + + Result result() { + assert state == State.PROPOSED || state == State.COMMITTED; + assert result != null; + return result; + } + + void commit() { + if (state == State.COMMITTED) { + return; + } + assert state == State.PROPOSED; + state = State.COMMITTED; + + if (result != null) { + result.close(); + } + } + + void abort() { + abort(AbortedStreamException.get()); + } + + void abort(Throwable cause) { + if (state == State.ABORTED) { + return; + } + assert state == State.EXECUTING || state == State.PROPOSED; + assert ctx != null && res != null; + state = State.ABORTED; + + if (result != null) { + // Frees intermediate resources. + result.close(); + } + + final RequestLogBuilder logBuilder = ctx.logBuilder(); + // Set response content with null to make sure that the log is complete. + logBuilder.responseContent(null, null); + logBuilder.responseContentPreview(null); + res.abort(cause); + + if (!proposalFuture.isDone()) { + proposalFuture.completeExceptionally(cause); + } + } + + private void executeAttemptRequest() { + assert state == State.INITIALIZED; + assert ctx != null; + final HttpRequest req = ctx.request(); + assert req != null; + final ClientRequestContextExtension ctxExtension = + ctx.as(ClientRequestContextExtension.class); + if ((number > 1) && ctxExtension != null && ctx.endpoint() == null) { + // clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(ctx); + // if the endpoint hasn't been selected, + // try to initialize the ctx with a new endpoint/event loop + res = initContextAndExecuteWithFallback( + delegate, ctxExtension, HttpResponse::of, + (context, cause) -> + HttpResponse.ofFailure(cause), req, false); + } else { + res = executeWithFallback(delegate, ctx, + (context, cause) -> + HttpResponse.ofFailure(cause), req, false); + } + + state = State.EXECUTING; + } + + private void handleAggRes() { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + res.aggregate().handle((aggRes, resCause) -> { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + if (resCause != null) { + ctx.logBuilder().endRequest(resCause); + ctx.logBuilder().endResponse(resCause); + proposeResult(HttpResponse.ofFailure(resCause), resCause); + } else { + completeLogIfBytesNotTransferred(aggRes); + ctx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { + proposeResult(aggRes.toHttpResponse(), null); + }); + } + return null; + }); + } + + private void handleStreamingRes() { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + final SplitHttpResponse splitRes = res.split(); + splitRes.headers().handle((resHeaders, headersCause) -> { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + final Throwable resCause; + if (headersCause == null) { + final RequestLog log = ctx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); + resCause = log != null ? log.responseCause() : null; + } else { + resCause = Exceptions.peel(headersCause); + } + completeLogIfBytesNotTransferred(resHeaders, resCause); + + ctx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + if (rctx.config().needsContentInRule() && resCause == null) { + final HttpResponse unsplitRes = splitRes.unsplit(); + final HttpResponseDuplicator resDuplicator = + unsplitRes.toDuplicator(ctx.eventLoop().withoutContext(), + ctx.maxResponseLength()); + try { + res = resDuplicator.duplicate(); + final TruncatingHttpResponse truncatingAttemptRes = + new TruncatingHttpResponse(resDuplicator.duplicate(), + rctx.config().maxContentLength()); + resDuplicator.close(); + proposeResult(truncatingAttemptRes, null); + } catch (Throwable cause) { + resDuplicator.abort(cause); + abort(cause); + } + } else { + final HttpResponse unsplitRes; + if (resCause != null) { + splitRes.body().abort(resCause); + unsplitRes = HttpResponse.ofFailure(resCause); + } else { + unsplitRes = splitRes.unsplit(); + } + + proposeResult(unsplitRes, resCause); + } + }); + return null; + }); + } + + private void proposeResult(@Nullable HttpResponse resToPropose, @Nullable Throwable resCause) { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + if (resCause != null) { + resCause = Exceptions.peel(resCause); + } + + state = State.PROPOSED; + result = new Result(ctx, resToPropose, resCause); + proposalFuture.complete(result); + } + + private void completeLogIfBytesNotTransferred(AggregatedHttpResponse aggRes) { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { + final RequestLogBuilder attemptLogBuilder = ctx.logBuilder(); + attemptLogBuilder.endRequest(); + attemptLogBuilder.responseHeaders(aggRes.headers()); + if (!aggRes.trailers().isEmpty()) { + attemptLogBuilder.responseTrailers(aggRes.trailers()); + } + attemptLogBuilder.endResponse(); + } + } + + private void completeLogIfBytesNotTransferred( + @Nullable ResponseHeaders headers, + @Nullable Throwable resCause) { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { + final RequestLogBuilder logBuilder = ctx.logBuilder(); + if (resCause != null) { + logBuilder.endRequest(resCause); + logBuilder.endResponse(resCause); + } else { + logBuilder.endRequest(); + if (headers != null) { + logBuilder.responseHeaders(headers); + } + res.whenComplete().handle((unused, cause) -> { + if (cause != null) { + logBuilder.endResponse(cause); + } else { + logBuilder.endResponse(); + } + return null; + }); + } + } + } + } } From 26a4e0d420d24dab403e0376a0ddbb1e1d6adb44 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 27 Jun 2025 09:14:15 +0200 Subject: [PATCH 04/42] [WIP] refactor: continue modelling: push commit and abort into RetryingContext - put RetryingContext and RetryAttempt into separate classes --- .../armeria/client/retry/RetryAttempt.java | 433 +++++++++++++++ .../armeria/client/retry/RetryingClient.java | 523 +----------------- .../armeria/client/retry/RetryingContext.java | 93 ++++ 3 files changed, 553 insertions(+), 496 deletions(-) create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java new file mode 100644 index 00000000000..0add07c1f51 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java @@ -0,0 +1,433 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.retry; + +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.newAttemptContext; +import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; +import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; + +import java.time.Duration; +import java.util.Date; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linecorp.armeria.client.Client; +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.common.AggregatedHttpResponse; +import com.linecorp.armeria.common.HttpHeaderNames; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.HttpResponseDuplicator; +import com.linecorp.armeria.common.RequestHeadersBuilder; +import com.linecorp.armeria.common.ResponseHeaders; +import com.linecorp.armeria.common.SplitHttpResponse; +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.logging.RequestLog; +import com.linecorp.armeria.common.logging.RequestLogAccess; +import com.linecorp.armeria.common.logging.RequestLogBuilder; +import com.linecorp.armeria.common.logging.RequestLogProperty; +import com.linecorp.armeria.common.stream.AbortedStreamException; +import com.linecorp.armeria.common.util.Exceptions; +import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; +import com.linecorp.armeria.internal.client.ClientRequestContextExtension; +import com.linecorp.armeria.internal.client.TruncatingHttpResponse; + +import io.netty.handler.codec.DateFormatter; + +class RetryAttempt { + private static final Logger logger = LoggerFactory.getLogger(RetryAttempt.class); + + enum State { + // State of the attempt after construction and before the call to execute() or abort(). + INITIALIZED, + // State of the attempt before the outer call to execute() and before the inner call to complete() + // or abort(). + EXECUTING, + // State of the attempt before the call to commit() and before the outer call to commit() or abort() + // It means that this attempt ended up with a response, with or without content depending on the + // RetryRule, and it is up to the caller to decide on whether to commit on this response (i.e., + // returning this response from this client) or to abort to continue retrying with + // another attempt. + COMPLETED, + // After the outer call to commit(). Terminal state. + COMMITTED, + ABORTED + // After the outer call to abort() or after an unexpected exception. Terminal state. + // After setting this state, all attempt-related resources are freed. + } + + private State state; + + private final CompletableFuture whenCompletedFuture; + + RetryingContext rctx; + + final int number; + + // Available only in Attempt.State.EXECUTING. + @Nullable + private ClientRequestContext ctx; + @Nullable + private HttpResponse res; + + // Available only after Attempt.State.COMPLETED. + @Nullable + private HttpResponse completedRes; + @Nullable + private Throwable completedResCause; + + Client delegate; + + RetryAttempt(RetryingContext rctx, Client delegate, int number) { + this.rctx = rctx; + this.delegate = delegate; + this.number = number; + whenCompletedFuture = new CompletableFuture<>(); + state = State.INITIALIZED; + } + + int number() { + return number; + } + + State state() { + return state; + } + + CompletableFuture execute() { + assert state == State.INITIALIZED; + final boolean isInitialAttempt = number <= 1; + + final HttpRequest req; + if (isInitialAttempt) { + req = rctx.reqDuplicator().duplicate(); + } else { + final RequestHeadersBuilder attemptHeadersBuilder = rctx.req().headers().toBuilder(); + attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, number - 1); + req = rctx.reqDuplicator().duplicate(attemptHeadersBuilder.build()); + } + + try { + ctx = newAttemptContext( + rctx.ctx(), req, rctx.ctx().rpcRequest(), isInitialAttempt); + } catch (Throwable t) { + abort(t); + return whenCompletedFuture; + } + + executeAttemptRequest(); + assert state == State.EXECUTING && res != null && ctx != null; + + if (!rctx.ctx().exchangeType().isResponseStreaming() || rctx.config().requiresResponseTrailers()) { + handleAggRes(); + } else { + handleStreamingRes(); + } + + return whenCompletedFuture; + } + + CompletionStage<@Nullable RetryDecision> shouldRetry() { + assert state == State.COMPLETED; + assert ctx != null; + + if (rctx.config().needsContentInRule()) { + final RetryRuleWithContent retryRuleWithContent = + rctx.config().retryRuleWithContent(); + assert retryRuleWithContent != null; + return shouldBeRetriedWith(retryRuleWithContent); + } else { + final RetryRule retryRule = rctx.config().retryRule(); + assert retryRule != null; + return shouldBeRetriedWith(retryRule); + } + } + + long retryAfterMillis() { + assert state == State.COMPLETED; + assert ctx != null; + + final RequestLogAccess attemptLog = ctx.log(); + final String retryAfterValue; + final RequestLog requestLog = attemptLog.getIfAvailable(RequestLogProperty.RESPONSE_HEADERS); + retryAfterValue = requestLog != null ? + requestLog.responseHeaders().get(HttpHeaderNames.RETRY_AFTER) : null; + + if (retryAfterValue != null) { + try { + return Duration.ofSeconds(Integer.parseInt(retryAfterValue)).toMillis(); + } catch (Exception ignored) { + // Not a second value. + } + + try { + @SuppressWarnings("UseOfObsoleteDateTimeApi") + final Date retryAfterDate = DateFormatter.parseHttpDate(retryAfterValue); + if (retryAfterDate != null) { + return retryAfterDate.getTime() - System.currentTimeMillis(); + } + } catch (Exception ignored) { + // `parseHttpDate()` can raise an exception rather than returning `null` + // when the given value has more than 64 characters. + } + + logger.debug("The retryAfter: {}, from the server is neither an HTTP date nor a second.", + retryAfterValue); + } + + return -1; + } + + HttpResponse commit() { + if (state == State.COMMITTED) { + assert res != null; + return res; + } + + assert res != null; + + assert state == State.COMPLETED; + state = State.COMMITTED; + + if (completedRes != null) { + completedRes.abort(); + } + + return res; + } + + void abort() { + abort(AbortedStreamException.get()); + } + + void abort(Throwable cause) { + if (state == State.ABORTED) { + return; + } + assert state == State.EXECUTING || state == State.COMPLETED; + assert ctx != null && res != null; + state = State.ABORTED; + + if (completedRes != null) { + completedRes.abort(); + } + + final RequestLogBuilder logBuilder = ctx.logBuilder(); + // Set response content with null to make sure that the log is complete. + logBuilder.responseContent(null, null); + logBuilder.responseContentPreview(null); + res.abort(cause); + + if (!whenCompletedFuture.isDone()) { + whenCompletedFuture.completeExceptionally(cause); + } + } + + private void executeAttemptRequest() { + assert state == State.INITIALIZED; + assert ctx != null; + final HttpRequest req = ctx.request(); + assert req != null; + final ClientRequestContextExtension ctxExtension = + ctx.as(ClientRequestContextExtension.class); + if ((number > 1) && ctxExtension != null && ctx.endpoint() == null) { + // clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(ctx); + // if the endpoint hasn't been selected, + // try to initialize the ctx with a new endpoint/event loop + res = initContextAndExecuteWithFallback( + delegate, ctxExtension, HttpResponse::of, + (context, cause) -> + HttpResponse.ofFailure(cause), req, false); + } else { + res = executeWithFallback(delegate, ctx, + (context, cause) -> + HttpResponse.ofFailure(cause), req, false); + } + + state = State.EXECUTING; + } + + private void handleAggRes() { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + res.aggregate().handle((aggRes, resCause) -> { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + if (resCause != null) { + ctx.logBuilder().endRequest(resCause); + ctx.logBuilder().endResponse(resCause); + complete(HttpResponse.ofFailure(resCause), resCause); + } else { + completeLogIfBytesNotTransferred(aggRes); + ctx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { + complete(aggRes.toHttpResponse(), null); + }); + } + return null; + }); + } + + private void handleStreamingRes() { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + final SplitHttpResponse splitRes = res.split(); + splitRes.headers().handle((resHeaders, headersCause) -> { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + final Throwable resCause; + if (headersCause == null) { + final RequestLog log = ctx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); + resCause = log != null ? log.responseCause() : null; + } else { + resCause = Exceptions.peel(headersCause); + } + completeLogIfBytesNotTransferred(resHeaders, resCause); + + ctx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + if (rctx.config().needsContentInRule() && resCause == null) { + final HttpResponse unsplitRes = splitRes.unsplit(); + final HttpResponseDuplicator resDuplicator = + unsplitRes.toDuplicator(ctx.eventLoop().withoutContext(), + ctx.maxResponseLength()); + try { + res = resDuplicator.duplicate(); + final TruncatingHttpResponse truncatingAttemptRes = + new TruncatingHttpResponse(resDuplicator.duplicate(), + rctx.config().maxContentLength()); + resDuplicator.close(); + complete(truncatingAttemptRes, null); + } catch (Throwable cause) { + resDuplicator.abort(cause); + abort(cause); + } + } else { + final HttpResponse unsplitRes; + if (resCause != null) { + splitRes.body().abort(resCause); + unsplitRes = HttpResponse.ofFailure(resCause); + } else { + unsplitRes = splitRes.unsplit(); + } + + complete(unsplitRes, resCause); + } + }); + return null; + }); + } + + private void completeLogIfBytesNotTransferred(AggregatedHttpResponse aggRes) { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { + final RequestLogBuilder attemptLogBuilder = ctx.logBuilder(); + attemptLogBuilder.endRequest(); + attemptLogBuilder.responseHeaders(aggRes.headers()); + if (!aggRes.trailers().isEmpty()) { + attemptLogBuilder.responseTrailers(aggRes.trailers()); + } + attemptLogBuilder.endResponse(); + } + } + + private void completeLogIfBytesNotTransferred( + @Nullable ResponseHeaders headers, + @Nullable Throwable resCause) { + assert state == State.EXECUTING; + assert ctx != null && res != null; + + if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { + final RequestLogBuilder logBuilder = ctx.logBuilder(); + if (resCause != null) { + logBuilder.endRequest(resCause); + logBuilder.endResponse(resCause); + } else { + logBuilder.endRequest(); + if (headers != null) { + logBuilder.responseHeaders(headers); + } + res.whenComplete().handle((unused, cause) -> { + if (cause != null) { + logBuilder.endResponse(cause); + } else { + logBuilder.endResponse(); + } + return null; + }); + } + } + } + + private void complete(@Nullable HttpResponse resToCompleteWith, + @Nullable Throwable causeToCompleteWith) { + assert state == State.EXECUTING; + assert ctx != null && res != null; + state = State.COMPLETED; + + if (causeToCompleteWith != null) { + causeToCompleteWith = Exceptions.peel(causeToCompleteWith); + } + + completedRes = resToCompleteWith; + completedResCause = causeToCompleteWith; + whenCompletedFuture.complete(null); + } + + private CompletionStage<@Nullable RetryDecision> shouldBeRetriedWith(RetryRule retryRule) { + assert state == State.COMPLETED; + assert ctx != null; + + return retryRule.shouldRetry(ctx, completedResCause) + .handle((decision, cause) -> { + if (cause != null) { + logger.warn("Unexpected exception is raised from {}.", + retryRule, cause); + return null; + } + return decision; + }); + } + + private CompletionStage<@Nullable RetryDecision> shouldBeRetriedWith( + RetryRuleWithContent retryRuleWithContent) { + assert state == State.COMPLETED; + assert ctx != null; + + return retryRuleWithContent.shouldRetry(ctx, completedRes, completedResCause) + .handle((decision, cause) -> { + if (cause != null) { + logger.warn("Unexpected exception is raised from {}.", + retryRuleWithContent, cause); + return null; + } + return decision; + }); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 5cb6905116e..a6f5802befe 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -17,11 +17,7 @@ package com.linecorp.armeria.client.retry; import static com.google.common.base.Preconditions.checkArgument; -import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; -import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; -import java.time.Duration; -import java.util.Date; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.function.Function; @@ -29,34 +25,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.HttpClient; import com.linecorp.armeria.client.ResponseTimeoutException; -import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.AggregationOptions; -import com.linecorp.armeria.common.HttpHeaderNames; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpRequestDuplicator; import com.linecorp.armeria.common.HttpResponse; -import com.linecorp.armeria.common.HttpResponseDuplicator; import com.linecorp.armeria.common.Request; -import com.linecorp.armeria.common.RequestHeadersBuilder; -import com.linecorp.armeria.common.ResponseHeaders; -import com.linecorp.armeria.common.SplitHttpResponse; import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.logging.RequestLog; -import com.linecorp.armeria.common.logging.RequestLogAccess; -import com.linecorp.armeria.common.logging.RequestLogBuilder; -import com.linecorp.armeria.common.logging.RequestLogProperty; import com.linecorp.armeria.common.stream.AbortedStreamException; -import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; -import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; -import com.linecorp.armeria.internal.client.ClientRequestContextExtension; -import com.linecorp.armeria.internal.client.TruncatingHttpResponse; - -import io.netty.handler.codec.DateFormatter; /** * An {@link HttpClient} decorator that handles failures of an invocation and retries HTTP requests. @@ -250,7 +229,7 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro req.toDuplicator(ctx.eventLoop().withoutContext(), 0); final RetryingContext rctx = new RetryingContext( ctx, mappedRetryConfig(ctx), req, reqDuplicator, res, resFuture); - final Attempt attempt = new Attempt(rctx, unwrap(), 1); + final RetryAttempt attempt = new RetryAttempt(rctx, unwrap(), 1); doExecuteAttempt(rctx, attempt); } else { req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) @@ -263,7 +242,7 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro final HttpRequestDuplicator reqDuplicator = new AggregatedHttpRequestDuplicator(agg); final RetryingContext rctx = new RetryingContext( ctx, mappedRetryConfig(ctx), req, reqDuplicator, res, resFuture); - final Attempt attempt = new Attempt(rctx, unwrap(), 1); + final RetryAttempt attempt = new RetryAttempt(rctx, unwrap(), 1); doExecuteAttempt(rctx, attempt); } return null; @@ -272,14 +251,14 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro return res; } - private void doExecuteAttempt(RetryingContext rctx, Attempt attempt) { - assert attempt.state() == Attempt.State.INITIALIZED; + private void doExecuteAttempt(RetryingContext rctx, RetryAttempt attempt) { + assert attempt.state() == RetryAttempt.State.INITIALIZED; final boolean isInitialAttempt = attempt.number() <= 1; // The request or response has been aborted by the client before it receives a response, // so stop retrying. if (rctx.req().whenComplete().isCompletedExceptionally()) { rctx.req().whenComplete().handle((unused, cause) -> { - abort(rctx, cause, isInitialAttempt); + rctx.abort(cause, isInitialAttempt); return null; }); return; @@ -292,52 +271,54 @@ private void doExecuteAttempt(RetryingContext rctx, Attempt attempt) { } else { abortCause = AbortedStreamException.get(); } - abort(rctx, abortCause, isInitialAttempt); + rctx.abort(abortCause, isInitialAttempt); return null; }); return; } if (!setResponseTimeout(rctx.ctx())) { - abort(rctx, ResponseTimeoutException.get(), isInitialAttempt); + rctx.abort(ResponseTimeoutException.get(), isInitialAttempt); return; } - attempt.execute().handle((attemptProposal, unexpectedAttemptCause) -> { + attempt.execute().handle((unused, unexpectedAttemptCause) -> { if (unexpectedAttemptCause != null) { - assert attempt.state() == Attempt.State.ABORTED; - abort(rctx, unexpectedAttemptCause); + assert attempt.state() == RetryAttempt.State.ABORTED; + rctx.abort(unexpectedAttemptCause); return null; } - assert attempt.state() == Attempt.State.PROPOSED; + assert attempt.state() == RetryAttempt.State.COMPLETED; decideOnAttempt(rctx, attempt).handle((nextDelayMillis, unexpectedDecisionCause) -> { if (unexpectedDecisionCause != null) { attempt.abort(unexpectedDecisionCause); - assert attempt.state() == Attempt.State.ABORTED; - abort(rctx, unexpectedDecisionCause); + assert attempt.state() == RetryAttempt.State.ABORTED; + rctx.abort(unexpectedDecisionCause); return null; } - assert attempt.state() == Attempt.State.PROPOSED; + assert attempt.state() == RetryAttempt.State.COMPLETED; if (nextDelayMillis >= 0) { attempt.abort(); - assert attempt.state() == Attempt.State.ABORTED; + assert attempt.state() == RetryAttempt.State.ABORTED; scheduleNextRetry( - rctx.ctx(), schedulingCause -> abort(rctx, schedulingCause), + rctx.ctx(), schedulingCause -> rctx.abort(schedulingCause), () -> { - final Attempt nextAttempt = new Attempt(rctx, unwrap(), attempt.number() + 1); + final RetryAttempt nextAttempt = new RetryAttempt(rctx, + unwrap(), + attempt.number() + 1); doExecuteAttempt(rctx, nextAttempt); }, nextDelayMillis); return null; } - commit(rctx, attempt); - assert attempt.state() == Attempt.State.COMMITTED; + rctx.commit(attempt); + assert attempt.state() == RetryAttempt.State.COMMITTED; return null; }); @@ -345,39 +326,16 @@ private void doExecuteAttempt(RetryingContext rctx, Attempt attempt) { }); } - private CompletionStage decideOnAttempt(RetryingContext rctx, Attempt attempt) { - assert attempt.state() == Attempt.State.PROPOSED; - final Attempt.Result result = attempt.result(); - final CompletionStage<@Nullable RetryDecision> retryDecisionFuture; - - if (rctx.config().needsContentInRule()) { - final RetryRuleWithContent retryRuleWithContent = - rctx.config().retryRuleWithContent(); - assert retryRuleWithContent != null; - retryDecisionFuture = retryRuleWithContent - .shouldRetry(result.ctx(), result.res(), result.cause()) - .exceptionally(cause -> { - logger.warn("Unexpected exception is raised from {}.", retryRuleWithContent, cause); - return null; - }); - } else { - final RetryRule retryRule = rctx.config().retryRule(); - assert retryRule != null; - retryDecisionFuture = retryRule.shouldRetry(result.ctx(), result.cause()) - .exceptionally(cause -> { - logger.warn("Unexpected exception is raised from {}.", retryRule, - cause); - return null; - }); - } + private CompletionStage decideOnAttempt(RetryingContext rctx, RetryAttempt attempt) { + assert attempt.state() == RetryAttempt.State.COMPLETED; - return retryDecisionFuture.handle((decision, cause) -> { + return attempt.shouldRetry().handle((decision, cause) -> { assert cause == null; - assert attempt.state() == Attempt.State.PROPOSED; + assert attempt.state() == RetryAttempt.State.COMPLETED; final Backoff backoff = decision != null ? decision.backoff() : null; if (backoff != null) { - final long millisAfter = useRetryAfter ? getRetryAfterMillis(result.ctx()) : -1; - final long nextDelay = getNextDelay(result.ctx(), backoff, millisAfter); + final long millisAfter = useRetryAfter ? attempt.retryAfterMillis() : -1; + final long nextDelay = getNextDelay(rctx.ctx(), backoff, millisAfter); if (nextDelay >= 0) { return nextDelay; } @@ -386,431 +344,4 @@ private CompletionStage decideOnAttempt(RetryingContext rctx, Attempt atte return -1L; }); } - - private static void commit(RetryingContext rctx, Attempt attempt) { - assert attempt.state() == Attempt.State.PROPOSED; - attempt.commit(); - assert attempt.state() == Attempt.State.COMMITTED; - final Attempt.Result attemptResultToCommit = attempt.result(); - onRetryingComplete(rctx.ctx()); - rctx.resFuture().complete(attemptResultToCommit.res()); - rctx.reqDuplicator().close(); - } - - private static void abort(RetryingContext rctx, Throwable cause) { - abort(rctx, cause, false); - } - - private static void abort(RetryingContext rctx, Throwable cause, boolean endRequestLog) { - rctx.resFuture().completeExceptionally(cause); - rctx.reqDuplicator().abort(cause); - if (endRequestLog) { - rctx.ctx().logBuilder().endRequest(cause); - } - rctx.ctx().logBuilder().endResponse(cause); - } - - private static long getRetryAfterMillis(ClientRequestContext attemptCtx) { - final RequestLogAccess attemptLog = attemptCtx.log(); - final String retryAfterValue; - final RequestLog requestLog = attemptLog.getIfAvailable(RequestLogProperty.RESPONSE_HEADERS); - retryAfterValue = requestLog != null ? - requestLog.responseHeaders().get(HttpHeaderNames.RETRY_AFTER) : null; - - if (retryAfterValue != null) { - try { - return Duration.ofSeconds(Integer.parseInt(retryAfterValue)).toMillis(); - } catch (Exception ignored) { - // Not a second value. - } - - try { - @SuppressWarnings("UseOfObsoleteDateTimeApi") - final Date retryAfterDate = DateFormatter.parseHttpDate(retryAfterValue); - if (retryAfterDate != null) { - return retryAfterDate.getTime() - System.currentTimeMillis(); - } - } catch (Exception ignored) { - // `parseHttpDate()` can raise an exception rather than returning `null` - // when the given value has more than 64 characters. - } - - logger.debug("The retryAfter: {}, from the server is neither an HTTP date nor a second.", - retryAfterValue); - } - - return -1; - } - - private static class RetryingContext { - private final ClientRequestContext ctx; - private final RetryConfig retryConfig; - private final HttpRequest req; - private final HttpRequestDuplicator reqDuplicator; - private final HttpResponse res; - private final CompletableFuture resFuture; - - RetryingContext(ClientRequestContext ctx, - RetryConfig retryConfig, - HttpRequest req, - HttpRequestDuplicator reqDuplicator, - HttpResponse res, - CompletableFuture resFuture) { - - this.ctx = ctx; - this.retryConfig = retryConfig; - this.req = req; - this.reqDuplicator = reqDuplicator; - this.res = res; - this.resFuture = resFuture; - } - - RetryConfig config() { - return retryConfig; - } - - ClientRequestContext ctx() { - return ctx; - } - - HttpRequestDuplicator reqDuplicator() { - return reqDuplicator; - } - - HttpRequest req() { - return req; - } - - HttpResponse res() { - return res; - } - - CompletableFuture resFuture() { - return resFuture; - } - } - - private static class Attempt { - private enum State { - // State of the attempt after construction and before the call to execute(). - INITIALIZED, - // State of the attempt before the outer call to execute() and before the inner call to propose(). - EXECUTING, - // State of the attempt before the call to commit() and before the outer call to commit() or abort() - // It means that this attempt ended up with a response, with or without content depending on the - // RetryRule, and it is up to the caller to decide on whether to commit on this response (i.e., - // returning this response from this client) or to abort it we are continue retrying). - PROPOSED, - // After the outer call to commit(). Terminal state. - COMMITTED, - ABORTED - // After the outer call to abort() or after an unexpected exception. Terminal state. - // After setting this state, all attempt-related resources are freed. - } - - static class Result { - private final ClientRequestContext ctx; - @Nullable - private final HttpResponse res; - @Nullable - private final Throwable cause; - - Result(ClientRequestContext ctx, @Nullable HttpResponse res, @Nullable Throwable cause) { - this.ctx = ctx; - this.res = res; - this.cause = cause; - } - - ClientRequestContext ctx() { - return ctx; - } - - @Nullable - HttpResponse res() { - return res; - } - - @Nullable - Throwable cause() { - return cause; - } - - void close() { - if (res != null) { - // abort() is idempotent - res.abort(); - } - } - } - - private State state; - - @Nullable - private Result result; - private final CompletableFuture proposalFuture; - - RetryingContext rctx; - - final int number; - - // Available only in Attempt.State.EXECUTING. - @Nullable - private ClientRequestContext ctx; - @Nullable - private HttpResponse res; - - Client delegate; - - Attempt(RetryingContext rctx, Client delegate, int number) { - this.rctx = rctx; - this.delegate = delegate; - this.number = number; - proposalFuture = new CompletableFuture<>(); - state = State.INITIALIZED; - } - - int number() { - return number; - } - - State state() { - return state; - } - - CompletableFuture execute() { - assert state == State.INITIALIZED; - final boolean isInitialAttempt = number <= 1; - - final HttpRequest attemptReq; - if (isInitialAttempt) { - attemptReq = rctx.reqDuplicator().duplicate(); - } else { - final RequestHeadersBuilder attemptHeadersBuilder = rctx.req().headers().toBuilder(); - attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, number - 1); - attemptReq = rctx.reqDuplicator().duplicate(attemptHeadersBuilder.build()); - } - - try { - ctx = newAttemptContext( - rctx.ctx(), attemptReq, rctx.ctx().rpcRequest(), isInitialAttempt); - } catch (Throwable t) { - abort(t); - return proposalFuture; - } - - executeAttemptRequest(); - assert state == State.EXECUTING && res != null && ctx != null; - - if (!rctx.ctx().exchangeType().isResponseStreaming() || rctx.config().requiresResponseTrailers()) { - handleAggRes(); - } else { - handleStreamingRes(); - } - - return proposalFuture; - } - - Result result() { - assert state == State.PROPOSED || state == State.COMMITTED; - assert result != null; - return result; - } - - void commit() { - if (state == State.COMMITTED) { - return; - } - assert state == State.PROPOSED; - state = State.COMMITTED; - - if (result != null) { - result.close(); - } - } - - void abort() { - abort(AbortedStreamException.get()); - } - - void abort(Throwable cause) { - if (state == State.ABORTED) { - return; - } - assert state == State.EXECUTING || state == State.PROPOSED; - assert ctx != null && res != null; - state = State.ABORTED; - - if (result != null) { - // Frees intermediate resources. - result.close(); - } - - final RequestLogBuilder logBuilder = ctx.logBuilder(); - // Set response content with null to make sure that the log is complete. - logBuilder.responseContent(null, null); - logBuilder.responseContentPreview(null); - res.abort(cause); - - if (!proposalFuture.isDone()) { - proposalFuture.completeExceptionally(cause); - } - } - - private void executeAttemptRequest() { - assert state == State.INITIALIZED; - assert ctx != null; - final HttpRequest req = ctx.request(); - assert req != null; - final ClientRequestContextExtension ctxExtension = - ctx.as(ClientRequestContextExtension.class); - if ((number > 1) && ctxExtension != null && ctx.endpoint() == null) { - // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(ctx); - // if the endpoint hasn't been selected, - // try to initialize the ctx with a new endpoint/event loop - res = initContextAndExecuteWithFallback( - delegate, ctxExtension, HttpResponse::of, - (context, cause) -> - HttpResponse.ofFailure(cause), req, false); - } else { - res = executeWithFallback(delegate, ctx, - (context, cause) -> - HttpResponse.ofFailure(cause), req, false); - } - - state = State.EXECUTING; - } - - private void handleAggRes() { - assert state == State.EXECUTING; - assert ctx != null && res != null; - - res.aggregate().handle((aggRes, resCause) -> { - assert state == State.EXECUTING; - assert ctx != null && res != null; - - if (resCause != null) { - ctx.logBuilder().endRequest(resCause); - ctx.logBuilder().endResponse(resCause); - proposeResult(HttpResponse.ofFailure(resCause), resCause); - } else { - completeLogIfBytesNotTransferred(aggRes); - ctx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { - proposeResult(aggRes.toHttpResponse(), null); - }); - } - return null; - }); - } - - private void handleStreamingRes() { - assert state == State.EXECUTING; - assert ctx != null && res != null; - - final SplitHttpResponse splitRes = res.split(); - splitRes.headers().handle((resHeaders, headersCause) -> { - assert state == State.EXECUTING; - assert ctx != null && res != null; - - final Throwable resCause; - if (headersCause == null) { - final RequestLog log = ctx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); - resCause = log != null ? log.responseCause() : null; - } else { - resCause = Exceptions.peel(headersCause); - } - completeLogIfBytesNotTransferred(resHeaders, resCause); - - ctx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { - assert state == State.EXECUTING; - assert ctx != null && res != null; - - if (rctx.config().needsContentInRule() && resCause == null) { - final HttpResponse unsplitRes = splitRes.unsplit(); - final HttpResponseDuplicator resDuplicator = - unsplitRes.toDuplicator(ctx.eventLoop().withoutContext(), - ctx.maxResponseLength()); - try { - res = resDuplicator.duplicate(); - final TruncatingHttpResponse truncatingAttemptRes = - new TruncatingHttpResponse(resDuplicator.duplicate(), - rctx.config().maxContentLength()); - resDuplicator.close(); - proposeResult(truncatingAttemptRes, null); - } catch (Throwable cause) { - resDuplicator.abort(cause); - abort(cause); - } - } else { - final HttpResponse unsplitRes; - if (resCause != null) { - splitRes.body().abort(resCause); - unsplitRes = HttpResponse.ofFailure(resCause); - } else { - unsplitRes = splitRes.unsplit(); - } - - proposeResult(unsplitRes, resCause); - } - }); - return null; - }); - } - - private void proposeResult(@Nullable HttpResponse resToPropose, @Nullable Throwable resCause) { - assert state == State.EXECUTING; - assert ctx != null && res != null; - - if (resCause != null) { - resCause = Exceptions.peel(resCause); - } - - state = State.PROPOSED; - result = new Result(ctx, resToPropose, resCause); - proposalFuture.complete(result); - } - - private void completeLogIfBytesNotTransferred(AggregatedHttpResponse aggRes) { - assert state == State.EXECUTING; - assert ctx != null && res != null; - - if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { - final RequestLogBuilder attemptLogBuilder = ctx.logBuilder(); - attemptLogBuilder.endRequest(); - attemptLogBuilder.responseHeaders(aggRes.headers()); - if (!aggRes.trailers().isEmpty()) { - attemptLogBuilder.responseTrailers(aggRes.trailers()); - } - attemptLogBuilder.endResponse(); - } - } - - private void completeLogIfBytesNotTransferred( - @Nullable ResponseHeaders headers, - @Nullable Throwable resCause) { - assert state == State.EXECUTING; - assert ctx != null && res != null; - - if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { - final RequestLogBuilder logBuilder = ctx.logBuilder(); - if (resCause != null) { - logBuilder.endRequest(resCause); - logBuilder.endResponse(resCause); - } else { - logBuilder.endRequest(); - if (headers != null) { - logBuilder.responseHeaders(headers); - } - res.whenComplete().handle((unused, cause) -> { - if (cause != null) { - logBuilder.endResponse(cause); - } else { - logBuilder.endResponse(); - } - return null; - }); - } - } - } - } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java new file mode 100644 index 00000000000..ff81fb6340b --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java @@ -0,0 +1,93 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.retry; + +import java.util.concurrent.CompletableFuture; + +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.common.HttpRequestDuplicator; +import com.linecorp.armeria.common.HttpResponse; + +class RetryingContext { + private final ClientRequestContext ctx; + private final RetryConfig retryConfig; + private final HttpRequest req; + private final HttpRequestDuplicator reqDuplicator; + private final HttpResponse res; + private final CompletableFuture resFuture; + + RetryingContext(ClientRequestContext ctx, + RetryConfig retryConfig, + HttpRequest req, + HttpRequestDuplicator reqDuplicator, + HttpResponse res, + CompletableFuture resFuture) { + + this.ctx = ctx; + this.retryConfig = retryConfig; + this.req = req; + this.reqDuplicator = reqDuplicator; + this.res = res; + this.resFuture = resFuture; + } + + RetryConfig config() { + return retryConfig; + } + + ClientRequestContext ctx() { + return ctx; + } + + HttpRequestDuplicator reqDuplicator() { + return reqDuplicator; + } + + HttpRequest req() { + return req; + } + + HttpResponse res() { + return res; + } + + CompletableFuture resFuture() { + return resFuture; + } + + void commit(RetryAttempt attempt) { + assert attempt.state() == RetryAttempt.State.COMPLETED; + final HttpResponse attemptRes = attempt.commit(); + ctx.logBuilder().endResponseWithLastChild(); + resFuture.complete(attemptRes); + reqDuplicator.close(); + } + + void abort(Throwable cause) { + abort(cause, false); + } + + void abort(Throwable cause, boolean endRequestLog) { + resFuture.completeExceptionally(cause); + reqDuplicator.abort(cause); + if (endRequestLog) { + ctx.logBuilder().endRequest(cause); + } + ctx.logBuilder().endResponse(cause); + } +} From b31c646b06d9ce502c084ca498c625f1f17f3854 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 27 Jun 2025 10:37:00 +0200 Subject: [PATCH 05/42] [WIP] refactor: further extraction --- .../client/retry/AbstractRetryingClient.java | 3 +- .../armeria/client/retry/RetryAttempt.java | 44 ++++----- .../armeria/client/retry/RetryingClient.java | 40 ++------ .../armeria/client/retry/RetryingContext.java | 96 ++++++++++++++++--- 4 files changed, 107 insertions(+), 76 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 00c516be514..8d0d42e11b6 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -175,8 +175,7 @@ protected static void scheduleNextRetry(ClientRequestContext ctx, * * @return {@code true} if the response timeout is set, {@code false} if it can't be set due to the timeout */ - @SuppressWarnings("MethodMayBeStatic") // Intentionally left non-static for better user experience. - protected final boolean setResponseTimeout(ClientRequestContext ctx) { + protected static boolean setResponseTimeout(ClientRequestContext ctx) { requireNonNull(ctx, "ctx"); final long responseTimeoutMillis = state(ctx).responseTimeoutMillis(); if (responseTimeoutMillis < 0) { diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java index 0add07c1f51..71ba0551414 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java @@ -16,8 +16,6 @@ package com.linecorp.armeria.client.retry; -import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; -import static com.linecorp.armeria.client.retry.AbstractRetryingClient.newAttemptContext; import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; @@ -36,7 +34,6 @@ import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.HttpResponseDuplicator; -import com.linecorp.armeria.common.RequestHeadersBuilder; import com.linecorp.armeria.common.ResponseHeaders; import com.linecorp.armeria.common.SplitHttpResponse; import com.linecorp.armeria.common.annotation.Nullable; @@ -114,27 +111,16 @@ State state() { CompletableFuture execute() { assert state == State.INITIALIZED; - final boolean isInitialAttempt = number <= 1; - - final HttpRequest req; - if (isInitialAttempt) { - req = rctx.reqDuplicator().duplicate(); - } else { - final RequestHeadersBuilder attemptHeadersBuilder = rctx.req().headers().toBuilder(); - attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, number - 1); - req = rctx.reqDuplicator().duplicate(attemptHeadersBuilder.build()); - } + state = State.EXECUTING; try { - ctx = newAttemptContext( - rctx.ctx(), req, rctx.ctx().rpcRequest(), isInitialAttempt); + ctx = rctx.newAttemptContext(number); } catch (Throwable t) { abort(t); return whenCompletedFuture; } - executeAttemptRequest(); - assert state == State.EXECUTING && res != null && ctx != null; + res = executeAttemptRequest(); if (!rctx.ctx().exchangeType().isResponseStreaming() || rctx.config().requiresResponseTrailers()) { handleAggRes(); @@ -202,9 +188,8 @@ HttpResponse commit() { return res; } - assert res != null; - assert state == State.COMPLETED; + assert res != null; state = State.COMMITTED; if (completedRes != null) { @@ -219,9 +204,15 @@ void abort() { } void abort(Throwable cause) { - if (state == State.ABORTED) { + if (state == State.ABORTED || state == State.COMMITTED) { + return; + } + + if (state == State.INITIALIZED) { + assert ctx == null && res == null; return; } + assert state == State.EXECUTING || state == State.COMPLETED; assert ctx != null && res != null; state = State.ABORTED; @@ -241,11 +232,13 @@ void abort(Throwable cause) { } } - private void executeAttemptRequest() { + private HttpResponse executeAttemptRequest() { assert state == State.INITIALIZED; assert ctx != null; + final HttpRequest req = ctx.request(); assert req != null; + final ClientRequestContextExtension ctxExtension = ctx.as(ClientRequestContextExtension.class); if ((number > 1) && ctxExtension != null && ctx.endpoint() == null) { @@ -253,17 +246,15 @@ private void executeAttemptRequest() { ClientPendingThrowableUtil.removePendingThrowable(ctx); // if the endpoint hasn't been selected, // try to initialize the ctx with a new endpoint/event loop - res = initContextAndExecuteWithFallback( + return initContextAndExecuteWithFallback( delegate, ctxExtension, HttpResponse::of, (context, cause) -> HttpResponse.ofFailure(cause), req, false); } else { - res = executeWithFallback(delegate, ctx, + return executeWithFallback(delegate, ctx, (context, cause) -> HttpResponse.ofFailure(cause), req, false); } - - state = State.EXECUTING; } private void handleAggRes() { @@ -385,10 +376,9 @@ private void completeLogIfBytesNotTransferred( } } - private void complete(@Nullable HttpResponse resToCompleteWith, + private void complete(HttpResponse resToCompleteWith, @Nullable Throwable causeToCompleteWith) { assert state == State.EXECUTING; - assert ctx != null && res != null; state = State.COMPLETED; if (causeToCompleteWith != null) { diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index a6f5802befe..0d150f02f97 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -27,14 +27,12 @@ import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.HttpClient; -import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.common.AggregationOptions; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpRequestDuplicator; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.stream.AbortedStreamException; import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; /** @@ -252,33 +250,10 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro } private void doExecuteAttempt(RetryingContext rctx, RetryAttempt attempt) { - assert attempt.state() == RetryAttempt.State.INITIALIZED; - final boolean isInitialAttempt = attempt.number() <= 1; - // The request or response has been aborted by the client before it receives a response, - // so stop retrying. - if (rctx.req().whenComplete().isCompletedExceptionally()) { - rctx.req().whenComplete().handle((unused, cause) -> { - rctx.abort(cause, isInitialAttempt); - return null; - }); - return; - } - if (rctx.res().isComplete()) { - rctx.res().whenComplete().handle((result, cause) -> { - final Throwable abortCause; - if (cause != null) { - abortCause = cause; - } else { - abortCause = AbortedStreamException.get(); - } - rctx.abort(abortCause, isInitialAttempt); - return null; - }); - return; - } - - if (!setResponseTimeout(rctx.ctx())) { - rctx.abort(ResponseTimeoutException.get(), isInitialAttempt); + if (rctx.isCompleted()) { + // The request or response has been aborted by the client before it receives a response, + // so stop retrying. + attempt.abort(); return; } @@ -306,11 +281,10 @@ private void doExecuteAttempt(RetryingContext rctx, RetryAttempt attempt) { assert attempt.state() == RetryAttempt.State.ABORTED; scheduleNextRetry( - rctx.ctx(), schedulingCause -> rctx.abort(schedulingCause), + rctx.ctx(), rctx::abort, () -> { - final RetryAttempt nextAttempt = new RetryAttempt(rctx, - unwrap(), - attempt.number() + 1); + final RetryAttempt nextAttempt = new RetryAttempt( + rctx, unwrap(), getTotalAttempts(rctx.ctx())); doExecuteAttempt(rctx, nextAttempt); }, nextDelayMillis); diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java index ff81fb6340b..ebdf7d789cd 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java @@ -16,14 +16,25 @@ package com.linecorp.armeria.client.retry; +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; + import java.util.concurrent.CompletableFuture; import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpRequestDuplicator; import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.RequestHeadersBuilder; +import com.linecorp.armeria.common.stream.AbortedStreamException; class RetryingContext { + private enum State { + RETRYING, + COMPLETING, + COMPLETED + } + private final ClientRequestContext ctx; private final RetryConfig retryConfig; private final HttpRequest req; @@ -31,6 +42,8 @@ class RetryingContext { private final HttpResponse res; private final CompletableFuture resFuture; + private State state; + RetryingContext(ClientRequestContext ctx, RetryConfig retryConfig, HttpRequest req, @@ -44,6 +57,7 @@ class RetryingContext { this.reqDuplicator = reqDuplicator; this.res = res; this.resFuture = resFuture; + state = State.RETRYING; } RetryConfig config() { @@ -54,24 +68,71 @@ ClientRequestContext ctx() { return ctx; } - HttpRequestDuplicator reqDuplicator() { - return reqDuplicator; - } + boolean isCompleted() { + if (state == State.COMPLETING || state == State.COMPLETED) { + return true; + } - HttpRequest req() { - return req; - } + assert state == State.RETRYING; + + // The request or response has been aborted by the client before it receives a response, + // so stop retrying. + if (req.whenComplete().isCompletedExceptionally()) { + state = State.COMPLETING; + req.whenComplete().handle((unused, cause) -> { + abort(cause); + return null; + }); + return true; + } - HttpResponse res() { - return res; + if (res.isComplete()) { + state = State.COMPLETING; + res.whenComplete().handle((result, cause) -> { + final Throwable abortCause; + if (cause != null) { + abortCause = cause; + } else { + abortCause = AbortedStreamException.get(); + } + abort(abortCause); + return null; + }); + return true; + } + + return false; } - CompletableFuture resFuture() { - return resFuture; + public ClientRequestContext newAttemptContext(int attemptNumber) { + final boolean isInitialAttempt = attemptNumber <= 1; + + if (!AbstractRetryingClient.setResponseTimeout(ctx)) { + throw ResponseTimeoutException.get(); + } + + final HttpRequest attemptReq; + if (isInitialAttempt) { + attemptReq = reqDuplicator.duplicate(); + } else { + final RequestHeadersBuilder attemptHeadersBuilder = req.headers().toBuilder(); + attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, attemptNumber - 1); + attemptReq = reqDuplicator.duplicate(attemptHeadersBuilder.build()); + } + + return AbstractRetryingClient.newAttemptContext( + ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); } void commit(RetryAttempt attempt) { + if (state == State.COMPLETED) { + // Already completed. + return; + } assert attempt.state() == RetryAttempt.State.COMPLETED; + assert state == State.RETRYING; + state = State.COMPLETING; + final HttpResponse attemptRes = attempt.commit(); ctx.logBuilder().endResponseWithLastChild(); resFuture.complete(attemptRes); @@ -79,15 +140,22 @@ void commit(RetryAttempt attempt) { } void abort(Throwable cause) { - abort(cause, false); - } + if (state == State.COMPLETED) { + // Already completed. + return; + } + + assert state == State.RETRYING || state == State.COMPLETING; + state = State.COMPLETED; - void abort(Throwable cause, boolean endRequestLog) { resFuture.completeExceptionally(cause); reqDuplicator.abort(cause); - if (endRequestLog) { + + // todo(szymon): verify that this safe to do so we can avoid isInitialAttempt check + if (!ctx.log().isRequestComplete()) { ctx.logBuilder().endRequest(cause); } + ctx.logBuilder().endResponse(cause); } } From eeb179f7ce7ebd6a5282b6e803e25bda382ef6cf Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 27 Jun 2025 14:30:06 +0200 Subject: [PATCH 06/42] fix: RetryClientTest - also do some cleanup in RetryAttempt --- .../armeria/client/retry/RetryAttempt.java | 111 +++++++++++------- .../armeria/client/retry/RetryingClient.java | 29 +++-- .../armeria/client/retry/RetryingContext.java | 5 +- 3 files changed, 86 insertions(+), 59 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java index 71ba0551414..01d0bf1f599 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java @@ -53,22 +53,22 @@ class RetryAttempt { private static final Logger logger = LoggerFactory.getLogger(RetryAttempt.class); enum State { - // State of the attempt after construction and before the call to execute() or abort(). + // Initial state after constructing an `Attempt`. + // Caller can only invoke `execute` or `abort`. INITIALIZED, - // State of the attempt before the outer call to execute() and before the inner call to complete() - // or abort(). + // State after calling `execute` + // `ctx` and `res` are now available + // The attempt response is underway but did not complete yet. EXECUTING, - // State of the attempt before the call to commit() and before the outer call to commit() or abort() - // It means that this attempt ended up with a response, with or without content depending on the - // RetryRule, and it is up to the caller to decide on whether to commit on this response (i.e., - // returning this response from this client) or to abort to continue retrying with - // another attempt. + // State after the (maybe exceptional) result of the attempt response was processed. + // `res` is available. `truncatedRes` and `resCause` are also available if applicable. + // Caller can only invoke `shouldRetry`, `commit` or `abort`. COMPLETED, - // After the outer call to commit(). Terminal state. + // State after a call to `commit`. Terminal state, caller cannot make further calls. COMMITTED, + // State after a call to `commit`. Terminal state, caller cannot make further calls. + // `res` is aborted. ABORTED - // After the outer call to abort() or after an unexpected exception. Terminal state. - // After setting this state, all attempt-related resources are freed. } private State state; @@ -87,9 +87,9 @@ enum State { // Available only after Attempt.State.COMPLETED. @Nullable - private HttpResponse completedRes; + private HttpResponse truncatedRes; @Nullable - private Throwable completedResCause; + private Throwable resCause; Client delegate; @@ -98,11 +98,11 @@ enum State { this.delegate = delegate; this.number = number; whenCompletedFuture = new CompletableFuture<>(); - state = State.INITIALIZED; - } - int number() { - return number; + truncatedRes = null; + resCause = null; + + state = State.INITIALIZED; } State state() { @@ -192,8 +192,8 @@ HttpResponse commit() { assert res != null; state = State.COMMITTED; - if (completedRes != null) { - completedRes.abort(); + if (truncatedRes != null) { + truncatedRes.abort(); } return res; @@ -217,8 +217,8 @@ void abort(Throwable cause) { assert ctx != null && res != null; state = State.ABORTED; - if (completedRes != null) { - completedRes.abort(); + if (truncatedRes != null) { + truncatedRes.abort(); } final RequestLogBuilder logBuilder = ctx.logBuilder(); @@ -227,13 +227,11 @@ void abort(Throwable cause) { logBuilder.responseContentPreview(null); res.abort(cause); - if (!whenCompletedFuture.isDone()) { - whenCompletedFuture.completeExceptionally(cause); - } + whenCompletedFuture.completeExceptionally(cause); } private HttpResponse executeAttemptRequest() { - assert state == State.INITIALIZED; + assert state == State.EXECUTING; assert ctx != null; final HttpRequest req = ctx.request(); @@ -295,6 +293,7 @@ private void handleStreamingRes() { } else { resCause = Exceptions.peel(headersCause); } + completeLogIfBytesNotTransferred(resHeaders, resCause); ctx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { @@ -306,27 +305,21 @@ private void handleStreamingRes() { final HttpResponseDuplicator resDuplicator = unsplitRes.toDuplicator(ctx.eventLoop().withoutContext(), ctx.maxResponseLength()); - try { - res = resDuplicator.duplicate(); + // todo(szymon): We do not call duplicator.abort(cause); but res.abort on an exception. + // Is this okay? + final HttpResponse duplicatedRes = resDuplicator.duplicate(); final TruncatingHttpResponse truncatingAttemptRes = new TruncatingHttpResponse(resDuplicator.duplicate(), rctx.config().maxContentLength()); resDuplicator.close(); - complete(truncatingAttemptRes, null); - } catch (Throwable cause) { - resDuplicator.abort(cause); - abort(cause); - } + completeWithTruncated(duplicatedRes, truncatingAttemptRes); } else { - final HttpResponse unsplitRes; if (resCause != null) { splitRes.body().abort(resCause); - unsplitRes = HttpResponse.ofFailure(resCause); + complete(HttpResponse.ofFailure(resCause), resCause); } else { - unsplitRes = splitRes.unsplit(); + complete(splitRes.unsplit(), null); } - - complete(unsplitRes, resCause); } }); return null; @@ -376,17 +369,27 @@ private void completeLogIfBytesNotTransferred( } } - private void complete(HttpResponse resToCompleteWith, - @Nullable Throwable causeToCompleteWith) { + private void completeWithTruncated(HttpResponse res, TruncatingHttpResponse truncatedRes) { assert state == State.EXECUTING; state = State.COMPLETED; - if (causeToCompleteWith != null) { - causeToCompleteWith = Exceptions.peel(causeToCompleteWith); + this.res = res; + this.truncatedRes = truncatedRes; + resCause = null; + whenCompletedFuture.complete(null); + } + + private void complete(HttpResponse res, @Nullable Throwable resCause) { + assert state == State.EXECUTING; + state = State.COMPLETED; + + if (resCause != null) { + resCause = Exceptions.peel(resCause); } - completedRes = resToCompleteWith; - completedResCause = causeToCompleteWith; + this.res = res; + truncatedRes = null; + this.resCause = resCause; whenCompletedFuture.complete(null); } @@ -394,7 +397,7 @@ private void complete(HttpResponse resToCompleteWith, assert state == State.COMPLETED; assert ctx != null; - return retryRule.shouldRetry(ctx, completedResCause) + return retryRule.shouldRetry(ctx, resCause) .handle((decision, cause) -> { if (cause != null) { logger.warn("Unexpected exception is raised from {}.", @@ -410,7 +413,25 @@ private void complete(HttpResponse resToCompleteWith, assert state == State.COMPLETED; assert ctx != null; - return retryRuleWithContent.shouldRetry(ctx, completedRes, completedResCause) + @Nullable + final HttpResponse resForRule; + @Nullable + final Throwable causeForRule; + + if (resCause != null) { + resForRule = null; + causeForRule = resCause; + } else { + if (truncatedRes == null) { + resForRule = res; + } else { + resForRule = truncatedRes; + } + + causeForRule = null; + } + + return retryRuleWithContent.shouldRetry(ctx, resForRule, causeForRule) .handle((decision, cause) -> { if (cause != null) { logger.warn("Unexpected exception is raised from {}.", diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 0d150f02f97..365d0e7863a 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -33,6 +33,7 @@ import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; /** @@ -303,19 +304,23 @@ private void doExecuteAttempt(RetryingContext rctx, RetryAttempt attempt) { private CompletionStage decideOnAttempt(RetryingContext rctx, RetryAttempt attempt) { assert attempt.state() == RetryAttempt.State.COMPLETED; - return attempt.shouldRetry().handle((decision, cause) -> { - assert cause == null; - assert attempt.state() == RetryAttempt.State.COMPLETED; - final Backoff backoff = decision != null ? decision.backoff() : null; - if (backoff != null) { - final long millisAfter = useRetryAfter ? attempt.retryAfterMillis() : -1; - final long nextDelay = getNextDelay(rctx.ctx(), backoff, millisAfter); - if (nextDelay >= 0) { - return nextDelay; + try { + return attempt.shouldRetry().handle((decision, cause) -> { + assert cause == null; + assert attempt.state() == RetryAttempt.State.COMPLETED; + final Backoff backoff = decision != null ? decision.backoff() : null; + if (backoff != null) { + final long millisAfter = useRetryAfter ? attempt.retryAfterMillis() : -1; + final long nextDelay = getNextDelay(rctx.ctx(), backoff, millisAfter); + if (nextDelay >= 0) { + return nextDelay; + } } - } - return -1L; - }); + return -1L; + }); + } catch (Throwable t) { + return UnmodifiableFuture.exceptionallyCompletedFuture(t); + } } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java index ebdf7d789cd..7cb768bded4 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java @@ -131,7 +131,7 @@ void commit(RetryAttempt attempt) { } assert attempt.state() == RetryAttempt.State.COMPLETED; assert state == State.RETRYING; - state = State.COMPLETING; + state = State.COMPLETED; final HttpResponse attemptRes = attempt.commit(); ctx.logBuilder().endResponseWithLastChild(); @@ -148,7 +148,6 @@ void abort(Throwable cause) { assert state == State.RETRYING || state == State.COMPLETING; state = State.COMPLETED; - resFuture.completeExceptionally(cause); reqDuplicator.abort(cause); // todo(szymon): verify that this safe to do so we can avoid isInitialAttempt check @@ -157,5 +156,7 @@ void abort(Throwable cause) { } ctx.logBuilder().endResponse(cause); + + resFuture.completeExceptionally(cause); } } From 6d1ca15e58b6b912366ae85258f2bf1e5346ada6 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 27 Jun 2025 17:18:13 +0200 Subject: [PATCH 07/42] fix: GrpcWebRetryTest - provide a copy of response with content, otherwise we will have a double subscription --- .../armeria/client/retry/RetryAttempt.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java index 01d0bf1f599..fd75d858908 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java @@ -87,7 +87,7 @@ enum State { // Available only after Attempt.State.COMPLETED. @Nullable - private HttpResponse truncatedRes; + private HttpResponse resWithContent; @Nullable private Throwable resCause; @@ -99,7 +99,7 @@ enum State { this.number = number; whenCompletedFuture = new CompletableFuture<>(); - truncatedRes = null; + resWithContent = null; resCause = null; state = State.INITIALIZED; @@ -192,8 +192,8 @@ HttpResponse commit() { assert res != null; state = State.COMMITTED; - if (truncatedRes != null) { - truncatedRes.abort(); + if (resWithContent != null) { + resWithContent.abort(); } return res; @@ -217,8 +217,8 @@ void abort(Throwable cause) { assert ctx != null && res != null; state = State.ABORTED; - if (truncatedRes != null) { - truncatedRes.abort(); + if (resWithContent != null) { + resWithContent.abort(); } final RequestLogBuilder logBuilder = ctx.logBuilder(); @@ -270,7 +270,7 @@ private void handleAggRes() { } else { completeLogIfBytesNotTransferred(aggRes); ctx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { - complete(aggRes.toHttpResponse(), null); + completeWithContent(aggRes.toHttpResponse(), aggRes.toHttpResponse()); }); } return null; @@ -312,7 +312,7 @@ private void handleStreamingRes() { new TruncatingHttpResponse(resDuplicator.duplicate(), rctx.config().maxContentLength()); resDuplicator.close(); - completeWithTruncated(duplicatedRes, truncatingAttemptRes); + completeWithContent(duplicatedRes, truncatingAttemptRes); } else { if (resCause != null) { splitRes.body().abort(resCause); @@ -369,12 +369,12 @@ private void completeLogIfBytesNotTransferred( } } - private void completeWithTruncated(HttpResponse res, TruncatingHttpResponse truncatedRes) { + private void completeWithContent(HttpResponse res, HttpResponse resWithContent) { assert state == State.EXECUTING; state = State.COMPLETED; this.res = res; - this.truncatedRes = truncatedRes; + this.resWithContent = resWithContent; resCause = null; whenCompletedFuture.complete(null); } @@ -388,7 +388,7 @@ private void complete(HttpResponse res, @Nullable Throwable resCause) { } this.res = res; - truncatedRes = null; + resWithContent = null; this.resCause = resCause; whenCompletedFuture.complete(null); } @@ -422,10 +422,10 @@ private void complete(HttpResponse res, @Nullable Throwable resCause) { resForRule = null; causeForRule = resCause; } else { - if (truncatedRes == null) { + if (resWithContent == null) { resForRule = res; } else { - resForRule = truncatedRes; + resForRule = resWithContent; } causeForRule = null; From 5ea6ebdefc96fd9316f3b64124d87a9d58520e02 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 27 Jun 2025 17:24:54 +0200 Subject: [PATCH 08/42] docs: remove comments --- .../java/com/linecorp/armeria/client/retry/RetryAttempt.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java index fd75d858908..986a5ae6087 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java @@ -79,13 +79,11 @@ enum State { final int number; - // Available only in Attempt.State.EXECUTING. @Nullable private ClientRequestContext ctx; @Nullable private HttpResponse res; - // Available only after Attempt.State.COMPLETED. @Nullable private HttpResponse resWithContent; @Nullable From 75018163912412833a5a3b2e449b5880bb35c30d Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 27 Jun 2025 17:30:28 +0200 Subject: [PATCH 09/42] docs: correct comment --- .../java/com/linecorp/armeria/client/retry/RetryAttempt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java index 986a5ae6087..8e049ed1760 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java @@ -66,7 +66,7 @@ enum State { COMPLETED, // State after a call to `commit`. Terminal state, caller cannot make further calls. COMMITTED, - // State after a call to `commit`. Terminal state, caller cannot make further calls. + // State after a call to `abort`. Terminal state, caller cannot make further calls. // `res` is aborted. ABORTED } From a9d9cd24493e4cee1b041521e6e844f34b57c791 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 27 Jun 2025 17:32:45 +0200 Subject: [PATCH 10/42] refactor: deprecate underdefined `onRetryingComplete` --- .../linecorp/armeria/client/retry/AbstractRetryingClient.java | 3 +++ .../com/linecorp/armeria/client/retry/RetryingRpcClient.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 8d0d42e11b6..aa845dcc493 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -102,7 +102,10 @@ protected final RetryConfigMapping mapping() { /** * This should be called when retrying is finished. + * + * @deprecated Call ctx.logBuilder().endResponseWithLastChild(); instead of this method. */ + @Deprecated protected static void onRetryingComplete(ClientRequestContext ctx) { ctx.logBuilder().endResponseWithLastChild(); } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index dd44bce2d95..7f0011f21f9 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -226,7 +226,7 @@ private RpcResponse executeAttempt(RpcRequest req, ClientRequestContext attemptC private static void onRetryComplete(ClientRequestContext ctx, CompletableFuture resFuture, ClientRequestContext attemptCtx, RpcResponse attemptRes) { - onRetryingComplete(ctx); + ctx.logBuilder().endResponseWithLastChild(); final HttpRequest attemptReq = attemptCtx.request(); if (attemptReq != null) { ctx.updateRequest(attemptReq); From 6a0e105cdf726e3ecef6e5f79d1d50e765b09ad4 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 27 Jun 2025 17:35:58 +0200 Subject: [PATCH 11/42] fix: assert RETRYING state when creating attempt context --- .../java/com/linecorp/armeria/client/retry/RetryingContext.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java index 7cb768bded4..35685c2627c 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java @@ -105,6 +105,8 @@ boolean isCompleted() { } public ClientRequestContext newAttemptContext(int attemptNumber) { + assert state == State.RETRYING; + final boolean isInitialAttempt = attemptNumber <= 1; if (!AbstractRetryingClient.setResponseTimeout(ctx)) { From e428900febcfd4cb2bc59d27994e89844c029fb8 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 27 Jun 2025 18:23:22 +0200 Subject: [PATCH 12/42] refactor: move req duplicator init into RetryingContext --- .../armeria/client/retry/RetryingClient.java | 34 +++------ .../armeria/client/retry/RetryingContext.java | 69 +++++++++++++++---- 2 files changed, 64 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 365d0e7863a..bb53b8107e3 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -27,14 +27,11 @@ import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.HttpClient; -import com.linecorp.armeria.common.AggregationOptions; import com.linecorp.armeria.common.HttpRequest; -import com.linecorp.armeria.common.HttpRequestDuplicator; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.util.UnmodifiableFuture; -import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; /** * An {@link HttpClient} decorator that handles failures of an invocation and retries HTTP requests. @@ -223,30 +220,17 @@ public static Function newDecorator(RetryRul protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) throws Exception { final CompletableFuture resFuture = new CompletableFuture<>(); final HttpResponse res = HttpResponse.of(resFuture, ctx.eventLoop()); - if (ctx.exchangeType().isRequestStreaming()) { - final HttpRequestDuplicator reqDuplicator = - req.toDuplicator(ctx.eventLoop().withoutContext(), 0); - final RetryingContext rctx = new RetryingContext( - ctx, mappedRetryConfig(ctx), req, reqDuplicator, res, resFuture); + final RetryingContext rctx = new RetryingContext(ctx, mappedRetryConfig(ctx), res, resFuture, req); + rctx.init().handle((initSuccessful, unused) -> { + if (!initSuccessful) { + return null; + } + final RetryAttempt attempt = new RetryAttempt(rctx, unwrap(), 1); doExecuteAttempt(rctx, attempt); - } else { - req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) - .handle((agg, reqCause) -> { - if (reqCause != null) { - resFuture.completeExceptionally(reqCause); - ctx.logBuilder().endRequest(reqCause); - ctx.logBuilder().endResponse(reqCause); - } else { - final HttpRequestDuplicator reqDuplicator = new AggregatedHttpRequestDuplicator(agg); - final RetryingContext rctx = new RetryingContext( - ctx, mappedRetryConfig(ctx), req, reqDuplicator, res, resFuture); - final RetryAttempt attempt = new RetryAttempt(rctx, unwrap(), 1); - doExecuteAttempt(rctx, attempt); - } - return null; - }); - } + + return null; + }); return res; } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java index 35685c2627c..6ff54a62300 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java @@ -22,42 +22,47 @@ import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.ResponseTimeoutException; +import com.linecorp.armeria.common.AggregationOptions; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpRequestDuplicator; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.RequestHeadersBuilder; +import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.stream.AbortedStreamException; +import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; class RetryingContext { private enum State { - RETRYING, + UNINITIALIZED, + INITIALIZING, + INITIALIZED, COMPLETING, COMPLETED } private final ClientRequestContext ctx; private final RetryConfig retryConfig; - private final HttpRequest req; - private final HttpRequestDuplicator reqDuplicator; private final HttpResponse res; private final CompletableFuture resFuture; - + private final HttpRequest req; private State state; + @Nullable + private HttpRequestDuplicator reqDuplicator; + RetryingContext(ClientRequestContext ctx, RetryConfig retryConfig, - HttpRequest req, - HttpRequestDuplicator reqDuplicator, HttpResponse res, - CompletableFuture resFuture) { + CompletableFuture resFuture, + HttpRequest req) { this.ctx = ctx; this.retryConfig = retryConfig; - this.req = req; - this.reqDuplicator = reqDuplicator; this.res = res; this.resFuture = resFuture; - state = State.RETRYING; + this.req = req; + state = State.UNINITIALIZED; + reqDuplicator = null; // will be initialized in init(). } RetryConfig config() { @@ -68,12 +73,45 @@ ClientRequestContext ctx() { return ctx; } + CompletableFuture init() { + assert state == State.UNINITIALIZED; + state = State.INITIALIZING; + final CompletableFuture initFuture = new CompletableFuture<>(); + + if (ctx.exchangeType().isRequestStreaming()) { + reqDuplicator = + req.toDuplicator(ctx.eventLoop().withoutContext(), 0); + state = State.INITIALIZED; + initFuture.complete(true); + } else { + req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) + .handle((agg, reqCause) -> { + assert state == State.INITIALIZING; + + if (reqCause != null) { + resFuture.completeExceptionally(reqCause); + ctx.logBuilder().endRequest(reqCause); + ctx.logBuilder().endResponse(reqCause); + state = State.COMPLETED; + initFuture.complete(false); + } else { + reqDuplicator = new AggregatedHttpRequestDuplicator(agg); + state = State.INITIALIZED; + initFuture.complete(true); + } + return null; + }); + } + + return initFuture; + } + boolean isCompleted() { if (state == State.COMPLETING || state == State.COMPLETED) { return true; } - assert state == State.RETRYING; + assert state == State.INITIALIZED; // The request or response has been aborted by the client before it receives a response, // so stop retrying. @@ -105,7 +143,8 @@ boolean isCompleted() { } public ClientRequestContext newAttemptContext(int attemptNumber) { - assert state == State.RETRYING; + assert state == State.INITIALIZED; + assert reqDuplicator != null; final boolean isInitialAttempt = attemptNumber <= 1; @@ -132,7 +171,8 @@ void commit(RetryAttempt attempt) { return; } assert attempt.state() == RetryAttempt.State.COMPLETED; - assert state == State.RETRYING; + assert state == State.INITIALIZED; + assert reqDuplicator != null; state = State.COMPLETED; final HttpResponse attemptRes = attempt.commit(); @@ -147,7 +187,8 @@ void abort(Throwable cause) { return; } - assert state == State.RETRYING || state == State.COMPLETING; + assert state == State.INITIALIZED || state == State.COMPLETING; + assert reqDuplicator != null; state = State.COMPLETED; reqDuplicator.abort(cause); From 784c54ed9b948a24e954facaed9bb7562895bb4e Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 4 Jul 2025 13:48:31 +0200 Subject: [PATCH 13/42] refactor: move attempt context creation and attempt execution into `RetryingContext` --- .../armeria/client/retry/RetryAttempt.java | 95 ++-------------- .../armeria/client/retry/RetryingClient.java | 25 ++--- .../armeria/client/retry/RetryingContext.java | 105 ++++++++++++------ 3 files changed, 89 insertions(+), 136 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java index 8e049ed1760..28535800ba3 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java @@ -16,9 +16,6 @@ package com.linecorp.armeria.client.retry; -import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; -import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; - import java.time.Duration; import java.util.Date; import java.util.concurrent.CompletableFuture; @@ -27,11 +24,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.HttpHeaderNames; -import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.HttpResponseDuplicator; import com.linecorp.armeria.common.ResponseHeaders; @@ -43,8 +38,6 @@ import com.linecorp.armeria.common.logging.RequestLogProperty; import com.linecorp.armeria.common.stream.AbortedStreamException; import com.linecorp.armeria.common.util.Exceptions; -import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; -import com.linecorp.armeria.internal.client.ClientRequestContextExtension; import com.linecorp.armeria.internal.client.TruncatingHttpResponse; import io.netty.handler.codec.DateFormatter; @@ -54,10 +47,6 @@ class RetryAttempt { enum State { // Initial state after constructing an `Attempt`. - // Caller can only invoke `execute` or `abort`. - INITIALIZED, - // State after calling `execute` - // `ctx` and `res` are now available // The attempt response is underway but did not complete yet. EXECUTING, // State after the (maybe exceptional) result of the attempt response was processed. @@ -73,15 +62,9 @@ enum State { private State state; + private final RetryingContext rctx; + private final ClientRequestContext ctx; private final CompletableFuture whenCompletedFuture; - - RetryingContext rctx; - - final int number; - - @Nullable - private ClientRequestContext ctx; - @Nullable private HttpResponse res; @Nullable @@ -89,49 +72,30 @@ enum State { @Nullable private Throwable resCause; - Client delegate; - - RetryAttempt(RetryingContext rctx, Client delegate, int number) { + RetryAttempt(RetryingContext rctx, ClientRequestContext ctx, HttpResponse res) { this.rctx = rctx; - this.delegate = delegate; - this.number = number; + this.ctx = ctx; + this.res = res; whenCompletedFuture = new CompletableFuture<>(); resWithContent = null; resCause = null; - state = State.INITIALIZED; - } - - State state() { - return state; - } - - CompletableFuture execute() { - assert state == State.INITIALIZED; state = State.EXECUTING; - try { - ctx = rctx.newAttemptContext(number); - } catch (Throwable t) { - abort(t); - return whenCompletedFuture; - } - - res = executeAttemptRequest(); - if (!rctx.ctx().exchangeType().isResponseStreaming() || rctx.config().requiresResponseTrailers()) { handleAggRes(); } else { handleStreamingRes(); } + } - return whenCompletedFuture; + State state() { + return state; } CompletionStage<@Nullable RetryDecision> shouldRetry() { assert state == State.COMPLETED; - assert ctx != null; if (rctx.config().needsContentInRule()) { final RetryRuleWithContent retryRuleWithContent = @@ -147,7 +111,6 @@ CompletableFuture execute() { long retryAfterMillis() { assert state == State.COMPLETED; - assert ctx != null; final RequestLogAccess attemptLog = ctx.log(); final String retryAfterValue; @@ -182,12 +145,10 @@ long retryAfterMillis() { HttpResponse commit() { if (state == State.COMMITTED) { - assert res != null; return res; } assert state == State.COMPLETED; - assert res != null; state = State.COMMITTED; if (resWithContent != null) { @@ -206,13 +167,7 @@ void abort(Throwable cause) { return; } - if (state == State.INITIALIZED) { - assert ctx == null && res == null; - return; - } - assert state == State.EXECUTING || state == State.COMPLETED; - assert ctx != null && res != null; state = State.ABORTED; if (resWithContent != null) { @@ -228,38 +183,15 @@ void abort(Throwable cause) { whenCompletedFuture.completeExceptionally(cause); } - private HttpResponse executeAttemptRequest() { - assert state == State.EXECUTING; - assert ctx != null; - - final HttpRequest req = ctx.request(); - assert req != null; - - final ClientRequestContextExtension ctxExtension = - ctx.as(ClientRequestContextExtension.class); - if ((number > 1) && ctxExtension != null && ctx.endpoint() == null) { - // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(ctx); - // if the endpoint hasn't been selected, - // try to initialize the ctx with a new endpoint/event loop - return initContextAndExecuteWithFallback( - delegate, ctxExtension, HttpResponse::of, - (context, cause) -> - HttpResponse.ofFailure(cause), req, false); - } else { - return executeWithFallback(delegate, ctx, - (context, cause) -> - HttpResponse.ofFailure(cause), req, false); - } + public CompletableFuture whenCompleted() { + return whenCompletedFuture; } private void handleAggRes() { assert state == State.EXECUTING; - assert ctx != null && res != null; res.aggregate().handle((aggRes, resCause) -> { assert state == State.EXECUTING; - assert ctx != null && res != null; if (resCause != null) { ctx.logBuilder().endRequest(resCause); @@ -277,12 +209,10 @@ private void handleAggRes() { private void handleStreamingRes() { assert state == State.EXECUTING; - assert ctx != null && res != null; final SplitHttpResponse splitRes = res.split(); splitRes.headers().handle((resHeaders, headersCause) -> { assert state == State.EXECUTING; - assert ctx != null && res != null; final Throwable resCause; if (headersCause == null) { @@ -296,7 +226,6 @@ private void handleStreamingRes() { ctx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { assert state == State.EXECUTING; - assert ctx != null && res != null; if (rctx.config().needsContentInRule() && resCause == null) { final HttpResponse unsplitRes = splitRes.unsplit(); @@ -326,7 +255,6 @@ private void handleStreamingRes() { private void completeLogIfBytesNotTransferred(AggregatedHttpResponse aggRes) { assert state == State.EXECUTING; - assert ctx != null && res != null; if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { final RequestLogBuilder attemptLogBuilder = ctx.logBuilder(); @@ -343,7 +271,6 @@ private void completeLogIfBytesNotTransferred( @Nullable ResponseHeaders headers, @Nullable Throwable resCause) { assert state == State.EXECUTING; - assert ctx != null && res != null; if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { final RequestLogBuilder logBuilder = ctx.logBuilder(); @@ -393,7 +320,6 @@ private void complete(HttpResponse res, @Nullable Throwable resCause) { private CompletionStage<@Nullable RetryDecision> shouldBeRetriedWith(RetryRule retryRule) { assert state == State.COMPLETED; - assert ctx != null; return retryRule.shouldRetry(ctx, resCause) .handle((decision, cause) -> { @@ -409,7 +335,6 @@ private void complete(HttpResponse res, @Nullable Throwable resCause) { private CompletionStage<@Nullable RetryDecision> shouldBeRetriedWith( RetryRuleWithContent retryRuleWithContent) { assert state == State.COMPLETED; - assert ctx != null; @Nullable final HttpResponse resForRule; diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index bb53b8107e3..d0993d074c8 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -226,23 +226,24 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro return null; } - final RetryAttempt attempt = new RetryAttempt(rctx, unwrap(), 1); - doExecuteAttempt(rctx, attempt); + doExecuteAttempt(rctx); return null; }); return res; } - private void doExecuteAttempt(RetryingContext rctx, RetryAttempt attempt) { - if (rctx.isCompleted()) { - // The request or response has been aborted by the client before it receives a response, - // so stop retrying. - attempt.abort(); + private void doExecuteAttempt(RetryingContext rctx) { + @Nullable + final RetryAttempt attempt = rctx.newRetryAttempt(getTotalAttempts(rctx.ctx()), unwrap()); + + if (attempt == null) { + // Retrying completed already, error handling + // is done inside `RetryingContext`. return; } - attempt.execute().handle((unused, unexpectedAttemptCause) -> { + attempt.whenCompleted().handleAsync((unused, unexpectedAttemptCause) -> { if (unexpectedAttemptCause != null) { assert attempt.state() == RetryAttempt.State.ABORTED; rctx.abort(unexpectedAttemptCause); @@ -267,11 +268,7 @@ private void doExecuteAttempt(RetryingContext rctx, RetryAttempt attempt) { scheduleNextRetry( rctx.ctx(), rctx::abort, - () -> { - final RetryAttempt nextAttempt = new RetryAttempt( - rctx, unwrap(), getTotalAttempts(rctx.ctx())); - doExecuteAttempt(rctx, nextAttempt); - }, + () -> doExecuteAttempt(rctx), nextDelayMillis); return null; } @@ -282,7 +279,7 @@ private void doExecuteAttempt(RetryingContext rctx, RetryAttempt attempt) { }); return null; - }); + }, rctx.ctx().eventLoop()); } private CompletionStage decideOnAttempt(RetryingContext rctx, RetryAttempt attempt) { diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java index 6ff54a62300..bd8d6f6add7 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java @@ -17,9 +17,12 @@ package com.linecorp.armeria.client.retry; import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; +import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; +import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; import java.util.concurrent.CompletableFuture; +import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.common.AggregationOptions; @@ -30,6 +33,8 @@ import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.stream.AbortedStreamException; import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; +import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; +import com.linecorp.armeria.internal.client.ClientRequestContextExtension; class RetryingContext { private enum State { @@ -106,43 +111,12 @@ CompletableFuture init() { return initFuture; } - boolean isCompleted() { - if (state == State.COMPLETING || state == State.COMPLETED) { - return true; - } - - assert state == State.INITIALIZED; - - // The request or response has been aborted by the client before it receives a response, - // so stop retrying. - if (req.whenComplete().isCompletedExceptionally()) { - state = State.COMPLETING; - req.whenComplete().handle((unused, cause) -> { - abort(cause); - return null; - }); - return true; - } - - if (res.isComplete()) { - state = State.COMPLETING; - res.whenComplete().handle((result, cause) -> { - final Throwable abortCause; - if (cause != null) { - abortCause = cause; - } else { - abortCause = AbortedStreamException.get(); - } - abort(abortCause); - return null; - }); - return true; + @Nullable + public RetryAttempt newRetryAttempt(int attemptNumber, Client delegate) { + if (isCompleted()) { + return null; } - return false; - } - - public ClientRequestContext newAttemptContext(int attemptNumber) { assert state == State.INITIALIZED; assert reqDuplicator != null; @@ -153,6 +127,7 @@ public ClientRequestContext newAttemptContext(int attemptNumber) { } final HttpRequest attemptReq; + final ClientRequestContext attemptCtx; if (isInitialAttempt) { attemptReq = reqDuplicator.duplicate(); } else { @@ -161,8 +136,28 @@ public ClientRequestContext newAttemptContext(int attemptNumber) { attemptReq = reqDuplicator.duplicate(attemptHeadersBuilder.build()); } - return AbstractRetryingClient.newAttemptContext( - ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); + attemptCtx = AbstractRetryingClient.newAttemptContext( + ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); + + final HttpResponse attemptRes; + final ClientRequestContextExtension attemptCtxExt = + attemptCtx.as(ClientRequestContextExtension.class); + if (!isInitialAttempt && attemptCtxExt != null && attemptCtx.endpoint() == null) { + // clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); + // if the endpoint hasn't been selected, + // try to initialize the attempCtx with a new endpoint/event loop + attemptRes = initContextAndExecuteWithFallback( + delegate, attemptCtxExt, HttpResponse::of, + (context, cause) -> + HttpResponse.ofFailure(cause), attemptReq, false); + } else { + attemptRes = executeWithFallback(delegate, attemptCtx, + (context, cause) -> + HttpResponse.ofFailure(cause), attemptReq, false); + } + + return new RetryAttempt(this, attemptCtx, attemptRes); } void commit(RetryAttempt attempt) { @@ -202,4 +197,40 @@ void abort(Throwable cause) { resFuture.completeExceptionally(cause); } + + private boolean isCompleted() { + if (state == State.COMPLETING || state == State.COMPLETED) { + return true; + } + + assert state == State.INITIALIZED; + + // The request or response has been aborted by the client before it receives a response, + // so stop retrying. + if (req.whenComplete().isCompletedExceptionally()) { + state = State.COMPLETING; + req.whenComplete().handle((unused, cause) -> { + abort(cause); + return null; + }); + return true; + } + + if (res.isComplete()) { + state = State.COMPLETING; + res.whenComplete().handle((result, cause) -> { + final Throwable abortCause; + if (cause != null) { + abortCause = cause; + } else { + abortCause = AbortedStreamException.get(); + } + abort(abortCause); + return null; + }); + return true; + } + + return false; + } } From ceb30ed955a546cd4c2c7b8824fe78d3421c9c28 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 27 Jul 2025 13:19:05 +0200 Subject: [PATCH 14/42] feat: log when rctx initialization failed --- .../armeria/client/retry/RetryingClient.java | 3 ++- .../armeria/client/retry/RetryingContext.java | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index d0993d074c8..56564337c41 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -221,8 +221,9 @@ protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) thro final CompletableFuture resFuture = new CompletableFuture<>(); final HttpResponse res = HttpResponse.of(resFuture, ctx.eventLoop()); final RetryingContext rctx = new RetryingContext(ctx, mappedRetryConfig(ctx), res, resFuture, req); - rctx.init().handle((initSuccessful, unused) -> { + rctx.init().handle((initSuccessful, initCause) -> { if (!initSuccessful) { + logger.debug("RetryingContext initialization failed, not retrying: {}", rctx); return null; } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java index bd8d6f6add7..35fa4105cd1 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java @@ -22,6 +22,8 @@ import java.util.concurrent.CompletableFuture; +import com.google.common.base.MoreObjects; + import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.ResponseTimeoutException; @@ -233,4 +235,18 @@ private boolean isCompleted() { return false; } + + @Override + public String toString() { + return MoreObjects + .toStringHelper(this) + .add("ctx", ctx) + .add("retryConfig", retryConfig) + .add("res", res) + .add("resFuture", resFuture) + .add("req", req) + .add("state", state) + .add("reqDuplicator", reqDuplicator) + .toString(); + } } From 337c99211305cc3d0b4c540b01a40acb3548ae6a Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 17 Aug 2025 12:49:33 +0200 Subject: [PATCH 15/42] refactor: extract State from AbstractRetryingClient into RetryingContexts --- .../client/retry/AbstractRetryingClient.java | 340 +++----------- .../client/retry/HttpRetryAttempt.java | 388 ++++++++++++++++ .../client/retry/HttpRetryingContext.java | 438 ++++++++++++++++++ .../armeria/client/retry/RetryAttempt.java | 347 +------------- .../armeria/client/retry/RetryingClient.java | 99 +--- .../armeria/client/retry/RetryingContext.java | 233 +--------- .../client/retry/RetryingRpcClient.java | 114 +---- .../armeria/client/retry/RpcRetryAttempt.java | 133 ++++++ .../client/retry/RpcRetryingContext.java | 341 ++++++++++++++ .../RetryingClientLoadBalancingTest.java | 11 +- 10 files changed, 1388 insertions(+), 1056 deletions(-) create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index aa845dcc493..6cc78f5210f 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -15,32 +15,20 @@ */ package com.linecorp.armeria.client.retry; -import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.linecorp.armeria.client.Client; -import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; -import com.linecorp.armeria.client.Endpoint; import com.linecorp.armeria.client.SimpleDecoratingClient; import com.linecorp.armeria.common.HttpHeaderNames; -import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.Response; -import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.util.TimeoutMode; -import com.linecorp.armeria.internal.client.ClientUtil; import io.netty.util.AsciiString; -import io.netty.util.AttributeKey; -import io.netty.util.concurrent.ScheduledFuture; /** * A {@link Client} decorator that handles failures of remote invocation and retries requests. @@ -48,7 +36,8 @@ * @param the {@link Request} type * @param the {@link Response} type */ -public abstract class AbstractRetryingClient +public abstract class AbstractRetryingClient + , R extends RetryingContext> extends SimpleDecoratingClient { private static final Logger logger = LoggerFactory.getLogger(AbstractRetryingClient.class); @@ -59,10 +48,7 @@ public abstract class AbstractRetryingClient STATE = - AttributeKey.valueOf(AbstractRetryingClient.class, "STATE"); - - private final RetryConfigMapping mapping; + private final RetryConfigMapping retryMapping; @Nullable private final RetryConfig retryConfig; @@ -71,289 +57,73 @@ public abstract class AbstractRetryingClient delegate, RetryConfigMapping mapping, @Nullable RetryConfig retryConfig) { + Client delegate, RetryConfigMapping retryMapping, @Nullable RetryConfig retryConfig) { super(delegate); - this.mapping = requireNonNull(mapping, "mapping"); + this.retryMapping = requireNonNull(retryMapping, "retryMapping"); this.retryConfig = retryConfig; } @Override public final O execute(ClientRequestContext ctx, I req) throws Exception { - final RetryConfig config = mapping.get(ctx, req); - requireNonNull(config, "mapping.get() returned null"); - - final State state = new State(config, ctx.responseTimeoutMillis()); - ctx.setAttr(STATE, state); - return doExecute(ctx, req); - } - - /** - * Returns the current {@link RetryConfigMapping} set for this client. - */ - protected final RetryConfigMapping mapping() { - return mapping; - } - - /** - * Invoked by {@link #execute(ClientRequestContext, Request)} - * after the deadline for response timeout is set. - */ - protected abstract O doExecute(ClientRequestContext ctx, I req) throws Exception; - - /** - * This should be called when retrying is finished. - * - * @deprecated Call ctx.logBuilder().endResponseWithLastChild(); instead of this method. - */ - @Deprecated - protected static void onRetryingComplete(ClientRequestContext ctx) { - ctx.logBuilder().endResponseWithLastChild(); - } - - /** - * Returns the {@link RetryRule}. - * - * @throws IllegalStateException if the {@link RetryRule} is not set - */ - protected final RetryRule retryRule() { - checkState(retryConfig != null, "No retryRule set. Are you using RetryConfigMapping?"); - final RetryRule retryRule = retryConfig.retryRule(); - checkState(retryRule != null, "retryRule is not set."); - return retryRule; - } - - /** - * Fetches the {@link RetryConfig} that was mapped by the configured {@link RetryConfigMapping} for a given - * logical request. - */ - final RetryConfig mappedRetryConfig(ClientRequestContext ctx) { - @SuppressWarnings("unchecked") - final RetryConfig config = (RetryConfig) state(ctx).config; - return config; - } - - /** - * Returns the {@link RetryRuleWithContent}. - * - * @throws IllegalStateException if the {@link RetryRuleWithContent} is not set - */ - protected final RetryRuleWithContent retryRuleWithContent() { - checkState(retryConfig != null, "No retryRuleWithContent set. Are you using RetryConfigMapping?"); - final RetryRuleWithContent retryRuleWithContent = retryConfig.retryRuleWithContent(); - checkState(retryRuleWithContent != null, "retryRuleWithContent is not set."); - return retryRuleWithContent; - } - - /** - * Schedules next retry. - */ - protected static void scheduleNextRetry(ClientRequestContext ctx, - Consumer actionOnException, - Runnable retryTask, long nextDelayMillis) { - try { - if (nextDelayMillis == 0) { - ctx.eventLoop().execute(retryTask); - } else { - @SuppressWarnings("unchecked") - final ScheduledFuture scheduledFuture = (ScheduledFuture) ctx - .eventLoop().schedule(retryTask, nextDelayMillis, TimeUnit.MILLISECONDS); - scheduledFuture.addListener(future -> { - if (future.isCancelled()) { - // future is cancelled when the client factory is closed. - actionOnException.accept(new IllegalStateException( - ClientFactory.class.getSimpleName() + " has been closed.")); - } else if (future.cause() != null) { - // Other unexpected exceptions. - actionOnException.accept(future.cause()); - } - }); + final RetryConfig retryConfigForReq = + retryConfig != null ? retryConfig + : requireNonNull(retryMapping.get(ctx, req), + "retryMapping.get() returned null"); + + final R rctx = getRetryingContext(ctx, retryConfigForReq, req); + rctx.init().handle((initSuccessful, initCause) -> { + if (!initSuccessful || initCause != null) { + // todo(szymon): comment here. + logger.debug("RetryingContext initialization failed, not retrying: {}", rctx, initCause); + return null; } - } catch (Throwable t) { - actionOnException.accept(t); - } - } - /** - * Resets the {@link ClientRequestContext#responseTimeoutMillis()}. - * - * @return {@code true} if the response timeout is set, {@code false} if it can't be set due to the timeout - */ - protected static boolean setResponseTimeout(ClientRequestContext ctx) { - requireNonNull(ctx, "ctx"); - final long responseTimeoutMillis = state(ctx).responseTimeoutMillis(); - if (responseTimeoutMillis < 0) { - return false; - } else if (responseTimeoutMillis == 0) { - ctx.clearResponseTimeout(); - return true; - } else { - ctx.setResponseTimeoutMillis(TimeoutMode.SET_FROM_NOW, responseTimeoutMillis); - return true; - } - } + executeAttempt(null, rctx); + return null; + }); - /** - * Returns the next delay which retry will be made after. The delay will be: - * - *

{@code Math.min(responseTimeoutMillis, Backoff.nextDelayMillis(int))} - * - * @return the number of milliseconds to wait for before attempting a retry. -1 if the - * {@code currentAttemptNo} exceeds the {@code maxAttempts} or the {@code nextDelay} is after - * the moment which timeout happens. - */ - protected final long getNextDelay(ClientRequestContext ctx, Backoff backoff) { - return getNextDelay(ctx, backoff, -1); + return rctx.res(); } - /** - * Returns the next delay which retry will be made after. The delay will be: - * - *

{@code Math.min(responseTimeoutMillis, Math.max(Backoff.nextDelayMillis(int), - * millisAfterFromServer))} - * - * @return the number of milliseconds to wait for before attempting a retry. -1 if the - * {@code currentAttemptNo} exceeds the {@code maxAttempts} or the {@code nextDelay} is after - * the moment which timeout happens. - */ - @SuppressWarnings("MethodMayBeStatic") // Intentionally left non-static for better user experience. - protected final long getNextDelay(ClientRequestContext ctx, Backoff backoff, long millisAfterFromServer) { - requireNonNull(ctx, "ctx"); - requireNonNull(backoff, "backoff"); - final State state = state(ctx); - final int currentAttemptNo = state.currentAttemptNoWith(backoff); - - if (currentAttemptNo < 0) { - logger.debug("Exceeded the default number of max attempt: {}", state.config.maxTotalAttempts()); - return -1; - } - - long nextDelay = backoff.nextDelayMillis(currentAttemptNo); - if (nextDelay < 0) { - logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); - return -1; - } - - nextDelay = Math.max(nextDelay, millisAfterFromServer); - if (state.timeoutForWholeRetryEnabled() && nextDelay > state.actualResponseTimeoutMillis()) { - // The nextDelay will be after the moment which timeout will happen. So return just -1. - return -1; - } - - return nextDelay; - } - - /** - * Returns the total number of attempts of the current request represented by the specified - * {@link ClientRequestContext}. - */ - protected static int getTotalAttempts(ClientRequestContext ctx) { - final State state = ctx.attr(STATE); - if (state == null) { - return 0; - } - return state.totalAttemptNo; - } - - /** - * Creates a new derived {@link ClientRequestContext}, replacing the requests. - * If {@link ClientRequestContext#endpointGroup()} exists, a new {@link Endpoint} will be selected. - * - * @deprecated Use {@link #newAttemptContext(ClientRequestContext, HttpRequest, RpcRequest, boolean)} - * instead. - */ - @Deprecated - protected static ClientRequestContext newDerivedContext(ClientRequestContext ctx, - @Nullable HttpRequest req, - @Nullable RpcRequest rpcReq, - boolean initialAttempt) { - return newAttemptContext(ctx, req, rpcReq, initialAttempt); - } - - /** - * Creates a new {@link ClientRequestContext} for a retry attempt by replacing the request in - * {@link ClientRequestContext} with {@code req} or {@code rpcReq}. - * If {@link ClientRequestContext#endpointGroup()} exists, a new {@link Endpoint} will be selected. - */ - protected static ClientRequestContext newAttemptContext(ClientRequestContext ctx, - @Nullable HttpRequest req, - @Nullable RpcRequest rpcReq, - boolean initialAttempt) { - return ClientUtil.newDerivedContext(ctx, req, rpcReq, initialAttempt); - } - - private static State state(ClientRequestContext ctx) { - final State state = ctx.attr(STATE); - assert state != null; - return state; - } - - private static final class State { - - private final RetryConfig config; - private final long deadlineNanos; - private final boolean isTimeoutEnabled; + abstract R getRetryingContext(ClientRequestContext ctx, RetryConfig config, I req); + private void executeAttempt(@Nullable Backoff lastBackoff, R rctx) { @Nullable - private Backoff lastBackoff; - private int currentAttemptNoWithLastBackoff; - private int totalAttemptNo; - - State(RetryConfig config, long responseTimeoutMillis) { - this.config = config; - - if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { - deadlineNanos = 0; - isTimeoutEnabled = false; - } else { - deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); - isTimeoutEnabled = true; - } - totalAttemptNo = 1; - } - - /** - * Returns the smaller value between {@link RetryConfig#responseTimeoutMillisForEachAttempt()} and - * remaining {@link #responseTimeoutMillis}. - * - * @return 0 if the response timeout for both of each request and whole retry is disabled or - * -1 if the elapsed time from the first request has passed {@code responseTimeoutMillis} - */ - long responseTimeoutMillis() { - if (!timeoutForWholeRetryEnabled()) { - return config.responseTimeoutMillisForEachAttempt(); - } - - final long actualResponseTimeoutMillis = actualResponseTimeoutMillis(); - - // Consider 0 or less than 0 of actualResponseTimeoutMillis as timed out. - if (actualResponseTimeoutMillis <= 0) { - return -1; - } - - if (config.responseTimeoutMillisForEachAttempt() > 0) { - return Math.min(config.responseTimeoutMillisForEachAttempt(), actualResponseTimeoutMillis); - } - - return actualResponseTimeoutMillis; - } + final A attempt = rctx.executeAttempt(lastBackoff, unwrap()); - boolean timeoutForWholeRetryEnabled() { - return isTimeoutEnabled; + if (attempt == null) { + // Retrying completed already, error handling + // is done inside `RetryingContext`. + return; } - long actualResponseTimeoutMillis() { - return TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); - } - - int currentAttemptNoWith(Backoff backoff) { - if (totalAttemptNo++ >= config.maxTotalAttempts()) { - return -1; - } - if (lastBackoff != backoff) { - lastBackoff = backoff; - currentAttemptNoWithLastBackoff = 1; - } - return currentAttemptNoWithLastBackoff++; - } + attempt.whenDecided() + .handle((decision, decisionCause) -> { + if (decisionCause != null) { + rctx.abort(decisionCause); + return null; + } + + final Backoff backoff = decision != null ? decision.backoff() : null; + final long nextRetryTimeNanos; + if (backoff != null) { + nextRetryTimeNanos = rctx.nextRetryTimeNanos(attempt, backoff); + } else { + nextRetryTimeNanos = Long.MAX_VALUE; + } + + if (nextRetryTimeNanos < Long.MAX_VALUE) { + rctx.abort(attempt); + rctx.scheduleNextRetry( + nextRetryTimeNanos, + () -> executeAttempt(backoff, rctx), + rctx::abort + ); + } else { + rctx.commit(attempt); + } + + return null; + }); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java new file mode 100644 index 00000000000..d92bfff909d --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java @@ -0,0 +1,388 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.retry; + +import java.time.Duration; +import java.util.Date; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.MoreObjects; + +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.common.AggregatedHttpResponse; +import com.linecorp.armeria.common.HttpHeaderNames; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.HttpResponseDuplicator; +import com.linecorp.armeria.common.ResponseHeaders; +import com.linecorp.armeria.common.SplitHttpResponse; +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.logging.RequestLog; +import com.linecorp.armeria.common.logging.RequestLogAccess; +import com.linecorp.armeria.common.logging.RequestLogBuilder; +import com.linecorp.armeria.common.logging.RequestLogProperty; +import com.linecorp.armeria.common.stream.AbortedStreamException; +import com.linecorp.armeria.common.util.Exceptions; +import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.armeria.internal.client.TruncatingHttpResponse; + +import io.netty.handler.codec.DateFormatter; + +class HttpRetryAttempt implements RetryAttempt { + private static final Logger logger = LoggerFactory.getLogger(HttpRetryAttempt.class); + + enum State { + // Initial state after constructing an `Attempt`. + // The attempt response is underway but did not complete yet. + EXECUTING, + // todo(szymon): doc + DECIDING, + // todo(szymon): doc + DECIDED, + // State after a call to `commit`. Terminal state, caller cannot make further calls. + COMMITTED, + // State after a call to `abort`. Terminal state, caller cannot make further calls. + // `res` is aborted. + ABORTED + } + + private State state; + + private final HttpRetryingContext rctx; + private final ClientRequestContext ctx; + private HttpResponse res; + @Nullable + private Throwable resCause; + @Nullable + private HttpResponse resWithContent; + private final CompletableFuture<@Nullable RetryDecision> whenDecidedFuture; + + HttpRetryAttempt(HttpRetryingContext rctx, ClientRequestContext ctx, HttpResponse res, + boolean isResponseStreaming) { + this.rctx = rctx; + this.ctx = ctx; + this.res = res; + resCause = null; + resWithContent = null; + whenDecidedFuture = new CompletableFuture<>(); + + state = State.EXECUTING; + + if (!isResponseStreaming || rctx.config().requiresResponseTrailers()) { + handleAggRes(); + } else { + handleStreamingRes(); + } + } + + private void decide() { + assert state == State.EXECUTING; + state = State.DECIDING; + + final CompletionStage<@Nullable RetryDecision> retryRuleDecisionFuture; + if (rctx.config().needsContentInRule()) { + final RetryRuleWithContent retryRuleWithContent = + rctx.config().retryRuleWithContent(); + assert retryRuleWithContent != null; + retryRuleDecisionFuture = shouldBeRetriedWith(retryRuleWithContent); + } else { + final RetryRule retryRule = rctx.config().retryRule(); + assert retryRule != null; + retryRuleDecisionFuture = shouldBeRetriedWith(retryRule); + } + + retryRuleDecisionFuture.whenComplete((result, exception) -> { + state = State.DECIDED; + if (exception != null) { + whenDecidedFuture.completeExceptionally(exception); + } else { + whenDecidedFuture.complete(result); + } + }); + } + + public HttpResponse commit() { + if (state == State.COMMITTED) { + return res; + } + + assert state == State.DECIDED; + state = State.COMMITTED; + + if (resWithContent != null) { + resWithContent.abort(); + } + + return res; + } + + public void abort() { + abort(AbortedStreamException.get()); + } + + public void abort(Throwable cause) { + if (state == State.ABORTED || state == State.COMMITTED) { + return; + } + + assert state == State.EXECUTING || state == State.DECIDED; + state = State.ABORTED; + + if (resWithContent != null) { + resWithContent.abort(); + } + + final RequestLogBuilder logBuilder = ctx.logBuilder(); + // Set response content with null to make sure that the log is complete. + logBuilder.responseContent(null, null); + logBuilder.responseContentPreview(null); + res.abort(cause); + + whenDecidedFuture.completeExceptionally(cause); + } + + @Override + public CompletableFuture<@Nullable RetryDecision> whenDecided() { + return whenDecidedFuture; + } + + ClientRequestContext ctx() { + return ctx; + } + + State state() { + return state; + } + + long retryAfterMillis() { + assert state == State.DECIDED; + + final RequestLogAccess attemptLog = ctx.log(); + final String retryAfterValue; + final RequestLog requestLog = attemptLog.getIfAvailable(RequestLogProperty.RESPONSE_HEADERS); + retryAfterValue = requestLog != null ? + requestLog.responseHeaders().get(HttpHeaderNames.RETRY_AFTER) : null; + + if (retryAfterValue != null) { + try { + return Duration.ofSeconds(Integer.parseInt(retryAfterValue)).toMillis(); + } catch (Exception ignored) { + // Not a second value. + } + + try { + @SuppressWarnings("UseOfObsoleteDateTimeApi") + final Date retryAfterDate = DateFormatter.parseHttpDate(retryAfterValue); + if (retryAfterDate != null) { + return retryAfterDate.getTime() - System.currentTimeMillis(); + } + } catch (Exception ignored) { + // `parseHttpDate()` can raise an exception rather than returning `null` + // when the given value has more than 64 characters. + } + + logger.debug("The retryAfter: {}, from the server is neither an HTTP date nor a second.", + retryAfterValue); + } + + return -1; + } + + private void handleAggRes() { + assert state == State.EXECUTING; + + res.aggregate().handle((aggRes, resCause) -> { + assert state == State.EXECUTING; + + if (resCause != null) { + ctx.logBuilder().endRequest(resCause); + ctx.logBuilder().endResponse(resCause); + complete(HttpResponse.ofFailure(resCause), resCause); + } else { + completeLogIfBytesNotTransferred(aggRes); + ctx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { + completeWithContent(aggRes.toHttpResponse(), aggRes.toHttpResponse()); + }); + } + return null; + }); + } + + private void handleStreamingRes() { + assert state == State.EXECUTING; + + final SplitHttpResponse splitRes = res.split(); + splitRes.headers().handle((resHeaders, headersCause) -> { + assert state == State.EXECUTING; + + final Throwable resCause; + if (headersCause == null) { + final RequestLog log = ctx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); + resCause = log != null ? log.responseCause() : null; + } else { + resCause = Exceptions.peel(headersCause); + } + + completeLogIfBytesNotTransferred(resHeaders, resCause); + + ctx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { + assert state == State.EXECUTING; + + if (rctx.config().needsContentInRule() && resCause == null) { + final HttpResponse unsplitRes = splitRes.unsplit(); + final HttpResponseDuplicator resDuplicator = + unsplitRes.toDuplicator(ctx.eventLoop().withoutContext(), + ctx.maxResponseLength()); + // todo(szymon): We do not call duplicator.abort(cause); but res.abort on an exception. + // Is this okay? + final HttpResponse duplicatedRes = resDuplicator.duplicate(); + final TruncatingHttpResponse truncatingAttemptRes = + new TruncatingHttpResponse(resDuplicator.duplicate(), + rctx.config().maxContentLength()); + resDuplicator.close(); + completeWithContent(duplicatedRes, truncatingAttemptRes); + } else { + if (resCause != null) { + splitRes.body().abort(resCause); + complete(HttpResponse.ofFailure(resCause), resCause); + } else { + complete(splitRes.unsplit(), null); + } + } + }); + return null; + }); + } + + private void completeLogIfBytesNotTransferred(AggregatedHttpResponse aggRes) { + assert state == State.EXECUTING; + + if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { + final RequestLogBuilder attemptLogBuilder = ctx.logBuilder(); + attemptLogBuilder.endRequest(); + attemptLogBuilder.responseHeaders(aggRes.headers()); + if (!aggRes.trailers().isEmpty()) { + attemptLogBuilder.responseTrailers(aggRes.trailers()); + } + attemptLogBuilder.endResponse(); + } + } + + private void completeLogIfBytesNotTransferred( + @Nullable ResponseHeaders headers, + @Nullable Throwable resCause) { + assert state == State.EXECUTING; + + if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { + final RequestLogBuilder logBuilder = ctx.logBuilder(); + if (resCause != null) { + logBuilder.endRequest(resCause); + logBuilder.endResponse(resCause); + } else { + logBuilder.endRequest(); + if (headers != null) { + logBuilder.responseHeaders(headers); + } + res.whenComplete().handle((unused, cause) -> { + if (cause != null) { + logBuilder.endResponse(cause); + } else { + logBuilder.endResponse(); + } + return null; + }); + } + } + } + + private void completeWithContent(HttpResponse res, HttpResponse resWithContent) { + assert state == State.EXECUTING; + + this.res = res; + this.resWithContent = resWithContent; + resCause = null; + decide(); + } + + private void complete(HttpResponse res, @Nullable Throwable resCause) { + assert state == State.EXECUTING; + + if (resCause != null) { + resCause = Exceptions.peel(resCause); + } + + this.res = res; + resWithContent = null; + this.resCause = resCause; + decide(); + } + + private CompletionStage<@Nullable RetryDecision> shouldBeRetriedWith(RetryRule retryRule) { + assert state == State.DECIDING; + + try { + return retryRule.shouldRetry(ctx, resCause); + } catch (Throwable t) { + return UnmodifiableFuture.exceptionallyCompletedFuture(t); + } + } + + private CompletionStage<@Nullable RetryDecision> shouldBeRetriedWith( + RetryRuleWithContent retryRuleWithContent) { + assert state == State.DECIDING; + + @Nullable + final HttpResponse resForRule; + @Nullable + final Throwable causeForRule; + + if (resCause != null) { + resForRule = null; + causeForRule = resCause; + } else { + if (resWithContent == null) { + resForRule = res; + } else { + resForRule = resWithContent; + } + + causeForRule = null; + } + + try { + return retryRuleWithContent.shouldRetry(ctx, resForRule, causeForRule); + } catch (Throwable t) { + return UnmodifiableFuture.exceptionallyCompletedFuture(t); + } + } + + @Override + public String toString() { + return MoreObjects + .toStringHelper(this) + .add("state", state) + .add("rctx", rctx) + .add("ctx", ctx) + .add("res", res) + .add("resCause", resCause) + .add("resWithContent", resWithContent) + .toString(); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java new file mode 100644 index 00000000000..4438d5d8e32 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java @@ -0,0 +1,438 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.retry; + +import static com.google.common.base.Preconditions.checkState; +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; +import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; +import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.MoreObjects; + +import com.linecorp.armeria.client.Client; +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.client.ResponseTimeoutException; +import com.linecorp.armeria.common.AggregationOptions; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.common.HttpRequestDuplicator; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.RequestHeadersBuilder; +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.stream.AbortedStreamException; +import com.linecorp.armeria.common.util.TimeoutMode; +import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; +import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; +import com.linecorp.armeria.internal.client.ClientRequestContextExtension; +import com.linecorp.armeria.internal.client.ClientUtil; + +import io.netty.util.concurrent.ScheduledFuture; + +final class HttpRetryingContext implements RetryingContext { + private static final Logger logger = LoggerFactory.getLogger(HttpRetryingContext.class); + + private enum State { + UNINITIALIZED, + INITIALIZING, + INITIALIZED, + COMPLETING, + COMPLETED + } + + private State state; + private final ClientRequestContext ctx; + private final RetryConfig retryConfig; + private final HttpResponse res; + private final CompletableFuture resFuture; + private final HttpRequest req; + @Nullable + private HttpRequestDuplicator reqDuplicator; + + private int numberAttemptsSoFar; + + private final long deadlineTimeNanos; + private final boolean hasDeadline; + + @Nullable + private Backoff lastBackoff; + private int numberAttemptsWithBackoffSoFar; + + private final boolean useRetryAfter; + List attemptsSoFar; + + HttpRetryingContext(ClientRequestContext ctx, + RetryConfig retryConfig, + HttpResponse res, + CompletableFuture resFuture, + HttpRequest req, + boolean useRetryAfter) { + + state = State.UNINITIALIZED; + this.ctx = ctx; + this.retryConfig = retryConfig; + this.resFuture = resFuture; + this.res = res; + this.req = req; + reqDuplicator = null; // will be initialized in init(). + + numberAttemptsSoFar = 0; + + final long responseTimeoutMillis = ctx.responseTimeoutMillis(); + if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { + deadlineTimeNanos = 0; + hasDeadline = false; + } else { + deadlineTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); + hasDeadline = true; + } + + lastBackoff = null; + numberAttemptsWithBackoffSoFar = 0; + + this.useRetryAfter = useRetryAfter; + attemptsSoFar = new LinkedList<>(); + } + + @Override + public CompletableFuture init() { + assert state == State.UNINITIALIZED; + state = State.INITIALIZING; + final CompletableFuture initFuture = new CompletableFuture<>(); + + if (ctx.exchangeType().isRequestStreaming()) { + reqDuplicator = + req.toDuplicator(ctx.eventLoop().withoutContext(), 0); + state = State.INITIALIZED; + initFuture.complete(true); + } else { + req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) + .handle((agg, reqCause) -> { + assert state == State.INITIALIZING; + + if (reqCause != null) { + resFuture.completeExceptionally(reqCause); + ctx.logBuilder().endRequest(reqCause); + ctx.logBuilder().endResponse(reqCause); + state = State.COMPLETED; + initFuture.complete(false); + } else { + reqDuplicator = new AggregatedHttpRequestDuplicator(agg); + state = State.INITIALIZED; + initFuture.complete(true); + } + return null; + }); + } + + return initFuture; + } + + @Override + @Nullable + public HttpRetryAttempt executeAttempt(@Nullable Backoff backoff, + Client delegate) { + if (isCompleted()) { + return null; + } + + assert state == State.INITIALIZED; + assert reqDuplicator != null; + + if (numberAttemptsSoFar >= retryConfig.maxTotalAttempts()) { + throw new IllegalStateException( + "Exceeded the maximum number of attempts: " + retryConfig.maxTotalAttempts()); + } + + final int attemptNumber = ++numberAttemptsSoFar; + + if (backoff != null) { + if (lastBackoff != backoff) { + lastBackoff = backoff; + numberAttemptsWithBackoffSoFar = 0; + } + numberAttemptsWithBackoffSoFar++; + } + + final boolean isInitialAttempt = attemptNumber <= 1; + + if (!setResponseTimeout()) { + throw ResponseTimeoutException.get(); + } + + final HttpRequest attemptReq; + final ClientRequestContext attemptCtx; + if (isInitialAttempt) { + attemptReq = reqDuplicator.duplicate(); + } else { + final RequestHeadersBuilder attemptHeadersBuilder = req.headers().toBuilder(); + attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, attemptNumber - 1); + attemptReq = reqDuplicator.duplicate(attemptHeadersBuilder.build()); + } + + attemptCtx = ClientUtil.newDerivedContext(ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); + + final HttpResponse attemptRes; + final ClientRequestContextExtension attemptCtxExt = + attemptCtx.as(ClientRequestContextExtension.class); + if (!isInitialAttempt && attemptCtxExt != null && attemptCtx.endpoint() == null) { + // clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); + // if the endpoint hasn't been selected, + // try to initialize the attempCtx with a new endpoint/event loop + attemptRes = initContextAndExecuteWithFallback( + delegate, attemptCtxExt, HttpResponse::of, + (context, cause) -> + HttpResponse.ofFailure(cause), attemptReq, false); + } else { + attemptRes = executeWithFallback(delegate, attemptCtx, + (context, cause) -> + HttpResponse.ofFailure(cause), attemptReq, false); + } + + final HttpRetryAttempt attempt = new HttpRetryAttempt(this, attemptCtx, attemptRes, + ctx.exchangeType().isResponseStreaming()); + attemptsSoFar.add(attempt); + return attempt; + } + + // returns Long.MAX_VALUE if no retry is possible. + @Override + public long nextRetryTimeNanos(HttpRetryAttempt attempt, Backoff backoff) { + if (state != State.INITIALIZED) { + return Long.MAX_VALUE; + } + + if (numberAttemptsSoFar >= retryConfig.maxTotalAttempts()) { + logger.debug("Exceeded the default number of max attempt: {}", retryConfig.maxTotalAttempts()); + return Long.MAX_VALUE; + } + + final int numberAttemptsWithThisBackoffSoFar; + if (lastBackoff != backoff) { + numberAttemptsWithThisBackoffSoFar = 1; + } else { + numberAttemptsWithThisBackoffSoFar = numberAttemptsWithBackoffSoFar; + } + + long nextRetryDelayMillis = backoff.nextDelayMillis(numberAttemptsWithThisBackoffSoFar); + if (nextRetryDelayMillis < 0) { + logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); + return Long.MAX_VALUE; + } + + nextRetryDelayMillis = Math.max(nextRetryDelayMillis, useRetryAfter ? attempt.retryAfterMillis() : -1); + final long nextDelayTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(nextRetryDelayMillis); + + if (hasDeadline && nextDelayTimeNanos > deadlineTimeNanos) { + // The next retry will be after the response deadline. So return just Long.MAX_VALUE. + return Long.MAX_VALUE; + } + + return nextDelayTimeNanos; + } + + @Override + public void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, + Consumer exceptionHandler) { + try { + final long nextRetryDelayMillis = TimeUnit.NANOSECONDS.toMillis( + nextRetryTimeNanos - System.nanoTime()); + if (nextRetryDelayMillis <= 0) { + ctx.eventLoop().execute(retryTask); + } else { + @SuppressWarnings("unchecked") + final ScheduledFuture scheduledFuture = (ScheduledFuture) ctx + .eventLoop().schedule(retryTask, nextRetryDelayMillis, TimeUnit.MILLISECONDS); + scheduledFuture.addListener(future -> { + if (future.isCancelled()) { + // future is cancelled when the client factory is closed. + exceptionHandler.accept(new IllegalStateException( + ClientFactory.class.getSimpleName() + " has been closed.")); + } else if (future.cause() != null) { + // Other unexpected exceptions. + exceptionHandler.accept(future.cause()); + } + }); + } + } catch (Throwable t) { + exceptionHandler.accept(t); + } + } + + @Override + public void commit(HttpRetryAttempt attemptToCommit) { + if (state == State.COMPLETED) { + // Already completed. + return; + } + checkState(attemptToCommit.state() == HttpRetryAttempt.State.DECIDED); + assert state == State.INITIALIZED; + assert reqDuplicator != null; + state = State.COMPLETED; + + for (final HttpRetryAttempt attempt : attemptsSoFar) { + if (attempt != attemptToCommit) { + // todo(szymon): check state. + attempt.abort(); + } + } + + final HttpResponse attemptRes = attemptToCommit.commit(); + // todo(szymon): replace with endResponseWithChild + ctx.logBuilder().endResponseWithChild(attemptToCommit.ctx().log()); + resFuture.complete(attemptRes); + reqDuplicator.close(); + } + + @Override + public void abort(HttpRetryAttempt attempt) { + assert state == State.INITIALIZED || state == State.COMPLETING || state == State.COMPLETED; + attempt.abort(); + } + + @Override + public void abort(Throwable cause) { + if (state == State.COMPLETED) { + // Already completed. + return; + } + + assert state == State.INITIALIZED || state == State.COMPLETING; + assert reqDuplicator != null; + state = State.COMPLETED; + + for (final HttpRetryAttempt attempt : attemptsSoFar) { + // todo(szymon): check state. + attempt.abort(); + } + + reqDuplicator.abort(cause); + + // todo(szymon): verify that this safe to do so we can avoid isInitialAttempt check + if (!ctx.log().isRequestComplete()) { + ctx.logBuilder().endRequest(cause); + } + ctx.logBuilder().endResponse(cause); + + resFuture.completeExceptionally(cause); + } + + private boolean isCompleted() { + if (state == State.COMPLETING || state == State.COMPLETED) { + return true; + } + + assert state == State.INITIALIZED; + + // The request or response has been aborted by the client before it receives a response, + // so stop retrying. + if (req.whenComplete().isCompletedExceptionally()) { + state = State.COMPLETING; + req.whenComplete().handle((unused, cause) -> { + abort(cause); + return null; + }); + return true; + } + + if (res.isComplete()) { + state = State.COMPLETING; + res.whenComplete().handle((result, cause) -> { + final Throwable abortCause; + if (cause != null) { + abortCause = cause; + } else { + abortCause = AbortedStreamException.get(); + } + abort(abortCause); + return null; + }); + return true; + } + + return false; + } + + @Override + public HttpResponse res() { + return res; + } + + RetryConfig config() { + return retryConfig; + } + + public boolean setResponseTimeout() { + final long responseTimeoutMillis = responseTimeoutMillis(); + if (responseTimeoutMillis < 0) { + return false; + } else if (responseTimeoutMillis == 0) { + ctx.clearResponseTimeout(); + return true; + } else { + ctx.setResponseTimeoutMillis(TimeoutMode.SET_FROM_NOW, responseTimeoutMillis); + return true; + } + } + + private long responseTimeoutMillis() { + if (!hasDeadline) { + return retryConfig.responseTimeoutMillisForEachAttempt(); + } + + final long remainingTimeUntilDeadlineMillis = TimeUnit.NANOSECONDS.toMillis( + deadlineTimeNanos - System.nanoTime()); + + // Consider 0 or less than 0 of actualResponseTimeoutMillis as timed out. + if (remainingTimeUntilDeadlineMillis <= 0) { + return -1; + } + + if (retryConfig.responseTimeoutMillisForEachAttempt() > 0) { + return Math.min(retryConfig.responseTimeoutMillisForEachAttempt(), + remainingTimeUntilDeadlineMillis); + } + + return remainingTimeUntilDeadlineMillis; + } + + @Override + public String toString() { + return MoreObjects + .toStringHelper(this) + .add("state", state) + .add("ctx", ctx) + .add("retryConfig", retryConfig) + .add("req", req) + .add("res", res) + .add("deadlineTimeNanos", deadlineTimeNanos) + .add("hasDeadline", hasDeadline) + .add("lastBackoff", lastBackoff) + .add("numberAttemptsSoFar", numberAttemptsSoFar) + .add("numberAttemptsWithBackoffSoFar", numberAttemptsWithBackoffSoFar) + .toString(); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java index 28535800ba3..ac290e9db60 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java @@ -16,352 +16,11 @@ package com.linecorp.armeria.client.retry; -import java.time.Duration; -import java.util.Date; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.linecorp.armeria.client.ClientRequestContext; -import com.linecorp.armeria.common.AggregatedHttpResponse; -import com.linecorp.armeria.common.HttpHeaderNames; -import com.linecorp.armeria.common.HttpResponse; -import com.linecorp.armeria.common.HttpResponseDuplicator; -import com.linecorp.armeria.common.ResponseHeaders; -import com.linecorp.armeria.common.SplitHttpResponse; +import com.linecorp.armeria.common.Response; import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.logging.RequestLog; -import com.linecorp.armeria.common.logging.RequestLogAccess; -import com.linecorp.armeria.common.logging.RequestLogBuilder; -import com.linecorp.armeria.common.logging.RequestLogProperty; -import com.linecorp.armeria.common.stream.AbortedStreamException; -import com.linecorp.armeria.common.util.Exceptions; -import com.linecorp.armeria.internal.client.TruncatingHttpResponse; - -import io.netty.handler.codec.DateFormatter; - -class RetryAttempt { - private static final Logger logger = LoggerFactory.getLogger(RetryAttempt.class); - - enum State { - // Initial state after constructing an `Attempt`. - // The attempt response is underway but did not complete yet. - EXECUTING, - // State after the (maybe exceptional) result of the attempt response was processed. - // `res` is available. `truncatedRes` and `resCause` are also available if applicable. - // Caller can only invoke `shouldRetry`, `commit` or `abort`. - COMPLETED, - // State after a call to `commit`. Terminal state, caller cannot make further calls. - COMMITTED, - // State after a call to `abort`. Terminal state, caller cannot make further calls. - // `res` is aborted. - ABORTED - } - - private State state; - - private final RetryingContext rctx; - private final ClientRequestContext ctx; - private final CompletableFuture whenCompletedFuture; - private HttpResponse res; - - @Nullable - private HttpResponse resWithContent; - @Nullable - private Throwable resCause; - - RetryAttempt(RetryingContext rctx, ClientRequestContext ctx, HttpResponse res) { - this.rctx = rctx; - this.ctx = ctx; - this.res = res; - whenCompletedFuture = new CompletableFuture<>(); - - resWithContent = null; - resCause = null; - - state = State.EXECUTING; - - if (!rctx.ctx().exchangeType().isResponseStreaming() || rctx.config().requiresResponseTrailers()) { - handleAggRes(); - } else { - handleStreamingRes(); - } - } - - State state() { - return state; - } - - CompletionStage<@Nullable RetryDecision> shouldRetry() { - assert state == State.COMPLETED; - - if (rctx.config().needsContentInRule()) { - final RetryRuleWithContent retryRuleWithContent = - rctx.config().retryRuleWithContent(); - assert retryRuleWithContent != null; - return shouldBeRetriedWith(retryRuleWithContent); - } else { - final RetryRule retryRule = rctx.config().retryRule(); - assert retryRule != null; - return shouldBeRetriedWith(retryRule); - } - } - - long retryAfterMillis() { - assert state == State.COMPLETED; - - final RequestLogAccess attemptLog = ctx.log(); - final String retryAfterValue; - final RequestLog requestLog = attemptLog.getIfAvailable(RequestLogProperty.RESPONSE_HEADERS); - retryAfterValue = requestLog != null ? - requestLog.responseHeaders().get(HttpHeaderNames.RETRY_AFTER) : null; - - if (retryAfterValue != null) { - try { - return Duration.ofSeconds(Integer.parseInt(retryAfterValue)).toMillis(); - } catch (Exception ignored) { - // Not a second value. - } - - try { - @SuppressWarnings("UseOfObsoleteDateTimeApi") - final Date retryAfterDate = DateFormatter.parseHttpDate(retryAfterValue); - if (retryAfterDate != null) { - return retryAfterDate.getTime() - System.currentTimeMillis(); - } - } catch (Exception ignored) { - // `parseHttpDate()` can raise an exception rather than returning `null` - // when the given value has more than 64 characters. - } - - logger.debug("The retryAfter: {}, from the server is neither an HTTP date nor a second.", - retryAfterValue); - } - - return -1; - } - - HttpResponse commit() { - if (state == State.COMMITTED) { - return res; - } - - assert state == State.COMPLETED; - state = State.COMMITTED; - - if (resWithContent != null) { - resWithContent.abort(); - } - - return res; - } - - void abort() { - abort(AbortedStreamException.get()); - } - - void abort(Throwable cause) { - if (state == State.ABORTED || state == State.COMMITTED) { - return; - } - - assert state == State.EXECUTING || state == State.COMPLETED; - state = State.ABORTED; - - if (resWithContent != null) { - resWithContent.abort(); - } - - final RequestLogBuilder logBuilder = ctx.logBuilder(); - // Set response content with null to make sure that the log is complete. - logBuilder.responseContent(null, null); - logBuilder.responseContentPreview(null); - res.abort(cause); - - whenCompletedFuture.completeExceptionally(cause); - } - - public CompletableFuture whenCompleted() { - return whenCompletedFuture; - } - - private void handleAggRes() { - assert state == State.EXECUTING; - - res.aggregate().handle((aggRes, resCause) -> { - assert state == State.EXECUTING; - - if (resCause != null) { - ctx.logBuilder().endRequest(resCause); - ctx.logBuilder().endResponse(resCause); - complete(HttpResponse.ofFailure(resCause), resCause); - } else { - completeLogIfBytesNotTransferred(aggRes); - ctx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { - completeWithContent(aggRes.toHttpResponse(), aggRes.toHttpResponse()); - }); - } - return null; - }); - } - - private void handleStreamingRes() { - assert state == State.EXECUTING; - - final SplitHttpResponse splitRes = res.split(); - splitRes.headers().handle((resHeaders, headersCause) -> { - assert state == State.EXECUTING; - - final Throwable resCause; - if (headersCause == null) { - final RequestLog log = ctx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); - resCause = log != null ? log.responseCause() : null; - } else { - resCause = Exceptions.peel(headersCause); - } - - completeLogIfBytesNotTransferred(resHeaders, resCause); - - ctx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { - assert state == State.EXECUTING; - - if (rctx.config().needsContentInRule() && resCause == null) { - final HttpResponse unsplitRes = splitRes.unsplit(); - final HttpResponseDuplicator resDuplicator = - unsplitRes.toDuplicator(ctx.eventLoop().withoutContext(), - ctx.maxResponseLength()); - // todo(szymon): We do not call duplicator.abort(cause); but res.abort on an exception. - // Is this okay? - final HttpResponse duplicatedRes = resDuplicator.duplicate(); - final TruncatingHttpResponse truncatingAttemptRes = - new TruncatingHttpResponse(resDuplicator.duplicate(), - rctx.config().maxContentLength()); - resDuplicator.close(); - completeWithContent(duplicatedRes, truncatingAttemptRes); - } else { - if (resCause != null) { - splitRes.body().abort(resCause); - complete(HttpResponse.ofFailure(resCause), resCause); - } else { - complete(splitRes.unsplit(), null); - } - } - }); - return null; - }); - } - - private void completeLogIfBytesNotTransferred(AggregatedHttpResponse aggRes) { - assert state == State.EXECUTING; - - if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { - final RequestLogBuilder attemptLogBuilder = ctx.logBuilder(); - attemptLogBuilder.endRequest(); - attemptLogBuilder.responseHeaders(aggRes.headers()); - if (!aggRes.trailers().isEmpty()) { - attemptLogBuilder.responseTrailers(aggRes.trailers()); - } - attemptLogBuilder.endResponse(); - } - } - - private void completeLogIfBytesNotTransferred( - @Nullable ResponseHeaders headers, - @Nullable Throwable resCause) { - assert state == State.EXECUTING; - - if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { - final RequestLogBuilder logBuilder = ctx.logBuilder(); - if (resCause != null) { - logBuilder.endRequest(resCause); - logBuilder.endResponse(resCause); - } else { - logBuilder.endRequest(); - if (headers != null) { - logBuilder.responseHeaders(headers); - } - res.whenComplete().handle((unused, cause) -> { - if (cause != null) { - logBuilder.endResponse(cause); - } else { - logBuilder.endResponse(); - } - return null; - }); - } - } - } - - private void completeWithContent(HttpResponse res, HttpResponse resWithContent) { - assert state == State.EXECUTING; - state = State.COMPLETED; - - this.res = res; - this.resWithContent = resWithContent; - resCause = null; - whenCompletedFuture.complete(null); - } - - private void complete(HttpResponse res, @Nullable Throwable resCause) { - assert state == State.EXECUTING; - state = State.COMPLETED; - - if (resCause != null) { - resCause = Exceptions.peel(resCause); - } - - this.res = res; - resWithContent = null; - this.resCause = resCause; - whenCompletedFuture.complete(null); - } - - private CompletionStage<@Nullable RetryDecision> shouldBeRetriedWith(RetryRule retryRule) { - assert state == State.COMPLETED; - - return retryRule.shouldRetry(ctx, resCause) - .handle((decision, cause) -> { - if (cause != null) { - logger.warn("Unexpected exception is raised from {}.", - retryRule, cause); - return null; - } - return decision; - }); - } - - private CompletionStage<@Nullable RetryDecision> shouldBeRetriedWith( - RetryRuleWithContent retryRuleWithContent) { - assert state == State.COMPLETED; - - @Nullable - final HttpResponse resForRule; - @Nullable - final Throwable causeForRule; - - if (resCause != null) { - resForRule = null; - causeForRule = resCause; - } else { - if (resWithContent == null) { - resForRule = res; - } else { - resForRule = resWithContent; - } - - causeForRule = null; - } - return retryRuleWithContent.shouldRetry(ctx, resForRule, causeForRule) - .handle((decision, cause) -> { - if (cause != null) { - logger.warn("Unexpected exception is raised from {}.", - retryRuleWithContent, cause); - return null; - } - return decision; - }); - } +interface RetryAttempt { + CompletableFuture<@Nullable RetryDecision> whenDecided(); } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 56564337c41..b083184afed 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -19,28 +19,21 @@ import static com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; import java.util.function.Function; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.HttpClient; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.util.UnmodifiableFuture; /** * An {@link HttpClient} decorator that handles failures of an invocation and retries HTTP requests. */ -public final class RetryingClient extends AbstractRetryingClient +public final class RetryingClient + extends AbstractRetryingClient implements HttpClient { - - private static final Logger logger = LoggerFactory.getLogger(RetryingClient.class); - /** * Returns a new {@link RetryingClientBuilder} with the specified {@link RetryConfig}. * The {@link RetryConfig} object encapsulates {@link RetryRule} or {@link RetryRuleWithContent}, @@ -217,92 +210,10 @@ public static Function newDecorator(RetryRul } @Override - protected HttpResponse doExecute(ClientRequestContext ctx, HttpRequest req) throws Exception { + HttpRetryingContext getRetryingContext(ClientRequestContext ctx, RetryConfig config, + HttpRequest req) { final CompletableFuture resFuture = new CompletableFuture<>(); final HttpResponse res = HttpResponse.of(resFuture, ctx.eventLoop()); - final RetryingContext rctx = new RetryingContext(ctx, mappedRetryConfig(ctx), res, resFuture, req); - rctx.init().handle((initSuccessful, initCause) -> { - if (!initSuccessful) { - logger.debug("RetryingContext initialization failed, not retrying: {}", rctx); - return null; - } - - doExecuteAttempt(rctx); - - return null; - }); - return res; - } - - private void doExecuteAttempt(RetryingContext rctx) { - @Nullable - final RetryAttempt attempt = rctx.newRetryAttempt(getTotalAttempts(rctx.ctx()), unwrap()); - - if (attempt == null) { - // Retrying completed already, error handling - // is done inside `RetryingContext`. - return; - } - - attempt.whenCompleted().handleAsync((unused, unexpectedAttemptCause) -> { - if (unexpectedAttemptCause != null) { - assert attempt.state() == RetryAttempt.State.ABORTED; - rctx.abort(unexpectedAttemptCause); - return null; - } - - assert attempt.state() == RetryAttempt.State.COMPLETED; - - decideOnAttempt(rctx, attempt).handle((nextDelayMillis, unexpectedDecisionCause) -> { - if (unexpectedDecisionCause != null) { - attempt.abort(unexpectedDecisionCause); - assert attempt.state() == RetryAttempt.State.ABORTED; - rctx.abort(unexpectedDecisionCause); - return null; - } - - assert attempt.state() == RetryAttempt.State.COMPLETED; - - if (nextDelayMillis >= 0) { - attempt.abort(); - assert attempt.state() == RetryAttempt.State.ABORTED; - - scheduleNextRetry( - rctx.ctx(), rctx::abort, - () -> doExecuteAttempt(rctx), - nextDelayMillis); - return null; - } - - rctx.commit(attempt); - assert attempt.state() == RetryAttempt.State.COMMITTED; - return null; - }); - - return null; - }, rctx.ctx().eventLoop()); - } - - private CompletionStage decideOnAttempt(RetryingContext rctx, RetryAttempt attempt) { - assert attempt.state() == RetryAttempt.State.COMPLETED; - - try { - return attempt.shouldRetry().handle((decision, cause) -> { - assert cause == null; - assert attempt.state() == RetryAttempt.State.COMPLETED; - final Backoff backoff = decision != null ? decision.backoff() : null; - if (backoff != null) { - final long millisAfter = useRetryAfter ? attempt.retryAfterMillis() : -1; - final long nextDelay = getNextDelay(rctx.ctx(), backoff, millisAfter); - if (nextDelay >= 0) { - return nextDelay; - } - } - - return -1L; - }); - } catch (Throwable t) { - return UnmodifiableFuture.exceptionallyCompletedFuture(t); - } + return new HttpRetryingContext(ctx, config, res, resFuture, req, useRetryAfter); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java index 35fa4105cd1..8edf43a9548 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java @@ -16,237 +16,30 @@ package com.linecorp.armeria.client.retry; -import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; -import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; -import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; - import java.util.concurrent.CompletableFuture; - -import com.google.common.base.MoreObjects; +import java.util.function.Consumer; import com.linecorp.armeria.client.Client; -import com.linecorp.armeria.client.ClientRequestContext; -import com.linecorp.armeria.client.ResponseTimeoutException; -import com.linecorp.armeria.common.AggregationOptions; -import com.linecorp.armeria.common.HttpRequest; -import com.linecorp.armeria.common.HttpRequestDuplicator; -import com.linecorp.armeria.common.HttpResponse; -import com.linecorp.armeria.common.RequestHeadersBuilder; +import com.linecorp.armeria.common.Request; +import com.linecorp.armeria.common.Response; import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.stream.AbortedStreamException; -import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; -import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; -import com.linecorp.armeria.internal.client.ClientRequestContextExtension; - -class RetryingContext { - private enum State { - UNINITIALIZED, - INITIALIZING, - INITIALIZED, - COMPLETING, - COMPLETED - } - - private final ClientRequestContext ctx; - private final RetryConfig retryConfig; - private final HttpResponse res; - private final CompletableFuture resFuture; - private final HttpRequest req; - private State state; - - @Nullable - private HttpRequestDuplicator reqDuplicator; - - RetryingContext(ClientRequestContext ctx, - RetryConfig retryConfig, - HttpResponse res, - CompletableFuture resFuture, - HttpRequest req) { - this.ctx = ctx; - this.retryConfig = retryConfig; - this.res = res; - this.resFuture = resFuture; - this.req = req; - state = State.UNINITIALIZED; - reqDuplicator = null; // will be initialized in init(). - } - - RetryConfig config() { - return retryConfig; - } - - ClientRequestContext ctx() { - return ctx; - } - - CompletableFuture init() { - assert state == State.UNINITIALIZED; - state = State.INITIALIZING; - final CompletableFuture initFuture = new CompletableFuture<>(); - - if (ctx.exchangeType().isRequestStreaming()) { - reqDuplicator = - req.toDuplicator(ctx.eventLoop().withoutContext(), 0); - state = State.INITIALIZED; - initFuture.complete(true); - } else { - req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) - .handle((agg, reqCause) -> { - assert state == State.INITIALIZING; - - if (reqCause != null) { - resFuture.completeExceptionally(reqCause); - ctx.logBuilder().endRequest(reqCause); - ctx.logBuilder().endResponse(reqCause); - state = State.COMPLETED; - initFuture.complete(false); - } else { - reqDuplicator = new AggregatedHttpRequestDuplicator(agg); - state = State.INITIALIZED; - initFuture.complete(true); - } - return null; - }); - } - - return initFuture; - } +interface RetryingContext> { + CompletableFuture init(); @Nullable - public RetryAttempt newRetryAttempt(int attemptNumber, Client delegate) { - if (isCompleted()) { - return null; - } - - assert state == State.INITIALIZED; - assert reqDuplicator != null; - - final boolean isInitialAttempt = attemptNumber <= 1; - - if (!AbstractRetryingClient.setResponseTimeout(ctx)) { - throw ResponseTimeoutException.get(); - } - - final HttpRequest attemptReq; - final ClientRequestContext attemptCtx; - if (isInitialAttempt) { - attemptReq = reqDuplicator.duplicate(); - } else { - final RequestHeadersBuilder attemptHeadersBuilder = req.headers().toBuilder(); - attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, attemptNumber - 1); - attemptReq = reqDuplicator.duplicate(attemptHeadersBuilder.build()); - } - - attemptCtx = AbstractRetryingClient.newAttemptContext( - ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); - - final HttpResponse attemptRes; - final ClientRequestContextExtension attemptCtxExt = - attemptCtx.as(ClientRequestContextExtension.class); - if (!isInitialAttempt && attemptCtxExt != null && attemptCtx.endpoint() == null) { - // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); - // if the endpoint hasn't been selected, - // try to initialize the attempCtx with a new endpoint/event loop - attemptRes = initContextAndExecuteWithFallback( - delegate, attemptCtxExt, HttpResponse::of, - (context, cause) -> - HttpResponse.ofFailure(cause), attemptReq, false); - } else { - attemptRes = executeWithFallback(delegate, attemptCtx, - (context, cause) -> - HttpResponse.ofFailure(cause), attemptReq, false); - } - - return new RetryAttempt(this, attemptCtx, attemptRes); - } - - void commit(RetryAttempt attempt) { - if (state == State.COMPLETED) { - // Already completed. - return; - } - assert attempt.state() == RetryAttempt.State.COMPLETED; - assert state == State.INITIALIZED; - assert reqDuplicator != null; - state = State.COMPLETED; - - final HttpResponse attemptRes = attempt.commit(); - ctx.logBuilder().endResponseWithLastChild(); - resFuture.complete(attemptRes); - reqDuplicator.close(); - } - - void abort(Throwable cause) { - if (state == State.COMPLETED) { - // Already completed. - return; - } - - assert state == State.INITIALIZED || state == State.COMPLETING; - assert reqDuplicator != null; - state = State.COMPLETED; - - reqDuplicator.abort(cause); - - // todo(szymon): verify that this safe to do so we can avoid isInitialAttempt check - if (!ctx.log().isRequestComplete()) { - ctx.logBuilder().endRequest(cause); - } - - ctx.logBuilder().endResponse(cause); - - resFuture.completeExceptionally(cause); - } + A executeAttempt(@Nullable Backoff backoff, Client delegate); - private boolean isCompleted() { - if (state == State.COMPLETING || state == State.COMPLETED) { - return true; - } + long nextRetryTimeNanos(A attempt, Backoff backoff); - assert state == State.INITIALIZED; + void scheduleNextRetry(long retryTimeNanos, Runnable retryTask, + Consumer exceptionHandler); - // The request or response has been aborted by the client before it receives a response, - // so stop retrying. - if (req.whenComplete().isCompletedExceptionally()) { - state = State.COMPLETING; - req.whenComplete().handle((unused, cause) -> { - abort(cause); - return null; - }); - return true; - } + void commit(A attempt); - if (res.isComplete()) { - state = State.COMPLETING; - res.whenComplete().handle((result, cause) -> { - final Throwable abortCause; - if (cause != null) { - abortCause = cause; - } else { - abortCause = AbortedStreamException.get(); - } - abort(abortCause); - return null; - }); - return true; - } + void abort(A attempt); - return false; - } + void abort(Throwable cause); - @Override - public String toString() { - return MoreObjects - .toStringHelper(this) - .add("ctx", ctx) - .add("retryConfig", retryConfig) - .add("res", res) - .add("resFuture", resFuture) - .add("req", req) - .add("state", state) - .add("reqDuplicator", reqDuplicator) - .toString(); - } + O res(); } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 7f0011f21f9..65eae07a683 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -15,29 +15,20 @@ */ package com.linecorp.armeria.client.retry; -import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; -import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; - -import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.function.Function; import com.linecorp.armeria.client.ClientRequestContext; -import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.client.RpcClient; -import com.linecorp.armeria.client.endpoint.EndpointGroup; -import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; -import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; -import com.linecorp.armeria.internal.client.ClientRequestContextExtension; -import com.linecorp.armeria.internal.common.util.StringUtil; /** * An {@link RpcClient} decorator that handles failures of an invocation and retries RPC requests. */ -public final class RetryingRpcClient extends AbstractRetryingClient +public final class RetryingRpcClient + extends AbstractRetryingClient implements RpcClient { /** @@ -141,105 +132,10 @@ public static RetryingRpcClientBuilder builder(RetryConfigMapping m } @Override - protected RpcResponse doExecute(ClientRequestContext ctx, RpcRequest req) throws Exception { + RpcRetryingContext getRetryingContext( + ClientRequestContext ctx, RetryConfig retryConfig, RpcRequest req) { final CompletableFuture resFuture = new CompletableFuture<>(); final RpcResponse res = RpcResponse.from(resFuture); - doExecute0(ctx, req, res, resFuture); - return res; - } - - private void doExecute0(ClientRequestContext ctx, RpcRequest req, - RpcResponse res, CompletableFuture resFuture) { - final int attemptNo = getTotalAttempts(ctx); - final boolean isInitialAttempt = attemptNo <= 1; - if (res.isDone()) { - // The response has been cancelled by the client before it receives a response, so stop retrying. - handleException(ctx, resFuture, new CancellationException( - "the response returned to the client has been cancelled"), isInitialAttempt); - return; - } - if (!setResponseTimeout(ctx)) { - handleException(ctx, resFuture, ResponseTimeoutException.get(), isInitialAttempt); - return; - } - - final ClientRequestContext attemptCtx = newAttemptContext(ctx, null, req, isInitialAttempt); - - if (!isInitialAttempt) { - attemptCtx.mutateAdditionalRequestHeaders( - mutator -> mutator.add(ARMERIA_RETRY_COUNT, StringUtil.toString(attemptNo - 1))); - } - - final RpcResponse attemptRes = executeAttempt(req, attemptCtx, isInitialAttempt); - - final RetryConfig retryConfig = mappedRetryConfig(ctx); - final RetryRuleWithContent retryRule = - retryConfig.needsContentInRule() ? - retryConfig.retryRuleWithContent() : retryConfig.fromRetryRule(); - attemptRes.handle((unused1, cause) -> { - try { - assert retryRule != null; - retryRule.shouldRetry(attemptCtx, attemptRes, cause).handle((decision, unused3) -> { - final Backoff backoff = decision != null ? decision.backoff() : null; - if (backoff != null) { - final long nextDelay = getNextDelay(attemptCtx, backoff); - if (nextDelay < 0) { - onRetryComplete(ctx, resFuture, attemptCtx, attemptRes); - return null; - } - - scheduleNextRetry(ctx, cause0 -> handleException(ctx, resFuture, cause0, false), - () -> doExecute0(ctx, req, res, resFuture), nextDelay); - } else { - onRetryComplete(ctx, resFuture, attemptCtx, attemptRes); - } - return null; - }); - } catch (Throwable t) { - handleException(ctx, resFuture, t, false); - } - return null; - }); - } - - private RpcResponse executeAttempt(RpcRequest req, ClientRequestContext attemptCtx, - boolean isInitialAttempt) { - final RpcResponse attemptRes; - final ClientRequestContextExtension attemptCtxExtension = - attemptCtx.as(ClientRequestContextExtension.class); - final EndpointGroup endpointGroup = attemptCtx.endpointGroup(); - if (!isInitialAttempt && attemptCtxExtension != null && - endpointGroup != null && attemptCtx.endpoint() == null) { - // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); - // if the endpoint hasn't been selected, try to initialize the ctx with a new endpoint/event loop - attemptRes = initContextAndExecuteWithFallback(unwrap(), attemptCtxExtension, RpcResponse::from, - (unused, cause) -> RpcResponse.ofFailure(cause), - req, true); - } else { - attemptRes = executeWithFallback(unwrap(), attemptCtx, - (unused, cause) -> RpcResponse.ofFailure(cause), - req, true); - } - return attemptRes; - } - - private static void onRetryComplete(ClientRequestContext ctx, CompletableFuture resFuture, - ClientRequestContext attemptCtx, RpcResponse attemptRes) { - ctx.logBuilder().endResponseWithLastChild(); - final HttpRequest attemptReq = attemptCtx.request(); - if (attemptReq != null) { - ctx.updateRequest(attemptReq); - } - resFuture.complete(attemptRes); - } - - private static void handleException(ClientRequestContext ctx, CompletableFuture resFuture, - Throwable cause, boolean endRequestLog) { - resFuture.completeExceptionally(cause); - if (endRequestLog) { - ctx.logBuilder().endRequest(cause); - } - ctx.logBuilder().endResponse(cause); + return new RpcRetryingContext(ctx, retryConfig, resFuture, res, req); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java new file mode 100644 index 00000000000..30605dd733d --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java @@ -0,0 +1,133 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.retry; + +import java.util.concurrent.CompletableFuture; + +import com.google.common.base.MoreObjects; + +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.common.RpcResponse; +import com.linecorp.armeria.common.annotation.Nullable; + +final class RpcRetryAttempt implements RetryAttempt { + // todo(szymon): doc + enum State { + EXECUTING, + DECIDING, + DECIDED, + COMMITTED, + ABORTED + } + + State state; + + private final RpcRetryingContext rctx; + private final ClientRequestContext ctx; + private final RpcResponse res; + @Nullable + private Throwable resCause; + private final CompletableFuture<@Nullable RetryDecision> whenDecidedFuture; + + RpcRetryAttempt(RpcRetryingContext rctx, ClientRequestContext ctx, + RpcResponse res) { + this.rctx = rctx; + this.ctx = ctx; + this.res = res; + resCause = null; + whenDecidedFuture = new CompletableFuture<>(); + + state = State.EXECUTING; + + res.handle((unused, cause) -> { + resCause = cause; + decide(); + return null; + }); + } + + private void decide() { + assert state == State.EXECUTING; + state = State.DECIDING; + + final RetryRuleWithContent retryRule = + rctx.config().needsContentInRule() ? rctx.config().retryRuleWithContent() + : rctx.config().fromRetryRule(); + + assert retryRule != null; + try { + retryRule.shouldRetry(ctx, res, resCause) + .handle((decision, cause) -> { + state = State.DECIDED; + + if (cause == null) { + whenDecidedFuture.complete(decision); + } else { + whenDecidedFuture.completeExceptionally(cause); + } + return null; + }); + } catch (Throwable t) { + state = State.DECIDED; + whenDecidedFuture.completeExceptionally(t); + } + } + + RpcResponse commit() { + if (state == State.COMMITTED) { + return res; + } + + assert state == State.DECIDED; + state = State.COMMITTED; + return res; + } + + void abort() { + if (state == State.ABORTED) { + return; + } + + assert state == State.DECIDED; + state = State.ABORTED; + } + + @Override + public CompletableFuture<@Nullable RetryDecision> whenDecided() { + return whenDecidedFuture; + } + + ClientRequestContext ctx() { + return ctx; + } + + State state() { + return state; + } + + @Override + public String toString() { + return MoreObjects + .toStringHelper(this) + .add("state", state) + .add("rctx", rctx) + .add("ctx", ctx) + .add("res", res) + .add("resCause", resCause) + .toString(); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java new file mode 100644 index 00000000000..74d53e9124d --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java @@ -0,0 +1,341 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.retry; + +import static com.google.common.base.Preconditions.checkState; +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; +import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; +import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.MoreObjects; + +import com.linecorp.armeria.client.Client; +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.client.ResponseTimeoutException; +import com.linecorp.armeria.client.endpoint.EndpointGroup; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.common.RpcRequest; +import com.linecorp.armeria.common.RpcResponse; +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.util.TimeoutMode; +import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; +import com.linecorp.armeria.internal.client.ClientRequestContextExtension; +import com.linecorp.armeria.internal.client.ClientUtil; + +import io.netty.util.concurrent.ScheduledFuture; + +final class RpcRetryingContext implements RetryingContext { + private static final Logger logger = LoggerFactory.getLogger(RpcRetryingContext.class); + + private enum State { + INITIALIZED, + COMPLETED + } + + private State state; + private final ClientRequestContext ctx; + private final RetryConfig retryConfig; + private final CompletableFuture resFuture; + private final RpcResponse res; + private final RpcRequest req; + + private int numberAttemptsSoFar; + + private final long deadlineTimeNanos; + private final boolean hasDeadline; + + @Nullable + private Backoff lastBackoff; + private int numberAttemptsWithBackoffSoFar; + + List attemptsSoFar; + + RpcRetryingContext(ClientRequestContext ctx, + RetryConfig retryConfig, + CompletableFuture resFuture, + RpcResponse res, + RpcRequest req) { + state = State.INITIALIZED; + this.ctx = ctx; + this.retryConfig = retryConfig; + this.resFuture = resFuture; + this.res = res; + this.req = req; + + numberAttemptsSoFar = 0; + + final long responseTimeoutMillis = ctx.responseTimeoutMillis(); + if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { + deadlineTimeNanos = 0; + hasDeadline = false; + } else { + deadlineTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); + hasDeadline = true; + } + + lastBackoff = null; + numberAttemptsWithBackoffSoFar = 0; + + attemptsSoFar = new LinkedList<>(); + } + + @Override + public CompletableFuture init() { + assert state == State.INITIALIZED; + // Nothing special to initialize for RPC retrying. + return UnmodifiableFuture.completedFuture(true); + } + + @Override + @Nullable + public RpcRetryAttempt executeAttempt(@Nullable Backoff backoff, + Client delegate) { + if (res.isDone()) { + return null; + } + + assert state == State.INITIALIZED; + + if (numberAttemptsSoFar >= retryConfig.maxTotalAttempts()) { + throw new IllegalStateException( + "Exceeded the maximum number of attempts: " + retryConfig.maxTotalAttempts()); + } + + final int attemptNumber = ++numberAttemptsSoFar; + + if (backoff != null) { + if (lastBackoff != backoff) { + lastBackoff = backoff; + numberAttemptsWithBackoffSoFar = 0; + } + numberAttemptsWithBackoffSoFar++; + } + + final boolean isInitialAttempt = attemptNumber <= 1; + + if (!setResponseTimeout()) { + throw ResponseTimeoutException.get(); + } + + final ClientRequestContext attemptCtx = ClientUtil.newDerivedContext(ctx, null, req, isInitialAttempt); + + if (!isInitialAttempt) { + attemptCtx.mutateAdditionalRequestHeaders( + mutator -> mutator.add(ARMERIA_RETRY_COUNT, Integer.toString(attemptNumber - 1))); + } + + final RpcResponse attemptRes; + final ClientRequestContextExtension attemptCtxExtension = + attemptCtx.as(ClientRequestContextExtension.class); + final EndpointGroup endpointGroup = attemptCtx.endpointGroup(); + if (!isInitialAttempt && attemptCtxExtension != null && + endpointGroup != null && attemptCtx.endpoint() == null) { + // Clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); + // Initialize the context with a new endpoint/event loop if not selected yet + attemptRes = initContextAndExecuteWithFallback(delegate, attemptCtxExtension, RpcResponse::from, + (unused, cause) -> RpcResponse.ofFailure(cause), + req, true); + } else { + attemptRes = executeWithFallback(delegate, attemptCtx, + (unused, cause) -> RpcResponse.ofFailure(cause), + req, true); + } + + final RpcRetryAttempt attempt = new RpcRetryAttempt(this, attemptCtx, attemptRes); + attemptsSoFar.add(attempt); + return attempt; + } + + @Override + public long nextRetryTimeNanos(RpcRetryAttempt attempt, Backoff backoff) { + if (state != State.INITIALIZED) { + return Long.MAX_VALUE; + } + + if (numberAttemptsSoFar >= retryConfig.maxTotalAttempts()) { + logger.debug("Exceeded the default number of max attempt: {}", retryConfig.maxTotalAttempts()); + return Long.MAX_VALUE; + } + + final int numberAttemptsWithThisBackoffSoFar; + if (lastBackoff != backoff) { + numberAttemptsWithThisBackoffSoFar = 1; + } else { + numberAttemptsWithThisBackoffSoFar = numberAttemptsWithBackoffSoFar; + } + + final long nextRetryDelayMillis = backoff.nextDelayMillis(numberAttemptsWithThisBackoffSoFar); + if (nextRetryDelayMillis < 0) { + logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); + return Long.MAX_VALUE; + } + + final long nextDelayTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(nextRetryDelayMillis); + if (hasDeadline && nextDelayTimeNanos > deadlineTimeNanos) { + // The next retry will be after the response deadline. So return just Long.MAX_VALUE. + return Long.MAX_VALUE; + } + return nextDelayTimeNanos; + } + + @Override + public void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, + Consumer exceptionHandler) { + try { + final long nextRetryDelayMillis = TimeUnit.NANOSECONDS.toMillis( + nextRetryTimeNanos - System.nanoTime()); + if (nextRetryDelayMillis <= 0) { + ctx.eventLoop().execute(retryTask); + } else { + @SuppressWarnings("unchecked") + final ScheduledFuture scheduledFuture = (ScheduledFuture) ctx + .eventLoop().schedule(retryTask, nextRetryDelayMillis, TimeUnit.MILLISECONDS); + scheduledFuture.addListener(future -> { + if (future.isCancelled()) { + exceptionHandler.accept(new IllegalStateException( + ClientFactory.class.getSimpleName() + " has been closed.")); + } else if (future.cause() != null) { + exceptionHandler.accept(future.cause()); + } + }); + } + } catch (Throwable t) { + exceptionHandler.accept(t); + } + } + + @Override + public void commit(RpcRetryAttempt attemptToCommit) { + if (state == State.COMPLETED) { + // Already completed, so just return. + return; + } + checkState(attemptToCommit.state() == RpcRetryAttempt.State.DECIDED); + assert state == State.INITIALIZED; + state = State.COMPLETED; + + for (final RpcRetryAttempt attempt : attemptsSoFar) { + if (attempt != attemptToCommit) { + // todo(szymon): check state. + attempt.abort(); + } + } + + final RpcResponse attemptRes = attemptToCommit.commit(); + + ctx.logBuilder().endResponseWithChild(attemptToCommit.ctx().log()); + final HttpRequest attemptReq = attemptToCommit.ctx().request(); + if (attemptReq != null) { + ctx.updateRequest(attemptReq); + } + resFuture.complete(attemptRes); + } + + @Override + public void abort(RpcRetryAttempt attempt) { + // Can be called in any state. + attempt.abort(); + } + + @Override + public void abort(Throwable cause) { + if (state == State.COMPLETED) { + return; + } + + assert state == State.INITIALIZED; + state = State.COMPLETED; + + for (final RpcRetryAttempt attempt : attemptsSoFar) { + // todo(szymon): check state. + attempt.abort(); + } + + // todo(szymon): verify that this safe to do so we can avoid isInitialAttempt check + if (!ctx.log().isRequestComplete()) { + ctx.logBuilder().endRequest(cause); + } + ctx.logBuilder().endResponse(cause); + + resFuture.completeExceptionally(cause); + } + + @Override + public RpcResponse res() { + return res; + } + + RetryConfig config() { + return retryConfig; + } + + private boolean setResponseTimeout() { + final long responseTimeoutMillis = responseTimeoutMillis(); + if (responseTimeoutMillis < 0) { + return false; + } else if (responseTimeoutMillis == 0) { + ctx.clearResponseTimeout(); + return true; + } else { + ctx.setResponseTimeoutMillis(TimeoutMode.SET_FROM_NOW, responseTimeoutMillis); + return true; + } + } + + private long responseTimeoutMillis() { + if (!hasDeadline) { + return retryConfig.responseTimeoutMillisForEachAttempt(); + } + + final long remaining = TimeUnit.NANOSECONDS.toMillis(deadlineTimeNanos - System.nanoTime()); + if (remaining <= 0) { + return -1; + } + if (retryConfig.responseTimeoutMillisForEachAttempt() > 0) { + return Math.min(retryConfig.responseTimeoutMillisForEachAttempt(), remaining); + } + return remaining; + } + + @Override + public String toString() { + return MoreObjects + .toStringHelper(this) + .add("ctx", ctx) + .add("retryConfig", retryConfig) + .add("req", req) + .add("res", res) + .add("deadlineTimeNanos", deadlineTimeNanos) + .add("hasDeadline", hasDeadline) + .add("lastBackoff", lastBackoff) + .add("numberAttemptsSoFar", numberAttemptsSoFar) + .add("numberAttemptsWithBackoffSoFar", numberAttemptsWithBackoffSoFar) + .toString(); + } +} diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientLoadBalancingTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientLoadBalancingTest.java index a2b52679da6..62d6a8fe43e 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientLoadBalancingTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientLoadBalancingTest.java @@ -16,6 +16,7 @@ package com.linecorp.armeria.client.retry; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; import static org.assertj.core.api.Assertions.assertThat; import java.net.InetSocketAddress; @@ -104,8 +105,10 @@ void test(TestMode mode) { status = null; } + final boolean isRetry = ctx.request().headers().contains(ARMERIA_RETRY_COUNT); + // Retry only once on failure. - if (!HttpStatus.OK.equals(status) && AbstractRetryingClient.getTotalAttempts(ctx) <= 1) { + if (!HttpStatus.OK.equals(status) && !isRetry) { return UnmodifiableFuture.completedFuture(RetryDecision.retry(Backoff.withoutDelay())); } else { return UnmodifiableFuture.completedFuture(RetryDecision.noRetry()); @@ -127,9 +130,9 @@ void test(TestMode mode) { case FAILURE: final List expectedPortsWhenRetried = ImmutableList.builder() - .addAll(expectedPorts) - .addAll(expectedPorts) - .build(); + .addAll(expectedPorts) + .addAll(expectedPorts) + .build(); assertThat(accessedPorts).isEqualTo(expectedPortsWhenRetried); break; } From 60c50bd6f86c08c6e728a20a7eb6b415a9b0feac Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 20 Aug 2025 12:42:33 +0100 Subject: [PATCH 16/42] refactor: extract counter bookkeeping into RetryCounter --- .../client/retry/HttpRetryingContext.java | 49 +++-------- .../armeria/client/retry/RetryCounter.java | 86 +++++++++++++++++++ .../client/retry/RpcRetryingContext.java | 44 ++-------- 3 files changed, 105 insertions(+), 74 deletions(-) create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java index 4438d5d8e32..4dc559435b4 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java @@ -70,16 +70,11 @@ private enum State { private final HttpRequest req; @Nullable private HttpRequestDuplicator reqDuplicator; - - private int numberAttemptsSoFar; + private final RetryCounter retryCounter; private final long deadlineTimeNanos; private final boolean hasDeadline; - @Nullable - private Backoff lastBackoff; - private int numberAttemptsWithBackoffSoFar; - private final boolean useRetryAfter; List attemptsSoFar; @@ -97,8 +92,7 @@ private enum State { this.res = res; this.req = req; reqDuplicator = null; // will be initialized in init(). - - numberAttemptsSoFar = 0; + retryCounter = new RetryCounter(retryConfig.maxTotalAttempts()); final long responseTimeoutMillis = ctx.responseTimeoutMillis(); if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { @@ -109,9 +103,6 @@ private enum State { hasDeadline = true; } - lastBackoff = null; - numberAttemptsWithBackoffSoFar = 0; - this.useRetryAfter = useRetryAfter; attemptsSoFar = new LinkedList<>(); } @@ -161,21 +152,9 @@ public HttpRetryAttempt executeAttempt(@Nullable Backoff backoff, assert state == State.INITIALIZED; assert reqDuplicator != null; - if (numberAttemptsSoFar >= retryConfig.maxTotalAttempts()) { - throw new IllegalStateException( - "Exceeded the maximum number of attempts: " + retryConfig.maxTotalAttempts()); - } - - final int attemptNumber = ++numberAttemptsSoFar; - - if (backoff != null) { - if (lastBackoff != backoff) { - lastBackoff = backoff; - numberAttemptsWithBackoffSoFar = 0; - } - numberAttemptsWithBackoffSoFar++; - } + retryCounter.recordAttemptWith(backoff); + final int attemptNumber = retryCounter.attemptNumber(); final boolean isInitialAttempt = attemptNumber <= 1; if (!setResponseTimeout()) { @@ -225,25 +204,21 @@ public long nextRetryTimeNanos(HttpRetryAttempt attempt, Backoff backoff) { return Long.MAX_VALUE; } - if (numberAttemptsSoFar >= retryConfig.maxTotalAttempts()) { + if (retryCounter.hasReachedMaxAttempts()) { logger.debug("Exceeded the default number of max attempt: {}", retryConfig.maxTotalAttempts()); return Long.MAX_VALUE; } - final int numberAttemptsWithThisBackoffSoFar; - if (lastBackoff != backoff) { - numberAttemptsWithThisBackoffSoFar = 1; - } else { - numberAttemptsWithThisBackoffSoFar = numberAttemptsWithBackoffSoFar; - } + final long nextRetryDelayForBackoffMillis = backoff.nextDelayMillis( + retryCounter.attemptNumberForBackoff(backoff) + 1); - long nextRetryDelayMillis = backoff.nextDelayMillis(numberAttemptsWithThisBackoffSoFar); - if (nextRetryDelayMillis < 0) { + if (nextRetryDelayForBackoffMillis < 0) { logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); return Long.MAX_VALUE; } - nextRetryDelayMillis = Math.max(nextRetryDelayMillis, useRetryAfter ? attempt.retryAfterMillis() : -1); + final long nextRetryDelayMillis = Math.max(nextRetryDelayForBackoffMillis, + useRetryAfter ? attempt.retryAfterMillis() : -1); final long nextDelayTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(nextRetryDelayMillis); if (hasDeadline && nextDelayTimeNanos > deadlineTimeNanos) { @@ -430,9 +405,7 @@ public String toString() { .add("res", res) .add("deadlineTimeNanos", deadlineTimeNanos) .add("hasDeadline", hasDeadline) - .add("lastBackoff", lastBackoff) - .add("numberAttemptsSoFar", numberAttemptsSoFar) - .add("numberAttemptsWithBackoffSoFar", numberAttemptsWithBackoffSoFar) + .add("retryCounter", retryCounter) .toString(); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java new file mode 100644 index 00000000000..7c301d8c31b --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java @@ -0,0 +1,86 @@ +/* + * Copyright 2025 LINE Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.linecorp.armeria.client.retry; + +import static java.util.Objects.requireNonNull; + +import com.google.common.base.MoreObjects; + +import com.linecorp.armeria.common.annotation.Nullable; + +final class RetryCounter { + private final int maxAttempts; + + private int numberAttemptsSoFar; + @Nullable + private Backoff lastBackoff; + private int numberAttemptsForLastBackoff; + + RetryCounter(int maxAttempts) { + if (maxAttempts <= 0) { + throw new IllegalArgumentException("maxAttempts: " + maxAttempts + " (expected: > 0)"); + } + this.maxAttempts = maxAttempts; + numberAttemptsSoFar = 0; + lastBackoff = null; + numberAttemptsForLastBackoff = 0; + } + + void recordAttemptWith(@Nullable Backoff backoff) { + if (hasReachedMaxAttempts()) { + throw new IllegalStateException("Exceeded the maximum number of attempts: " + maxAttempts); + } + + ++numberAttemptsSoFar; + + if (backoff != null) { + if (lastBackoff != backoff) { + lastBackoff = backoff; + numberAttemptsForLastBackoff = 0; + } + numberAttemptsForLastBackoff++; + } + } + + int attemptNumber() { + return numberAttemptsSoFar; + } + + int attemptNumberForBackoff(Backoff backoff) { + requireNonNull(backoff, "backoff"); + if (lastBackoff != backoff) { + return 0; + } else { + return numberAttemptsForLastBackoff; + } + } + + boolean hasReachedMaxAttempts() { + return numberAttemptsSoFar >= maxAttempts; + } + + @Override + public String toString() { + return MoreObjects + .toStringHelper(this) + .add("maxAttempts", maxAttempts) + .add("numberAttemptsSoFar", numberAttemptsSoFar) + .add("lastBackoff", lastBackoff) + .add("numberAttemptsWithThisBackoffSoFar", numberAttemptsForLastBackoff) + .toString(); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java index 74d53e9124d..82705c88662 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java @@ -63,16 +63,11 @@ private enum State { private final CompletableFuture resFuture; private final RpcResponse res; private final RpcRequest req; - - private int numberAttemptsSoFar; + private final RetryCounter retryCounter; private final long deadlineTimeNanos; private final boolean hasDeadline; - @Nullable - private Backoff lastBackoff; - private int numberAttemptsWithBackoffSoFar; - List attemptsSoFar; RpcRetryingContext(ClientRequestContext ctx, @@ -86,8 +81,7 @@ private enum State { this.resFuture = resFuture; this.res = res; this.req = req; - - numberAttemptsSoFar = 0; + retryCounter = new RetryCounter(retryConfig.maxTotalAttempts()); final long responseTimeoutMillis = ctx.responseTimeoutMillis(); if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { @@ -98,9 +92,6 @@ private enum State { hasDeadline = true; } - lastBackoff = null; - numberAttemptsWithBackoffSoFar = 0; - attemptsSoFar = new LinkedList<>(); } @@ -121,21 +112,9 @@ public RpcRetryAttempt executeAttempt(@Nullable Backoff backoff, assert state == State.INITIALIZED; - if (numberAttemptsSoFar >= retryConfig.maxTotalAttempts()) { - throw new IllegalStateException( - "Exceeded the maximum number of attempts: " + retryConfig.maxTotalAttempts()); - } - - final int attemptNumber = ++numberAttemptsSoFar; - - if (backoff != null) { - if (lastBackoff != backoff) { - lastBackoff = backoff; - numberAttemptsWithBackoffSoFar = 0; - } - numberAttemptsWithBackoffSoFar++; - } + retryCounter.recordAttemptWith(backoff); + final int attemptNumber = retryCounter.attemptNumber(); final boolean isInitialAttempt = attemptNumber <= 1; if (!setResponseTimeout()) { @@ -178,19 +157,14 @@ public long nextRetryTimeNanos(RpcRetryAttempt attempt, Backoff backoff) { return Long.MAX_VALUE; } - if (numberAttemptsSoFar >= retryConfig.maxTotalAttempts()) { + if (retryCounter.hasReachedMaxAttempts()) { logger.debug("Exceeded the default number of max attempt: {}", retryConfig.maxTotalAttempts()); return Long.MAX_VALUE; } - final int numberAttemptsWithThisBackoffSoFar; - if (lastBackoff != backoff) { - numberAttemptsWithThisBackoffSoFar = 1; - } else { - numberAttemptsWithThisBackoffSoFar = numberAttemptsWithBackoffSoFar; - } + final long nextRetryDelayMillis = backoff.nextDelayMillis( + retryCounter.attemptNumberForBackoff(backoff) + 1); - final long nextRetryDelayMillis = backoff.nextDelayMillis(numberAttemptsWithThisBackoffSoFar); if (nextRetryDelayMillis < 0) { logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); return Long.MAX_VALUE; @@ -333,9 +307,7 @@ public String toString() { .add("res", res) .add("deadlineTimeNanos", deadlineTimeNanos) .add("hasDeadline", hasDeadline) - .add("lastBackoff", lastBackoff) - .add("numberAttemptsSoFar", numberAttemptsSoFar) - .add("numberAttemptsWithBackoffSoFar", numberAttemptsWithBackoffSoFar) + .add("retryCounter", retryCounter) .toString(); } } From 111c71179ea4bfbac89f9007b2a8e2a43d58a1c4 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 20 Aug 2025 13:30:13 +0100 Subject: [PATCH 17/42] refactor: extract scheduling into RetryScheduler --- .../client/retry/HttpRetryingContext.java | 29 +-------- .../armeria/client/retry/RetryScheduler.java | 62 +++++++++++++++++++ .../client/retry/RpcRetryingContext.java | 27 +------- 3 files changed, 68 insertions(+), 50 deletions(-) create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java index 4dc559435b4..81623de11e5 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java @@ -33,7 +33,6 @@ import com.google.common.base.MoreObjects; import com.linecorp.armeria.client.Client; -import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.common.AggregationOptions; @@ -49,8 +48,6 @@ import com.linecorp.armeria.internal.client.ClientRequestContextExtension; import com.linecorp.armeria.internal.client.ClientUtil; -import io.netty.util.concurrent.ScheduledFuture; - final class HttpRetryingContext implements RetryingContext { private static final Logger logger = LoggerFactory.getLogger(HttpRetryingContext.class); @@ -71,6 +68,7 @@ private enum State { @Nullable private HttpRequestDuplicator reqDuplicator; private final RetryCounter retryCounter; + private final RetryScheduler scheduler; private final long deadlineTimeNanos; private final boolean hasDeadline; @@ -93,6 +91,7 @@ private enum State { this.req = req; reqDuplicator = null; // will be initialized in init(). retryCounter = new RetryCounter(retryConfig.maxTotalAttempts()); + scheduler = new RetryScheduler(ctx); final long responseTimeoutMillis = ctx.responseTimeoutMillis(); if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { @@ -232,29 +231,7 @@ public long nextRetryTimeNanos(HttpRetryAttempt attempt, Backoff backoff) { @Override public void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, Consumer exceptionHandler) { - try { - final long nextRetryDelayMillis = TimeUnit.NANOSECONDS.toMillis( - nextRetryTimeNanos - System.nanoTime()); - if (nextRetryDelayMillis <= 0) { - ctx.eventLoop().execute(retryTask); - } else { - @SuppressWarnings("unchecked") - final ScheduledFuture scheduledFuture = (ScheduledFuture) ctx - .eventLoop().schedule(retryTask, nextRetryDelayMillis, TimeUnit.MILLISECONDS); - scheduledFuture.addListener(future -> { - if (future.isCancelled()) { - // future is cancelled when the client factory is closed. - exceptionHandler.accept(new IllegalStateException( - ClientFactory.class.getSimpleName() + " has been closed.")); - } else if (future.cause() != null) { - // Other unexpected exceptions. - exceptionHandler.accept(future.cause()); - } - }); - } - } catch (Throwable t) { - exceptionHandler.accept(t); - } + scheduler.scheduleNextRetry(nextRetryTimeNanos, retryTask, exceptionHandler); } @Override diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java new file mode 100644 index 00000000000..9667d34d625 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -0,0 +1,62 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.client.retry; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.ClientRequestContext; + +import io.netty.util.concurrent.ScheduledFuture; + +/** + * Schedules retry tasks using the {@link ClientRequestContext}'s event loop. + */ +final class RetryScheduler { + private final ClientRequestContext ctx; + + RetryScheduler(ClientRequestContext ctx) { + this.ctx = ctx; + } + + void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, + Consumer exceptionHandler) { + try { + final long nextRetryDelayMillis = TimeUnit.NANOSECONDS.toMillis( + nextRetryTimeNanos - System.nanoTime()); + if (nextRetryDelayMillis <= 0) { + ctx.eventLoop().execute(retryTask); + } else { + @SuppressWarnings("unchecked") + final ScheduledFuture scheduledFuture = (ScheduledFuture) ctx + .eventLoop().schedule(retryTask, nextRetryDelayMillis, TimeUnit.MILLISECONDS); + scheduledFuture.addListener(future -> { + if (future.isCancelled()) { + // future is cancelled when the client factory is closed. + exceptionHandler.accept(new IllegalStateException( + ClientFactory.class.getSimpleName() + " has been closed.")); + } else if (future.cause() != null) { + // Other unexpected exceptions. + exceptionHandler.accept(future.cause()); + } + }); + } + } catch (Throwable t) { + exceptionHandler.accept(t); + } + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java index 82705c88662..2f890204b31 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java @@ -33,7 +33,6 @@ import com.google.common.base.MoreObjects; import com.linecorp.armeria.client.Client; -import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.client.endpoint.EndpointGroup; @@ -47,8 +46,6 @@ import com.linecorp.armeria.internal.client.ClientRequestContextExtension; import com.linecorp.armeria.internal.client.ClientUtil; -import io.netty.util.concurrent.ScheduledFuture; - final class RpcRetryingContext implements RetryingContext { private static final Logger logger = LoggerFactory.getLogger(RpcRetryingContext.class); @@ -64,6 +61,7 @@ private enum State { private final RpcResponse res; private final RpcRequest req; private final RetryCounter retryCounter; + private final RetryScheduler scheduler; private final long deadlineTimeNanos; private final boolean hasDeadline; @@ -82,6 +80,7 @@ private enum State { this.res = res; this.req = req; retryCounter = new RetryCounter(retryConfig.maxTotalAttempts()); + scheduler = new RetryScheduler(ctx); final long responseTimeoutMillis = ctx.responseTimeoutMillis(); if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { @@ -181,27 +180,7 @@ public long nextRetryTimeNanos(RpcRetryAttempt attempt, Backoff backoff) { @Override public void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, Consumer exceptionHandler) { - try { - final long nextRetryDelayMillis = TimeUnit.NANOSECONDS.toMillis( - nextRetryTimeNanos - System.nanoTime()); - if (nextRetryDelayMillis <= 0) { - ctx.eventLoop().execute(retryTask); - } else { - @SuppressWarnings("unchecked") - final ScheduledFuture scheduledFuture = (ScheduledFuture) ctx - .eventLoop().schedule(retryTask, nextRetryDelayMillis, TimeUnit.MILLISECONDS); - scheduledFuture.addListener(future -> { - if (future.isCancelled()) { - exceptionHandler.accept(new IllegalStateException( - ClientFactory.class.getSimpleName() + " has been closed.")); - } else if (future.cause() != null) { - exceptionHandler.accept(future.cause()); - } - }); - } - } catch (Throwable t) { - exceptionHandler.accept(t); - } + scheduler.scheduleNextRetry(nextRetryTimeNanos, retryTask, exceptionHandler); } @Override From 280aea47a68b5f7cf6b443411010f73a929b8ed4 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Wed, 20 Aug 2025 14:29:33 +0100 Subject: [PATCH 18/42] refactor: move res and req listeners to init function of the RetryingContexts --- .../client/retry/HttpRetryingContext.java | 70 +++++++------------ .../client/retry/RpcRetryingContext.java | 18 +++-- .../client/retry/RetryingClientTest.java | 47 +++++++++---- 3 files changed, 74 insertions(+), 61 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java index 81623de11e5..6f5c84727b6 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java @@ -55,7 +55,6 @@ private enum State { UNINITIALIZED, INITIALIZING, INITIALIZED, - COMPLETING, COMPLETED } @@ -137,6 +136,31 @@ public CompletableFuture init() { }); } + initFuture.whenComplete((initSuccessful, initCause) -> { + assert state == State.INITIALIZED || state == State.COMPLETED; + if (!initSuccessful || initCause != null) { + return; + } + + req.whenComplete().handle((unused, cause) -> { + if (cause != null) { + abort(cause); + } + return null; + }); + + res.whenComplete().handle((result, cause) -> { + final Throwable abortCause; + if (cause != null) { + abortCause = cause; + } else { + abortCause = AbortedStreamException.get(); + } + abort(abortCause); + return null; + }); + }); + return initFuture; } @@ -144,10 +168,6 @@ public CompletableFuture init() { @Nullable public HttpRetryAttempt executeAttempt(@Nullable Backoff backoff, Client delegate) { - if (isCompleted()) { - return null; - } - assert state == State.INITIALIZED; assert reqDuplicator != null; @@ -261,7 +281,7 @@ public void commit(HttpRetryAttempt attemptToCommit) { @Override public void abort(HttpRetryAttempt attempt) { - assert state == State.INITIALIZED || state == State.COMPLETING || state == State.COMPLETED; + assert state == State.INITIALIZED || state == State.COMPLETED; attempt.abort(); } @@ -272,7 +292,7 @@ public void abort(Throwable cause) { return; } - assert state == State.INITIALIZED || state == State.COMPLETING; + assert state == State.INITIALIZED; assert reqDuplicator != null; state = State.COMPLETED; @@ -292,42 +312,6 @@ public void abort(Throwable cause) { resFuture.completeExceptionally(cause); } - private boolean isCompleted() { - if (state == State.COMPLETING || state == State.COMPLETED) { - return true; - } - - assert state == State.INITIALIZED; - - // The request or response has been aborted by the client before it receives a response, - // so stop retrying. - if (req.whenComplete().isCompletedExceptionally()) { - state = State.COMPLETING; - req.whenComplete().handle((unused, cause) -> { - abort(cause); - return null; - }); - return true; - } - - if (res.isComplete()) { - state = State.COMPLETING; - res.whenComplete().handle((result, cause) -> { - final Throwable abortCause; - if (cause != null) { - abortCause = cause; - } else { - abortCause = AbortedStreamException.get(); - } - abort(abortCause); - return null; - }); - return true; - } - - return false; - } - @Override public HttpResponse res() { return res; diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java index 2f890204b31..56512c3c8c0 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java @@ -40,6 +40,7 @@ import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.stream.AbortedStreamException; import com.linecorp.armeria.common.util.TimeoutMode; import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; @@ -97,7 +98,18 @@ private enum State { @Override public CompletableFuture init() { assert state == State.INITIALIZED; - // Nothing special to initialize for RPC retrying. + + res.whenComplete().handle((result, cause) -> { + final Throwable abortCause; + if (cause != null) { + abortCause = cause; + } else { + abortCause = AbortedStreamException.get(); + } + abort(abortCause); + return null; + }); + return UnmodifiableFuture.completedFuture(true); } @@ -105,10 +117,6 @@ public CompletableFuture init() { @Nullable public RpcRetryAttempt executeAttempt(@Nullable Backoff backoff, Client delegate) { - if (res.isDone()) { - return null; - } - assert state == State.INITIALIZED; retryCounter.recordAttemptWith(backoff); diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java index c68446ad7e3..210dfec74de 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetryingClientTest.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; +import static org.awaitility.Awaitility.await; import java.time.Duration; import java.util.Arrays; @@ -55,6 +56,8 @@ import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.client.ClientRequestContextCaptor; +import com.linecorp.armeria.client.Clients; import com.linecorp.armeria.client.HttpClient; import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.client.UnprocessedRequestException; @@ -526,15 +529,15 @@ void honorRetryMapping() { void evaluatesMappingOnce() { final AtomicInteger evaluations = new AtomicInteger(0); final RetryConfigMapping mapping = - (ctx, req) -> { - evaluations.incrementAndGet(); - return RetryConfig - .builder0(RetryRule.builder() - .onStatus(HttpStatus.valueOf(500)) - .thenBackoff()) - .maxTotalAttempts(2) - .build(); - }; + (ctx, req) -> { + evaluations.incrementAndGet(); + return RetryConfig + .builder0(RetryRule.builder() + .onStatus(HttpStatus.valueOf(500)) + .thenBackoff()) + .maxTotalAttempts(2) + .build(); + }; final WebClient client = client(mapping); @@ -712,11 +715,24 @@ void doNotRetryWhenResponseIsAborted() throws Exception { @Test void doNotRetryWhenSubscriberIsCancelled() throws Exception { final WebClient client = client(retryAlways); - client.get("/subscriber-cancel").subscribe( + final AtomicInteger subscriberCancelServiceCallCounterWhenCancelled = new AtomicInteger(-1); + + final HttpResponse res; + final ClientRequestContext ctx; + try (ClientRequestContextCaptor captor = Clients.newContextCaptor()) { + res = client.get("/subscriber-cancel"); + ctx = captor.get(); + } + + res.subscribe( new Subscriber() { @Override public void onSubscribe(Subscription s) { - s.cancel(); // Cancel as soon as getting the subscription. + ctx.eventLoop().schedule(() -> { + subscriberCancelServiceCallCounterWhenCancelled.set( + subscriberCancelServiceCallCounter.get()); + s.cancel(); + }, 1000, TimeUnit.MILLISECONDS); } @Override @@ -729,8 +745,13 @@ public void onError(Throwable t) {} public void onComplete() {} }); - TimeUnit.SECONDS.sleep(1L); // Sleep to check if there's a retry. - assertThat(subscriberCancelServiceCallCounter.get()).isEqualTo(1); + await().untilAsserted(() -> { + // We should have made at least one service call during the second. + assertThat(subscriberCancelServiceCallCounter.get()).isGreaterThanOrEqualTo(1); + // After we cancelled we do not expect further service calls. + assertThat(subscriberCancelServiceCallCounterWhenCancelled.get()).isEqualTo( + subscriberCancelServiceCallCounter.get()); + }); } @Test From f1591feccc71beeca70ada0e8f73961186829b04 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 21 Aug 2025 08:13:20 +0100 Subject: [PATCH 19/42] refactor: rename numberAttemptsWithThisBackoffSoFar to numberAttemptsSoFarForLastBackoff in RetryCounter --- .../linecorp/armeria/client/retry/RetryCounter.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java index 7c301d8c31b..3a481ff6821 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java @@ -28,7 +28,7 @@ final class RetryCounter { private int numberAttemptsSoFar; @Nullable private Backoff lastBackoff; - private int numberAttemptsForLastBackoff; + private int numberAttemptsSoFarForLastBackoff; RetryCounter(int maxAttempts) { if (maxAttempts <= 0) { @@ -37,7 +37,7 @@ final class RetryCounter { this.maxAttempts = maxAttempts; numberAttemptsSoFar = 0; lastBackoff = null; - numberAttemptsForLastBackoff = 0; + numberAttemptsSoFarForLastBackoff = 0; } void recordAttemptWith(@Nullable Backoff backoff) { @@ -50,9 +50,9 @@ void recordAttemptWith(@Nullable Backoff backoff) { if (backoff != null) { if (lastBackoff != backoff) { lastBackoff = backoff; - numberAttemptsForLastBackoff = 0; + numberAttemptsSoFarForLastBackoff = 0; } - numberAttemptsForLastBackoff++; + numberAttemptsSoFarForLastBackoff++; } } @@ -65,7 +65,7 @@ int attemptNumberForBackoff(Backoff backoff) { if (lastBackoff != backoff) { return 0; } else { - return numberAttemptsForLastBackoff; + return numberAttemptsSoFarForLastBackoff; } } @@ -80,7 +80,7 @@ public String toString() { .add("maxAttempts", maxAttempts) .add("numberAttemptsSoFar", numberAttemptsSoFar) .add("lastBackoff", lastBackoff) - .add("numberAttemptsWithThisBackoffSoFar", numberAttemptsForLastBackoff) + .add("numberAttemptsSoFarForLastBackoff", numberAttemptsSoFarForLastBackoff) .toString(); } } From 2d2e65bc873cf9a964f457493321948e52c15382 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 22 Aug 2025 09:45:12 +0100 Subject: [PATCH 20/42] refactor: refactor scheduling and counting out of RetryingContext and hide RetryAttempt from AbstractRetryingClient --- .../client/retry/AbstractRetryingClient.java | 67 ++++----- .../client/retry/HttpRetryAttempt.java | 17 ++- .../client/retry/HttpRetryingContext.java | 131 +++++++++-------- .../armeria/client/retry/RetryAttempt.java | 26 ---- .../armeria/client/retry/RetryCounter.java | 2 + .../armeria/client/retry/RetryScheduler.java | 2 + .../armeria/client/retry/RetryingClient.java | 2 +- .../armeria/client/retry/RetryingContext.java | 11 +- .../client/retry/RetryingRpcClient.java | 2 +- .../armeria/client/retry/RpcRetryAttempt.java | 13 +- .../client/retry/RpcRetryingContext.java | 133 +++++++++--------- 11 files changed, 192 insertions(+), 214 deletions(-) delete mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 6cc78f5210f..36528a03c2b 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -37,7 +37,7 @@ * @param the {@link Response} type */ public abstract class AbstractRetryingClient - , R extends RetryingContext> + > extends SimpleDecoratingClient { private static final Logger logger = LoggerFactory.getLogger(AbstractRetryingClient.class); @@ -88,42 +88,33 @@ public final O execute(ClientRequestContext ctx, I req) throws Exception { abstract R getRetryingContext(ClientRequestContext ctx, RetryConfig config, I req); private void executeAttempt(@Nullable Backoff lastBackoff, R rctx) { - @Nullable - final A attempt = rctx.executeAttempt(lastBackoff, unwrap()); - - if (attempt == null) { - // Retrying completed already, error handling - // is done inside `RetryingContext`. - return; - } - - attempt.whenDecided() - .handle((decision, decisionCause) -> { - if (decisionCause != null) { - rctx.abort(decisionCause); - return null; - } - - final Backoff backoff = decision != null ? decision.backoff() : null; - final long nextRetryTimeNanos; - if (backoff != null) { - nextRetryTimeNanos = rctx.nextRetryTimeNanos(attempt, backoff); - } else { - nextRetryTimeNanos = Long.MAX_VALUE; - } - - if (nextRetryTimeNanos < Long.MAX_VALUE) { - rctx.abort(attempt); - rctx.scheduleNextRetry( - nextRetryTimeNanos, - () -> executeAttempt(backoff, rctx), - rctx::abort - ); - } else { - rctx.commit(attempt); - } - - return null; - }); + rctx.executeAttempt(lastBackoff, unwrap()) + .handle((decision, decisionCause) -> { + if (decisionCause != null) { + rctx.abort(decisionCause); + return null; + } + + final Backoff backoff = decision != null ? decision.backoff() : null; + final long nextRetryTimeNanos; + if (backoff != null) { + nextRetryTimeNanos = rctx.nextRetryTimeNanos(backoff); + } else { + nextRetryTimeNanos = Long.MAX_VALUE; + } + + if (nextRetryTimeNanos < Long.MAX_VALUE) { + rctx.abortAttempt(); + rctx.scheduleNextRetry( + nextRetryTimeNanos, + () -> executeAttempt(backoff, rctx), + rctx::abort + ); + } else { + rctx.commit(); + } + + return null; + }); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java index d92bfff909d..34a290be2bd 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java @@ -16,6 +16,8 @@ package com.linecorp.armeria.client.retry; +import static com.google.common.base.Preconditions.checkState; + import java.time.Duration; import java.util.Date; import java.util.concurrent.CompletableFuture; @@ -45,7 +47,7 @@ import io.netty.handler.codec.DateFormatter; -class HttpRetryAttempt implements RetryAttempt { +final class HttpRetryAttempt { private static final Logger logger = LoggerFactory.getLogger(HttpRetryAttempt.class); enum State { @@ -123,7 +125,7 @@ public HttpResponse commit() { return res; } - assert state == State.DECIDED; + checkState(state == State.DECIDED); state = State.COMMITTED; if (resWithContent != null) { @@ -138,11 +140,11 @@ public void abort() { } public void abort(Throwable cause) { - if (state == State.ABORTED || state == State.COMMITTED) { + if (state == State.ABORTED) { return; } - assert state == State.EXECUTING || state == State.DECIDED; + checkState(state == State.EXECUTING || state == State.DECIDING || state == State.DECIDED); state = State.ABORTED; if (resWithContent != null) { @@ -158,8 +160,7 @@ public void abort(Throwable cause) { whenDecidedFuture.completeExceptionally(cause); } - @Override - public CompletableFuture<@Nullable RetryDecision> whenDecided() { + CompletableFuture<@Nullable RetryDecision> whenDecided() { return whenDecidedFuture; } @@ -209,8 +210,6 @@ private void handleAggRes() { assert state == State.EXECUTING; res.aggregate().handle((aggRes, resCause) -> { - assert state == State.EXECUTING; - if (resCause != null) { ctx.logBuilder().endRequest(resCause); ctx.logBuilder().endResponse(resCause); @@ -375,10 +374,10 @@ private void complete(HttpResponse res, @Nullable Throwable resCause) { @Override public String toString() { + // We omit `rctx` to keep the representation compact. return MoreObjects .toStringHelper(this) .add("state", state) - .add("rctx", rctx) .add("ctx", ctx) .add("res", res) .add("resCause", resCause) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java index 6f5c84727b6..49b254da644 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java @@ -21,8 +21,6 @@ import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; -import java.util.LinkedList; -import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -43,40 +41,43 @@ import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.stream.AbortedStreamException; import com.linecorp.armeria.common.util.TimeoutMode; +import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; import com.linecorp.armeria.internal.client.ClientRequestContextExtension; import com.linecorp.armeria.internal.client.ClientUtil; -final class HttpRetryingContext implements RetryingContext { +final class HttpRetryingContext implements RetryingContext { private static final Logger logger = LoggerFactory.getLogger(HttpRetryingContext.class); private enum State { UNINITIALIZED, INITIALIZING, INITIALIZED, + EXECUTING, COMPLETED } private State state; private final ClientRequestContext ctx; - private final RetryConfig retryConfig; + private final RetryConfig config; private final HttpResponse res; private final CompletableFuture resFuture; private final HttpRequest req; @Nullable private HttpRequestDuplicator reqDuplicator; - private final RetryCounter retryCounter; + private final RetryCounter counter; private final RetryScheduler scheduler; private final long deadlineTimeNanos; private final boolean hasDeadline; private final boolean useRetryAfter; - List attemptsSoFar; + @Nullable + HttpRetryAttempt currentAttempt; HttpRetryingContext(ClientRequestContext ctx, - RetryConfig retryConfig, + RetryConfig config, HttpResponse res, CompletableFuture resFuture, HttpRequest req, @@ -84,12 +85,12 @@ private enum State { state = State.UNINITIALIZED; this.ctx = ctx; - this.retryConfig = retryConfig; + this.config = config; this.resFuture = resFuture; this.res = res; this.req = req; reqDuplicator = null; // will be initialized in init(). - retryCounter = new RetryCounter(retryConfig.maxTotalAttempts()); + counter = new RetryCounter(config.maxTotalAttempts()); scheduler = new RetryScheduler(ctx); final long responseTimeoutMillis = ctx.responseTimeoutMillis(); @@ -102,12 +103,12 @@ private enum State { } this.useRetryAfter = useRetryAfter; - attemptsSoFar = new LinkedList<>(); + currentAttempt = null; } @Override public CompletableFuture init() { - assert state == State.UNINITIALIZED; + checkState(state == State.UNINITIALIZED); state = State.INITIALIZING; final CompletableFuture initFuture = new CompletableFuture<>(); @@ -165,21 +166,27 @@ public CompletableFuture init() { } @Override - @Nullable - public HttpRetryAttempt executeAttempt(@Nullable Backoff backoff, - Client delegate) { - assert state == State.INITIALIZED; + public CompletableFuture<@Nullable RetryDecision> executeAttempt( + @Nullable Backoff backoff, + Client delegate) { + checkState(state == State.INITIALIZED); assert reqDuplicator != null; - - retryCounter.recordAttemptWith(backoff); - - final int attemptNumber = retryCounter.attemptNumber(); - final boolean isInitialAttempt = attemptNumber <= 1; + // We are not supporting concurrent attempts (yet). + // As such, we expect the previous attempt (if any) to be aborted before + // (abortion sets this field to null). + // assert state != State.EXECUTING; + assert currentAttempt == null; if (!setResponseTimeout()) { - throw ResponseTimeoutException.get(); + return UnmodifiableFuture.exceptionallyCompletedFuture(ResponseTimeoutException.get()); } + state = State.EXECUTING; + counter.recordAttemptWith(backoff); + + final int attemptNumber = counter.attemptNumber(); + final boolean isInitialAttempt = attemptNumber <= 1; + final HttpRequest attemptReq; final ClientRequestContext attemptCtx; if (isInitialAttempt) { @@ -212,24 +219,24 @@ public HttpRetryAttempt executeAttempt(@Nullable Backoff backoff, final HttpRetryAttempt attempt = new HttpRetryAttempt(this, attemptCtx, attemptRes, ctx.exchangeType().isResponseStreaming()); - attemptsSoFar.add(attempt); - return attempt; + currentAttempt = attempt; + return attempt.whenDecided(); } // returns Long.MAX_VALUE if no retry is possible. @Override - public long nextRetryTimeNanos(HttpRetryAttempt attempt, Backoff backoff) { - if (state != State.INITIALIZED) { - return Long.MAX_VALUE; - } + public long nextRetryTimeNanos(Backoff backoff) { + checkState(state == State.EXECUTING); + assert currentAttempt != null; + checkState(currentAttempt.state() == HttpRetryAttempt.State.DECIDED); - if (retryCounter.hasReachedMaxAttempts()) { - logger.debug("Exceeded the default number of max attempt: {}", retryConfig.maxTotalAttempts()); + if (counter.hasReachedMaxAttempts()) { + logger.debug("Exceeded the default number of max attempt: {}", config.maxTotalAttempts()); return Long.MAX_VALUE; } final long nextRetryDelayForBackoffMillis = backoff.nextDelayMillis( - retryCounter.attemptNumberForBackoff(backoff) + 1); + counter.attemptNumberForBackoff(backoff) + 1); if (nextRetryDelayForBackoffMillis < 0) { logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); @@ -237,7 +244,7 @@ public long nextRetryTimeNanos(HttpRetryAttempt attempt, Backoff backoff) { } final long nextRetryDelayMillis = Math.max(nextRetryDelayForBackoffMillis, - useRetryAfter ? attempt.retryAfterMillis() : -1); + useRetryAfter ? currentAttempt.retryAfterMillis() : -1); final long nextDelayTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(nextRetryDelayMillis); if (hasDeadline && nextDelayTimeNanos > deadlineTimeNanos) { @@ -251,38 +258,38 @@ public long nextRetryTimeNanos(HttpRetryAttempt attempt, Backoff backoff) { @Override public void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, Consumer exceptionHandler) { + checkState(state == State.INITIALIZED); + assert currentAttempt == null; scheduler.scheduleNextRetry(nextRetryTimeNanos, retryTask, exceptionHandler); } @Override - public void commit(HttpRetryAttempt attemptToCommit) { + public void commit() { if (state == State.COMPLETED) { // Already completed. return; } - checkState(attemptToCommit.state() == HttpRetryAttempt.State.DECIDED); - assert state == State.INITIALIZED; + checkState(state == State.EXECUTING); assert reqDuplicator != null; - state = State.COMPLETED; + assert currentAttempt != null; + checkState(currentAttempt.state() == HttpRetryAttempt.State.DECIDED); - for (final HttpRetryAttempt attempt : attemptsSoFar) { - if (attempt != attemptToCommit) { - // todo(szymon): check state. - attempt.abort(); - } - } + state = State.COMPLETED; - final HttpResponse attemptRes = attemptToCommit.commit(); + final HttpResponse attemptRes = currentAttempt.commit(); // todo(szymon): replace with endResponseWithChild - ctx.logBuilder().endResponseWithChild(attemptToCommit.ctx().log()); + ctx.logBuilder().endResponseWithChild(currentAttempt.ctx().log()); resFuture.complete(attemptRes); reqDuplicator.close(); } @Override - public void abort(HttpRetryAttempt attempt) { - assert state == State.INITIALIZED || state == State.COMPLETED; - attempt.abort(); + public void abortAttempt() { + checkState(state == State.EXECUTING); + checkState(currentAttempt != null, "No active attempt to abort"); + currentAttempt.abort(); + state = State.INITIALIZED; + currentAttempt = null; } @Override @@ -292,18 +299,17 @@ public void abort(Throwable cause) { return; } - assert state == State.INITIALIZED; - assert reqDuplicator != null; + if (state == State.EXECUTING) { + assert currentAttempt != null; + currentAttempt.abort(); + } + state = State.COMPLETED; - for (final HttpRetryAttempt attempt : attemptsSoFar) { - // todo(szymon): check state. - attempt.abort(); + if (reqDuplicator != null) { + reqDuplicator.abort(cause); } - reqDuplicator.abort(cause); - - // todo(szymon): verify that this safe to do so we can avoid isInitialAttempt check if (!ctx.log().isRequestComplete()) { ctx.logBuilder().endRequest(cause); } @@ -318,10 +324,10 @@ public HttpResponse res() { } RetryConfig config() { - return retryConfig; + return config; } - public boolean setResponseTimeout() { + private boolean setResponseTimeout() { final long responseTimeoutMillis = responseTimeoutMillis(); if (responseTimeoutMillis < 0) { return false; @@ -336,7 +342,7 @@ public boolean setResponseTimeout() { private long responseTimeoutMillis() { if (!hasDeadline) { - return retryConfig.responseTimeoutMillisForEachAttempt(); + return config.responseTimeoutMillisForEachAttempt(); } final long remainingTimeUntilDeadlineMillis = TimeUnit.NANOSECONDS.toMillis( @@ -347,8 +353,8 @@ private long responseTimeoutMillis() { return -1; } - if (retryConfig.responseTimeoutMillisForEachAttempt() > 0) { - return Math.min(retryConfig.responseTimeoutMillisForEachAttempt(), + if (config.responseTimeoutMillisForEachAttempt() > 0) { + return Math.min(config.responseTimeoutMillisForEachAttempt(), remainingTimeUntilDeadlineMillis); } @@ -361,12 +367,15 @@ public String toString() { .toStringHelper(this) .add("state", state) .add("ctx", ctx) - .add("retryConfig", retryConfig) + .add("config", config) .add("req", req) .add("res", res) + .add("useRetryAfter", useRetryAfter) .add("deadlineTimeNanos", deadlineTimeNanos) .add("hasDeadline", hasDeadline) - .add("retryCounter", retryCounter) + .add("counter", counter) + .add("scheduler", scheduler) + .add("currentAttempt", currentAttempt) .toString(); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java deleted file mode 100644 index ac290e9db60..00000000000 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryAttempt.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2025 LY Corporation - * - * LY Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.client.retry; - -import java.util.concurrent.CompletableFuture; - -import com.linecorp.armeria.common.Response; -import com.linecorp.armeria.common.annotation.Nullable; - -interface RetryAttempt { - CompletableFuture<@Nullable RetryDecision> whenDecided(); -} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java index 3a481ff6821..3ca567f5232 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java @@ -53,6 +53,8 @@ void recordAttemptWith(@Nullable Backoff backoff) { numberAttemptsSoFarForLastBackoff = 0; } numberAttemptsSoFarForLastBackoff++; + } else { + assert lastBackoff == null; } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index 9667d34d625..e69e99b69fc 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -59,4 +59,6 @@ void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, exceptionHandler.accept(t); } } + + // TODO: toString() method with info about current retry task. } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index b083184afed..35864c9d303 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -32,7 +32,7 @@ * An {@link HttpClient} decorator that handles failures of an invocation and retries HTTP requests. */ public final class RetryingClient - extends AbstractRetryingClient + extends AbstractRetryingClient implements HttpClient { /** * Returns a new {@link RetryingClientBuilder} with the specified {@link RetryConfig}. diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java index 8edf43a9548..adf87e60a35 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java @@ -24,20 +24,19 @@ import com.linecorp.armeria.common.Response; import com.linecorp.armeria.common.annotation.Nullable; -interface RetryingContext> { +interface RetryingContext { CompletableFuture init(); - @Nullable - A executeAttempt(@Nullable Backoff backoff, Client delegate); + CompletableFuture<@Nullable RetryDecision> executeAttempt(@Nullable Backoff backoff, Client delegate); - long nextRetryTimeNanos(A attempt, Backoff backoff); + long nextRetryTimeNanos(Backoff backoff); void scheduleNextRetry(long retryTimeNanos, Runnable retryTask, Consumer exceptionHandler); - void commit(A attempt); + void commit(); - void abort(A attempt); + void abortAttempt(); void abort(Throwable cause); diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 65eae07a683..050c577ef72 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -28,7 +28,7 @@ * An {@link RpcClient} decorator that handles failures of an invocation and retries RPC requests. */ public final class RetryingRpcClient - extends AbstractRetryingClient + extends AbstractRetryingClient implements RpcClient { /** diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java index 30605dd733d..2a2f5f0789d 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java @@ -16,6 +16,8 @@ package com.linecorp.armeria.client.retry; +import static com.google.common.base.Preconditions.checkState; + import java.util.concurrent.CompletableFuture; import com.google.common.base.MoreObjects; @@ -24,7 +26,7 @@ import com.linecorp.armeria.common.RpcResponse; import com.linecorp.armeria.common.annotation.Nullable; -final class RpcRetryAttempt implements RetryAttempt { +final class RpcRetryAttempt { // todo(szymon): doc enum State { EXECUTING, @@ -92,7 +94,7 @@ RpcResponse commit() { return res; } - assert state == State.DECIDED; + checkState(state == State.DECIDED); state = State.COMMITTED; return res; } @@ -102,12 +104,11 @@ void abort() { return; } - assert state == State.DECIDED; + checkState(state == State.EXECUTING || state == State.DECIDING || state == State.DECIDED); state = State.ABORTED; } - @Override - public CompletableFuture<@Nullable RetryDecision> whenDecided() { + CompletableFuture<@Nullable RetryDecision> whenDecided() { return whenDecidedFuture; } @@ -121,10 +122,10 @@ State state() { @Override public String toString() { + // We omit `rctx` to keep the representation compact. return MoreObjects .toStringHelper(this) .add("state", state) - .add("rctx", rctx) .add("ctx", ctx) .add("res", res) .add("resCause", resCause) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java index 56512c3c8c0..afaf8e719c0 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java @@ -21,8 +21,6 @@ import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; -import java.util.LinkedList; -import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -47,40 +45,42 @@ import com.linecorp.armeria.internal.client.ClientRequestContextExtension; import com.linecorp.armeria.internal.client.ClientUtil; -final class RpcRetryingContext implements RetryingContext { +final class RpcRetryingContext implements RetryingContext { private static final Logger logger = LoggerFactory.getLogger(RpcRetryingContext.class); private enum State { INITIALIZED, + EXECUTING, COMPLETED } private State state; private final ClientRequestContext ctx; - private final RetryConfig retryConfig; + private final RetryConfig config; private final CompletableFuture resFuture; private final RpcResponse res; private final RpcRequest req; - private final RetryCounter retryCounter; + private final RetryCounter counter; private final RetryScheduler scheduler; private final long deadlineTimeNanos; private final boolean hasDeadline; - List attemptsSoFar; + @Nullable + RpcRetryAttempt currentAttempt; RpcRetryingContext(ClientRequestContext ctx, - RetryConfig retryConfig, + RetryConfig config, CompletableFuture resFuture, RpcResponse res, RpcRequest req) { state = State.INITIALIZED; this.ctx = ctx; - this.retryConfig = retryConfig; + this.config = config; this.resFuture = resFuture; this.res = res; this.req = req; - retryCounter = new RetryCounter(retryConfig.maxTotalAttempts()); + counter = new RetryCounter(config.maxTotalAttempts()); scheduler = new RetryScheduler(ctx); final long responseTimeoutMillis = ctx.responseTimeoutMillis(); @@ -92,12 +92,12 @@ private enum State { hasDeadline = true; } - attemptsSoFar = new LinkedList<>(); + currentAttempt = null; } @Override public CompletableFuture init() { - assert state == State.INITIALIZED; + checkState(state == State.INITIALIZED); res.whenComplete().handle((result, cause) -> { final Throwable abortCause; @@ -114,20 +114,22 @@ public CompletableFuture init() { } @Override - @Nullable - public RpcRetryAttempt executeAttempt(@Nullable Backoff backoff, - Client delegate) { - assert state == State.INITIALIZED; - - retryCounter.recordAttemptWith(backoff); - - final int attemptNumber = retryCounter.attemptNumber(); - final boolean isInitialAttempt = attemptNumber <= 1; + public CompletableFuture<@Nullable RetryDecision> executeAttempt(@Nullable Backoff backoff, + Client delegate) { + checkState(state == State.INITIALIZED); + assert currentAttempt == null; if (!setResponseTimeout()) { - throw ResponseTimeoutException.get(); + return UnmodifiableFuture.exceptionallyCompletedFuture(ResponseTimeoutException.get()); } + state = State.EXECUTING; + + counter.recordAttemptWith(backoff); + + final int attemptNumber = counter.attemptNumber(); + final boolean isInitialAttempt = attemptNumber <= 1; + final ClientRequestContext attemptCtx = ClientUtil.newDerivedContext(ctx, null, req, isInitialAttempt); if (!isInitialAttempt) { @@ -154,23 +156,23 @@ public RpcRetryAttempt executeAttempt(@Nullable Backoff backoff, } final RpcRetryAttempt attempt = new RpcRetryAttempt(this, attemptCtx, attemptRes); - attemptsSoFar.add(attempt); - return attempt; + currentAttempt = attempt; + return attempt.whenDecided(); } @Override - public long nextRetryTimeNanos(RpcRetryAttempt attempt, Backoff backoff) { - if (state != State.INITIALIZED) { + public long nextRetryTimeNanos(Backoff backoff) { + if (state != State.EXECUTING) { return Long.MAX_VALUE; } - if (retryCounter.hasReachedMaxAttempts()) { - logger.debug("Exceeded the default number of max attempt: {}", retryConfig.maxTotalAttempts()); + if (counter.hasReachedMaxAttempts()) { + logger.debug("Exceeded the default number of max attempt: {}", config.maxTotalAttempts()); return Long.MAX_VALUE; } final long nextRetryDelayMillis = backoff.nextDelayMillis( - retryCounter.attemptNumberForBackoff(backoff) + 1); + counter.attemptNumberForBackoff(backoff) + 1); if (nextRetryDelayMillis < 0) { logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); @@ -188,30 +190,26 @@ public long nextRetryTimeNanos(RpcRetryAttempt attempt, Backoff backoff) { @Override public void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, Consumer exceptionHandler) { + checkState(state == State.INITIALIZED); + assert currentAttempt == null; scheduler.scheduleNextRetry(nextRetryTimeNanos, retryTask, exceptionHandler); } @Override - public void commit(RpcRetryAttempt attemptToCommit) { + public void commit() { if (state == State.COMPLETED) { // Already completed, so just return. return; } - checkState(attemptToCommit.state() == RpcRetryAttempt.State.DECIDED); - assert state == State.INITIALIZED; - state = State.COMPLETED; + checkState(state == State.EXECUTING); + assert currentAttempt != null; + checkState(currentAttempt.state() == RpcRetryAttempt.State.DECIDED); - for (final RpcRetryAttempt attempt : attemptsSoFar) { - if (attempt != attemptToCommit) { - // todo(szymon): check state. - attempt.abort(); - } - } - - final RpcResponse attemptRes = attemptToCommit.commit(); + state = State.COMPLETED; - ctx.logBuilder().endResponseWithChild(attemptToCommit.ctx().log()); - final HttpRequest attemptReq = attemptToCommit.ctx().request(); + final RpcResponse attemptRes = currentAttempt.commit(); + ctx.logBuilder().endResponseWithChild(currentAttempt.ctx().log()); + final HttpRequest attemptReq = currentAttempt.ctx().request(); if (attemptReq != null) { ctx.updateRequest(attemptReq); } @@ -219,9 +217,13 @@ public void commit(RpcRetryAttempt attemptToCommit) { } @Override - public void abort(RpcRetryAttempt attempt) { + public void abortAttempt() { + checkState(state == State.EXECUTING); + assert currentAttempt != null; // Can be called in any state. - attempt.abort(); + currentAttempt.abort(); + currentAttempt = null; + state = State.INITIALIZED; } @Override @@ -230,32 +232,20 @@ public void abort(Throwable cause) { return; } - assert state == State.INITIALIZED; - state = State.COMPLETED; - - for (final RpcRetryAttempt attempt : attemptsSoFar) { - // todo(szymon): check state. - attempt.abort(); + if (state == State.EXECUTING) { + assert currentAttempt != null; + currentAttempt.abort(); } - // todo(szymon): verify that this safe to do so we can avoid isInitialAttempt check + state = State.COMPLETED; + if (!ctx.log().isRequestComplete()) { ctx.logBuilder().endRequest(cause); } ctx.logBuilder().endResponse(cause); - resFuture.completeExceptionally(cause); } - @Override - public RpcResponse res() { - return res; - } - - RetryConfig config() { - return retryConfig; - } - private boolean setResponseTimeout() { final long responseTimeoutMillis = responseTimeoutMillis(); if (responseTimeoutMillis < 0) { @@ -271,30 +261,41 @@ private boolean setResponseTimeout() { private long responseTimeoutMillis() { if (!hasDeadline) { - return retryConfig.responseTimeoutMillisForEachAttempt(); + return config.responseTimeoutMillisForEachAttempt(); } final long remaining = TimeUnit.NANOSECONDS.toMillis(deadlineTimeNanos - System.nanoTime()); if (remaining <= 0) { return -1; } - if (retryConfig.responseTimeoutMillisForEachAttempt() > 0) { - return Math.min(retryConfig.responseTimeoutMillisForEachAttempt(), remaining); + if (config.responseTimeoutMillisForEachAttempt() > 0) { + return Math.min(config.responseTimeoutMillisForEachAttempt(), remaining); } return remaining; } + @Override + public RpcResponse res() { + return res; + } + + RetryConfig config() { + return config; + } + @Override public String toString() { return MoreObjects .toStringHelper(this) .add("ctx", ctx) - .add("retryConfig", retryConfig) + .add("retryConfig", config) .add("req", req) .add("res", res) .add("deadlineTimeNanos", deadlineTimeNanos) .add("hasDeadline", hasDeadline) - .add("retryCounter", retryCounter) + .add("counter", counter) + .add("scheduler", scheduler) + .add("currentAttempt", currentAttempt) .toString(); } } From 286e4bf25cf1ac0464c3398297c44771a2ac2d96 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 22 Aug 2025 09:48:44 +0100 Subject: [PATCH 21/42] refactor: use Precondition for parameter checks in RetryCounter --- .../linecorp/armeria/client/retry/RetryCounter.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java index 3ca567f5232..e949bd53341 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java @@ -16,6 +16,8 @@ package com.linecorp.armeria.client.retry; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; import com.google.common.base.MoreObjects; @@ -31,9 +33,7 @@ final class RetryCounter { private int numberAttemptsSoFarForLastBackoff; RetryCounter(int maxAttempts) { - if (maxAttempts <= 0) { - throw new IllegalArgumentException("maxAttempts: " + maxAttempts + " (expected: > 0)"); - } + checkArgument(maxAttempts > 0, "maxAttempts: %s (expected: > 0)", maxAttempts); this.maxAttempts = maxAttempts; numberAttemptsSoFar = 0; lastBackoff = null; @@ -41,9 +41,7 @@ final class RetryCounter { } void recordAttemptWith(@Nullable Backoff backoff) { - if (hasReachedMaxAttempts()) { - throw new IllegalStateException("Exceeded the maximum number of attempts: " + maxAttempts); - } + checkState(!hasReachedMaxAttempts(), "Exceeded the maximum number of attempts: %s", maxAttempts); ++numberAttemptsSoFar; From c4dd4a91cd8bb01947e7c3d65aa674e2eeb1988f Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 22 Aug 2025 11:52:10 +0100 Subject: [PATCH 22/42] fix: make HttpRetryingContext/HttpRetryAttempt thread-safe --- .../client/retry/HttpRetryingContext.java | 368 ++++++++++-------- 1 file changed, 196 insertions(+), 172 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java index 49b254da644..980bc44ff9c 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java @@ -58,6 +58,8 @@ private enum State { COMPLETED } + private final Object lock = new Object(); + private State state; private final ClientRequestContext ctx; private final RetryConfig config; @@ -108,214 +110,234 @@ private enum State { @Override public CompletableFuture init() { - checkState(state == State.UNINITIALIZED); - state = State.INITIALIZING; - final CompletableFuture initFuture = new CompletableFuture<>(); - - if (ctx.exchangeType().isRequestStreaming()) { - reqDuplicator = - req.toDuplicator(ctx.eventLoop().withoutContext(), 0); - state = State.INITIALIZED; - initFuture.complete(true); - } else { - req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) - .handle((agg, reqCause) -> { - assert state == State.INITIALIZING; - - if (reqCause != null) { - resFuture.completeExceptionally(reqCause); - ctx.logBuilder().endRequest(reqCause); - ctx.logBuilder().endResponse(reqCause); - state = State.COMPLETED; - initFuture.complete(false); - } else { - reqDuplicator = new AggregatedHttpRequestDuplicator(agg); - state = State.INITIALIZED; - initFuture.complete(true); - } - return null; - }); - } - - initFuture.whenComplete((initSuccessful, initCause) -> { - assert state == State.INITIALIZED || state == State.COMPLETED; - if (!initSuccessful || initCause != null) { - return; + synchronized (lock) { + checkState(state == State.UNINITIALIZED); + state = State.INITIALIZING; + + final CompletableFuture initFuture = new CompletableFuture<>(); + + if (ctx.exchangeType().isRequestStreaming()) { + reqDuplicator = + req.toDuplicator(ctx.eventLoop().withoutContext(), 0); + state = State.INITIALIZED; + initFuture.complete(true); + } else { + req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) + .handle((agg, reqCause) -> { + synchronized (lock) { + assert state == State.INITIALIZING; + + if (reqCause != null) { + resFuture.completeExceptionally(reqCause); + ctx.logBuilder().endRequest(reqCause); + ctx.logBuilder().endResponse(reqCause); + state = State.COMPLETED; + initFuture.complete(false); + } else { + reqDuplicator = new AggregatedHttpRequestDuplicator(agg); + state = State.INITIALIZED; + initFuture.complete(true); + } + } + return null; + }); } - req.whenComplete().handle((unused, cause) -> { - if (cause != null) { - abort(cause); + initFuture.whenComplete((initSuccessful, initCause) -> { + synchronized (lock) { + assert state == State.INITIALIZED || state == State.COMPLETED; + if (!initSuccessful || initCause != null) { + return; + } + + req.whenComplete().handle((unused, cause) -> { + if (cause != null) { + abort(cause); + } + return null; + }); + + res.whenComplete().handle((result, cause) -> { + final Throwable abortCause; + if (cause != null) { + abortCause = cause; + } else { + abortCause = AbortedStreamException.get(); + } + abort(abortCause); + return null; + }); } - return null; }); - res.whenComplete().handle((result, cause) -> { - final Throwable abortCause; - if (cause != null) { - abortCause = cause; - } else { - abortCause = AbortedStreamException.get(); - } - abort(abortCause); - return null; - }); - }); - - return initFuture; + return initFuture; + } } @Override public CompletableFuture<@Nullable RetryDecision> executeAttempt( @Nullable Backoff backoff, Client delegate) { - checkState(state == State.INITIALIZED); - assert reqDuplicator != null; - // We are not supporting concurrent attempts (yet). - // As such, we expect the previous attempt (if any) to be aborted before - // (abortion sets this field to null). - // assert state != State.EXECUTING; - assert currentAttempt == null; - - if (!setResponseTimeout()) { - return UnmodifiableFuture.exceptionallyCompletedFuture(ResponseTimeoutException.get()); - } + synchronized (lock) { + checkState(state == State.INITIALIZED); + assert reqDuplicator != null; + // We are not supporting concurrent attempts (yet). + // As such, we expect the previous attempt (if any) to be aborted before + // (abortion sets this field to null). + // assert state != State.EXECUTING; + assert currentAttempt == null; + + if (!setResponseTimeout()) { + return UnmodifiableFuture.exceptionallyCompletedFuture(ResponseTimeoutException.get()); + } - state = State.EXECUTING; - counter.recordAttemptWith(backoff); + state = State.EXECUTING; + counter.recordAttemptWith(backoff); - final int attemptNumber = counter.attemptNumber(); - final boolean isInitialAttempt = attemptNumber <= 1; + final int attemptNumber = counter.attemptNumber(); + final boolean isInitialAttempt = attemptNumber <= 1; - final HttpRequest attemptReq; - final ClientRequestContext attemptCtx; - if (isInitialAttempt) { - attemptReq = reqDuplicator.duplicate(); - } else { - final RequestHeadersBuilder attemptHeadersBuilder = req.headers().toBuilder(); - attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, attemptNumber - 1); - attemptReq = reqDuplicator.duplicate(attemptHeadersBuilder.build()); - } + final HttpRequest attemptReq; + final ClientRequestContext attemptCtx; + if (isInitialAttempt) { + attemptReq = reqDuplicator.duplicate(); + } else { + final RequestHeadersBuilder attemptHeadersBuilder = req.headers().toBuilder(); + attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, attemptNumber - 1); + attemptReq = reqDuplicator.duplicate(attemptHeadersBuilder.build()); + } - attemptCtx = ClientUtil.newDerivedContext(ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); - - final HttpResponse attemptRes; - final ClientRequestContextExtension attemptCtxExt = - attemptCtx.as(ClientRequestContextExtension.class); - if (!isInitialAttempt && attemptCtxExt != null && attemptCtx.endpoint() == null) { - // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); - // if the endpoint hasn't been selected, - // try to initialize the attempCtx with a new endpoint/event loop - attemptRes = initContextAndExecuteWithFallback( - delegate, attemptCtxExt, HttpResponse::of, - (context, cause) -> - HttpResponse.ofFailure(cause), attemptReq, false); - } else { - attemptRes = executeWithFallback(delegate, attemptCtx, - (context, cause) -> - HttpResponse.ofFailure(cause), attemptReq, false); - } + attemptCtx = ClientUtil.newDerivedContext(ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); + + final HttpResponse attemptRes; + final ClientRequestContextExtension attemptCtxExt = + attemptCtx.as(ClientRequestContextExtension.class); + if (!isInitialAttempt && attemptCtxExt != null && attemptCtx.endpoint() == null) { + // clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); + // if the endpoint hasn't been selected, + // try to initialize the attempCtx with a new endpoint/event loop + attemptRes = initContextAndExecuteWithFallback( + delegate, attemptCtxExt, HttpResponse::of, + (context, cause) -> + HttpResponse.ofFailure(cause), attemptReq, false); + } else { + attemptRes = executeWithFallback(delegate, attemptCtx, + (context, cause) -> + HttpResponse.ofFailure(cause), attemptReq, false); + } - final HttpRetryAttempt attempt = new HttpRetryAttempt(this, attemptCtx, attemptRes, - ctx.exchangeType().isResponseStreaming()); - currentAttempt = attempt; - return attempt.whenDecided(); + final HttpRetryAttempt attempt = new HttpRetryAttempt(this, attemptCtx, attemptRes, + ctx.exchangeType().isResponseStreaming()); + currentAttempt = attempt; + return attempt.whenDecided(); + } } // returns Long.MAX_VALUE if no retry is possible. @Override public long nextRetryTimeNanos(Backoff backoff) { - checkState(state == State.EXECUTING); - assert currentAttempt != null; - checkState(currentAttempt.state() == HttpRetryAttempt.State.DECIDED); + synchronized (lock) { + checkState(state == State.EXECUTING); + assert currentAttempt != null; + checkState(currentAttempt.state() == HttpRetryAttempt.State.DECIDED); - if (counter.hasReachedMaxAttempts()) { - logger.debug("Exceeded the default number of max attempt: {}", config.maxTotalAttempts()); - return Long.MAX_VALUE; - } + if (counter.hasReachedMaxAttempts()) { + logger.debug("Exceeded the default number of max attempt: {}", config.maxTotalAttempts()); + return Long.MAX_VALUE; + } - final long nextRetryDelayForBackoffMillis = backoff.nextDelayMillis( - counter.attemptNumberForBackoff(backoff) + 1); + final long nextRetryDelayForBackoffMillis = backoff.nextDelayMillis( + counter.attemptNumberForBackoff(backoff) + 1); - if (nextRetryDelayForBackoffMillis < 0) { - logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); - return Long.MAX_VALUE; - } + if (nextRetryDelayForBackoffMillis < 0) { + logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); + return Long.MAX_VALUE; + } - final long nextRetryDelayMillis = Math.max(nextRetryDelayForBackoffMillis, - useRetryAfter ? currentAttempt.retryAfterMillis() : -1); - final long nextDelayTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(nextRetryDelayMillis); + final long nextRetryDelayMillis = Math.max(nextRetryDelayForBackoffMillis, + useRetryAfter ? currentAttempt.retryAfterMillis() : -1); + final long nextDelayTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos( + nextRetryDelayMillis); - if (hasDeadline && nextDelayTimeNanos > deadlineTimeNanos) { - // The next retry will be after the response deadline. So return just Long.MAX_VALUE. - return Long.MAX_VALUE; - } + if (hasDeadline && nextDelayTimeNanos > deadlineTimeNanos) { + // The next retry will be after the response deadline. So return just Long.MAX_VALUE. + return Long.MAX_VALUE; + } - return nextDelayTimeNanos; + return nextDelayTimeNanos; + } } @Override public void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, Consumer exceptionHandler) { - checkState(state == State.INITIALIZED); - assert currentAttempt == null; - scheduler.scheduleNextRetry(nextRetryTimeNanos, retryTask, exceptionHandler); + synchronized (lock) { + checkState(state == State.INITIALIZED); + assert currentAttempt == null; + scheduler.scheduleNextRetry(nextRetryTimeNanos, retryTask, exceptionHandler); + } } @Override public void commit() { - if (state == State.COMPLETED) { - // Already completed. - return; + synchronized (lock) { + if (state == State.COMPLETED) { + // Already completed. + return; + } + checkState(state == State.EXECUTING); + assert reqDuplicator != null; + assert currentAttempt != null; + checkState(currentAttempt.state() == HttpRetryAttempt.State.DECIDED); + + state = State.COMPLETED; + + final HttpResponse attemptRes = currentAttempt.commit(); + // todo(szymon): replace with endResponseWithChild + ctx.logBuilder().endResponseWithChild(currentAttempt.ctx().log()); + resFuture.complete(attemptRes); + reqDuplicator.close(); } - checkState(state == State.EXECUTING); - assert reqDuplicator != null; - assert currentAttempt != null; - checkState(currentAttempt.state() == HttpRetryAttempt.State.DECIDED); - - state = State.COMPLETED; - - final HttpResponse attemptRes = currentAttempt.commit(); - // todo(szymon): replace with endResponseWithChild - ctx.logBuilder().endResponseWithChild(currentAttempt.ctx().log()); - resFuture.complete(attemptRes); - reqDuplicator.close(); } @Override public void abortAttempt() { - checkState(state == State.EXECUTING); - checkState(currentAttempt != null, "No active attempt to abort"); - currentAttempt.abort(); - state = State.INITIALIZED; - currentAttempt = null; + synchronized (lock) { + checkState(state == State.EXECUTING); + checkState(currentAttempt != null, "No active attempt to abort"); + currentAttempt.abort(); + state = State.INITIALIZED; + currentAttempt = null; + } } @Override public void abort(Throwable cause) { - if (state == State.COMPLETED) { - // Already completed. - return; - } + synchronized (lock) { + if (state == State.COMPLETED) { + // Already completed. + return; + } - if (state == State.EXECUTING) { - assert currentAttempt != null; - currentAttempt.abort(); - } + if (state == State.EXECUTING) { + assert currentAttempt != null; + currentAttempt.abort(); + } - state = State.COMPLETED; + state = State.COMPLETED; - if (reqDuplicator != null) { - reqDuplicator.abort(cause); - } + if (reqDuplicator != null) { + reqDuplicator.abort(cause); + } - if (!ctx.log().isRequestComplete()) { - ctx.logBuilder().endRequest(cause); - } - ctx.logBuilder().endResponse(cause); + if (!ctx.log().isRequestComplete()) { + ctx.logBuilder().endRequest(cause); + } + ctx.logBuilder().endResponse(cause); - resFuture.completeExceptionally(cause); + resFuture.completeExceptionally(cause); + } } @Override @@ -363,19 +385,21 @@ private long responseTimeoutMillis() { @Override public String toString() { - return MoreObjects - .toStringHelper(this) - .add("state", state) - .add("ctx", ctx) - .add("config", config) - .add("req", req) - .add("res", res) - .add("useRetryAfter", useRetryAfter) - .add("deadlineTimeNanos", deadlineTimeNanos) - .add("hasDeadline", hasDeadline) - .add("counter", counter) - .add("scheduler", scheduler) - .add("currentAttempt", currentAttempt) - .toString(); + synchronized (lock) { + return MoreObjects + .toStringHelper(this) + .add("state", state) + .add("ctx", ctx) + .add("config", config) + .add("req", req) + .add("res", res) + .add("useRetryAfter", useRetryAfter) + .add("deadlineTimeNanos", deadlineTimeNanos) + .add("hasDeadline", hasDeadline) + .add("counter", counter) + .add("scheduler", scheduler) + .add("currentAttempt", currentAttempt) + .toString(); + } } } From 1a174c9bbc455e49ef5258b049e8c864789eeafe Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 22 Aug 2025 12:02:08 +0100 Subject: [PATCH 23/42] fix: make RpcRetryingContext/RpcRetryAttempt thread-safe --- .../client/retry/RpcRetryingContext.java | 246 ++++++++++-------- 1 file changed, 133 insertions(+), 113 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java index afaf8e719c0..6b9022947d8 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java @@ -54,6 +54,8 @@ private enum State { COMPLETED } + private final Object lock = new Object(); + private State state; private final ClientRequestContext ctx; private final RetryConfig config; @@ -97,153 +99,169 @@ private enum State { @Override public CompletableFuture init() { - checkState(state == State.INITIALIZED); - - res.whenComplete().handle((result, cause) -> { - final Throwable abortCause; - if (cause != null) { - abortCause = cause; - } else { - abortCause = AbortedStreamException.get(); - } - abort(abortCause); - return null; - }); - - return UnmodifiableFuture.completedFuture(true); + synchronized (lock) { + checkState(state == State.INITIALIZED); + + res.whenComplete().handle((result, cause) -> { + final Throwable abortCause; + if (cause != null) { + abortCause = cause; + } else { + abortCause = AbortedStreamException.get(); + } + abort(abortCause); + return null; + }); + + return UnmodifiableFuture.completedFuture(true); + } } @Override public CompletableFuture<@Nullable RetryDecision> executeAttempt(@Nullable Backoff backoff, Client delegate) { - checkState(state == State.INITIALIZED); - assert currentAttempt == null; + synchronized (lock) { + checkState(state == State.INITIALIZED); + assert currentAttempt == null; - if (!setResponseTimeout()) { - return UnmodifiableFuture.exceptionallyCompletedFuture(ResponseTimeoutException.get()); - } + if (!setResponseTimeout()) { + return UnmodifiableFuture.exceptionallyCompletedFuture(ResponseTimeoutException.get()); + } - state = State.EXECUTING; + state = State.EXECUTING; - counter.recordAttemptWith(backoff); + counter.recordAttemptWith(backoff); - final int attemptNumber = counter.attemptNumber(); - final boolean isInitialAttempt = attemptNumber <= 1; + final int attemptNumber = counter.attemptNumber(); + final boolean isInitialAttempt = attemptNumber <= 1; - final ClientRequestContext attemptCtx = ClientUtil.newDerivedContext(ctx, null, req, isInitialAttempt); + final ClientRequestContext attemptCtx = ClientUtil.newDerivedContext(ctx, null, req, + isInitialAttempt); - if (!isInitialAttempt) { - attemptCtx.mutateAdditionalRequestHeaders( - mutator -> mutator.add(ARMERIA_RETRY_COUNT, Integer.toString(attemptNumber - 1))); - } + if (!isInitialAttempt) { + attemptCtx.mutateAdditionalRequestHeaders( + mutator -> mutator.add(ARMERIA_RETRY_COUNT, Integer.toString(attemptNumber - 1))); + } - final RpcResponse attemptRes; - final ClientRequestContextExtension attemptCtxExtension = - attemptCtx.as(ClientRequestContextExtension.class); - final EndpointGroup endpointGroup = attemptCtx.endpointGroup(); - if (!isInitialAttempt && attemptCtxExtension != null && - endpointGroup != null && attemptCtx.endpoint() == null) { - // Clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); - // Initialize the context with a new endpoint/event loop if not selected yet - attemptRes = initContextAndExecuteWithFallback(delegate, attemptCtxExtension, RpcResponse::from, - (unused, cause) -> RpcResponse.ofFailure(cause), - req, true); - } else { - attemptRes = executeWithFallback(delegate, attemptCtx, - (unused, cause) -> RpcResponse.ofFailure(cause), - req, true); - } + final RpcResponse attemptRes; + final ClientRequestContextExtension attemptCtxExtension = + attemptCtx.as(ClientRequestContextExtension.class); + final EndpointGroup endpointGroup = attemptCtx.endpointGroup(); + if (!isInitialAttempt && attemptCtxExtension != null && + endpointGroup != null && attemptCtx.endpoint() == null) { + // Clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); + // Initialize the context with a new endpoint/event loop if not selected yet + attemptRes = initContextAndExecuteWithFallback(delegate, attemptCtxExtension, RpcResponse::from, + (unused, cause) -> RpcResponse.ofFailure(cause), + req, true); + } else { + attemptRes = executeWithFallback(delegate, attemptCtx, + (unused, cause) -> RpcResponse.ofFailure(cause), + req, true); + } - final RpcRetryAttempt attempt = new RpcRetryAttempt(this, attemptCtx, attemptRes); - currentAttempt = attempt; - return attempt.whenDecided(); + final RpcRetryAttempt attempt = new RpcRetryAttempt(this, attemptCtx, attemptRes); + currentAttempt = attempt; + return attempt.whenDecided(); + } } @Override public long nextRetryTimeNanos(Backoff backoff) { - if (state != State.EXECUTING) { - return Long.MAX_VALUE; - } + synchronized (lock) { + if (state != State.EXECUTING) { + return Long.MAX_VALUE; + } - if (counter.hasReachedMaxAttempts()) { - logger.debug("Exceeded the default number of max attempt: {}", config.maxTotalAttempts()); - return Long.MAX_VALUE; - } + if (counter.hasReachedMaxAttempts()) { + logger.debug("Exceeded the default number of max attempt: {}", config.maxTotalAttempts()); + return Long.MAX_VALUE; + } - final long nextRetryDelayMillis = backoff.nextDelayMillis( - counter.attemptNumberForBackoff(backoff) + 1); + final long nextRetryDelayMillis = backoff.nextDelayMillis( + counter.attemptNumberForBackoff(backoff) + 1); - if (nextRetryDelayMillis < 0) { - logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); - return Long.MAX_VALUE; - } + if (nextRetryDelayMillis < 0) { + logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); + return Long.MAX_VALUE; + } - final long nextDelayTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(nextRetryDelayMillis); - if (hasDeadline && nextDelayTimeNanos > deadlineTimeNanos) { - // The next retry will be after the response deadline. So return just Long.MAX_VALUE. - return Long.MAX_VALUE; + final long nextDelayTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos( + nextRetryDelayMillis); + if (hasDeadline && nextDelayTimeNanos > deadlineTimeNanos) { + // The next retry will be after the response deadline. So return just Long.MAX_VALUE. + return Long.MAX_VALUE; + } + return nextDelayTimeNanos; } - return nextDelayTimeNanos; } @Override public void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, Consumer exceptionHandler) { - checkState(state == State.INITIALIZED); - assert currentAttempt == null; - scheduler.scheduleNextRetry(nextRetryTimeNanos, retryTask, exceptionHandler); + synchronized (lock) { + checkState(state == State.INITIALIZED); + assert currentAttempt == null; + scheduler.scheduleNextRetry(nextRetryTimeNanos, retryTask, exceptionHandler); + } } @Override public void commit() { - if (state == State.COMPLETED) { - // Already completed, so just return. - return; - } - checkState(state == State.EXECUTING); - assert currentAttempt != null; - checkState(currentAttempt.state() == RpcRetryAttempt.State.DECIDED); + synchronized (lock) { + if (state == State.COMPLETED) { + // Already completed, so just return. + return; + } + checkState(state == State.EXECUTING); + assert currentAttempt != null; + checkState(currentAttempt.state() == RpcRetryAttempt.State.DECIDED); - state = State.COMPLETED; + state = State.COMPLETED; - final RpcResponse attemptRes = currentAttempt.commit(); - ctx.logBuilder().endResponseWithChild(currentAttempt.ctx().log()); - final HttpRequest attemptReq = currentAttempt.ctx().request(); - if (attemptReq != null) { - ctx.updateRequest(attemptReq); + final RpcResponse attemptRes = currentAttempt.commit(); + ctx.logBuilder().endResponseWithChild(currentAttempt.ctx().log()); + final HttpRequest attemptReq = currentAttempt.ctx().request(); + if (attemptReq != null) { + ctx.updateRequest(attemptReq); + } + resFuture.complete(attemptRes); } - resFuture.complete(attemptRes); } @Override public void abortAttempt() { - checkState(state == State.EXECUTING); - assert currentAttempt != null; - // Can be called in any state. - currentAttempt.abort(); - currentAttempt = null; - state = State.INITIALIZED; + synchronized (lock) { + checkState(state == State.EXECUTING); + assert currentAttempt != null; + // Can be called in any state. + currentAttempt.abort(); + currentAttempt = null; + state = State.INITIALIZED; + } } @Override public void abort(Throwable cause) { - if (state == State.COMPLETED) { - return; - } + synchronized (lock) { + if (state == State.COMPLETED) { + return; + } - if (state == State.EXECUTING) { - assert currentAttempt != null; - currentAttempt.abort(); - } + if (state == State.EXECUTING) { + assert currentAttempt != null; + currentAttempt.abort(); + } - state = State.COMPLETED; + state = State.COMPLETED; - if (!ctx.log().isRequestComplete()) { - ctx.logBuilder().endRequest(cause); + if (!ctx.log().isRequestComplete()) { + ctx.logBuilder().endRequest(cause); + } + ctx.logBuilder().endResponse(cause); + resFuture.completeExceptionally(cause); } - ctx.logBuilder().endResponse(cause); - resFuture.completeExceptionally(cause); } private boolean setResponseTimeout() { @@ -285,17 +303,19 @@ RetryConfig config() { @Override public String toString() { - return MoreObjects - .toStringHelper(this) - .add("ctx", ctx) - .add("retryConfig", config) - .add("req", req) - .add("res", res) - .add("deadlineTimeNanos", deadlineTimeNanos) - .add("hasDeadline", hasDeadline) - .add("counter", counter) - .add("scheduler", scheduler) - .add("currentAttempt", currentAttempt) - .toString(); + synchronized (lock) { + return MoreObjects + .toStringHelper(this) + .add("ctx", ctx) + .add("state", state) + .add("req", req) + .add("res", res) + .add("deadlineTimeNanos", deadlineTimeNanos) + .add("hasDeadline", hasDeadline) + .add("counter", counter) + .add("scheduler", scheduler) + .add("currentAttempt", currentAttempt) + .toString(); + } } } From d55fc03a83cbc1d15ae8fbf8a72d1877420d917e Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 22 Aug 2025 13:02:25 +0100 Subject: [PATCH 24/42] docs: add doc to RetryingContext --- .../client/retry/AbstractRetryingClient.java | 2 +- .../client/retry/HttpRetryAttempt.java | 9 ------ .../client/retry/HttpRetryingContext.java | 1 - .../armeria/client/retry/RetryingContext.java | 31 +++++++++++++++++++ .../armeria/client/retry/RpcRetryAttempt.java | 1 - 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 36528a03c2b..3ad31a18886 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -73,7 +73,7 @@ public final O execute(ClientRequestContext ctx, I req) throws Exception { final R rctx = getRetryingContext(ctx, retryConfigForReq, req); rctx.init().handle((initSuccessful, initCause) -> { if (!initSuccessful || initCause != null) { - // todo(szymon): comment here. + rctx.abort(initCause); logger.debug("RetryingContext initialization failed, not retrying: {}", rctx, initCause); return null; } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java index 34a290be2bd..5598f9d251e 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java @@ -51,17 +51,10 @@ final class HttpRetryAttempt { private static final Logger logger = LoggerFactory.getLogger(HttpRetryAttempt.class); enum State { - // Initial state after constructing an `Attempt`. - // The attempt response is underway but did not complete yet. EXECUTING, - // todo(szymon): doc DECIDING, - // todo(szymon): doc DECIDED, - // State after a call to `commit`. Terminal state, caller cannot make further calls. COMMITTED, - // State after a call to `abort`. Terminal state, caller cannot make further calls. - // `res` is aborted. ABORTED } @@ -249,8 +242,6 @@ private void handleStreamingRes() { final HttpResponseDuplicator resDuplicator = unsplitRes.toDuplicator(ctx.eventLoop().withoutContext(), ctx.maxResponseLength()); - // todo(szymon): We do not call duplicator.abort(cause); but res.abort on an exception. - // Is this okay? final HttpResponse duplicatedRes = resDuplicator.duplicate(); final TruncatingHttpResponse truncatingAttemptRes = new TruncatingHttpResponse(resDuplicator.duplicate(), diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java index 980bc44ff9c..1fd6c0816b6 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java @@ -294,7 +294,6 @@ public void commit() { state = State.COMPLETED; final HttpResponse attemptRes = currentAttempt.commit(); - // todo(szymon): replace with endResponseWithChild ctx.logBuilder().endResponseWithChild(currentAttempt.ctx().log()); resFuture.complete(attemptRes); reqDuplicator.close(); diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java index adf87e60a35..4ea5f14b75b 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java @@ -20,10 +20,41 @@ import java.util.function.Consumer; import com.linecorp.armeria.client.Client; +import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.Response; import com.linecorp.armeria.common.annotation.Nullable; +/** + * Handle used by {@link AbstractRetryingClient} to drive the retrying process of a request. + * In particular, it provides methods to + * + *

    + *
  • initialize the retrying process ({@link #init()}),
    • + *
  • execute a request attempt ({@link #executeAttempt(Backoff, Client)}),
    • + *
  • decide to abort the attempt and schedule a new one ({@link #abortAttempt()} and + * {@link #scheduleNextRetry(long, Runnable, Consumer)}) or to
    • + *
  • accept the response attempt as the final response of the request ({@link #commit()}).
    • + *
+ * + *

+ * Notes: + *

    + *
  • + * Calls to methods of this interface must be in a certain order. The method call order + * can be "seen" in {@link AbstractRetryingClient#execute(ClientRequestContext, Request)}.
    + * Note that {@link #abort(Throwable)} and {@link #res()} + * can be called at any point, even before a call to {@link #init()}. + *
    • + *
  • Implementors of this interface must be thread-safe.
    • + *
+ *

+ * + * @param the {@link Request} type + * @param the {@link Response} type + * @see HttpRetryingContext + * @see RpcRetryingContext + */ interface RetryingContext { CompletableFuture init(); diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java index 2a2f5f0789d..b3c0876ed99 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java @@ -27,7 +27,6 @@ import com.linecorp.armeria.common.annotation.Nullable; final class RpcRetryAttempt { - // todo(szymon): doc enum State { EXECUTING, DECIDING, From 3e2b7fa6a0130d73bd40efacd5998b6454d0a354 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 7 Sep 2025 12:32:22 +0200 Subject: [PATCH 25/42] feat: refactor (Abstract)RetryingClient into RetriedRequest, RetryScheduler and RetryCounter - refactor of the RpcRetryingClient still TODO --- .../client/retry/AbortedAttemptException.java | 44 ++ .../client/retry/AbstractRetryingClient.java | 190 ++++- .../client/retry/DefaultRetryCounter.java | 85 +++ .../client/retry/DefaultRetryScheduler.java | 211 ++++++ .../client/retry/HttpRetryAttempt.java | 696 ++++++++++++------ .../client/retry/HttpRetryingContext.java | 404 ---------- .../client/retry/RetriedHttpRequest.java | 402 ++++++++++ .../armeria/client/retry/RetriedRequest.java | 150 ++++ .../armeria/client/retry/RetryCounter.java | 99 +-- .../armeria/client/retry/RetryScheduler.java | 102 ++- .../armeria/client/retry/RetryingClient.java | 28 +- .../client/retry/RetryingRpcClient.java | 15 +- .../armeria/client/retry/RpcRetryAttempt.java | 133 ---- .../client/retry/RpcRetryingContext.java | 321 -------- .../armeria/internal/client/ClientUtil.java | 49 ++ 15 files changed, 1706 insertions(+), 1223 deletions(-) create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/AbortedAttemptException.java create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryCounter.java create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java delete mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java delete mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java delete mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbortedAttemptException.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbortedAttemptException.java new file mode 100644 index 00000000000..053c84300f7 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbortedAttemptException.java @@ -0,0 +1,44 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.client.retry; + +import com.linecorp.armeria.common.Flags; + +/** + * A {@link RuntimeException} that is raised by a {@link RetriedRequest} to signal to the caller of + * {@link RetriedRequest#executeAttempt} that the attempt has been aborted because the request has been + * completed - either at call time or during the execution of the attempt. + */ +public final class AbortedAttemptException extends RuntimeException { + private static final long serialVersionUID = -1L; + private static final AbortedAttemptException INSTANCE = new AbortedAttemptException(true); + + /** + * Returns a {@link AbortedAttemptException} which may be a singleton or a new instance, depending on + * {@link Flags#verboseExceptionSampler()}'s decision. + */ + public static AbortedAttemptException get() { + return Flags.verboseExceptionSampler().isSampled( + AbortedAttemptException.class) ? + new AbortedAttemptException() : INSTANCE; + } + + private AbortedAttemptException() {} + + private AbortedAttemptException(@SuppressWarnings("unused") boolean dummy) { + super(null, null, false, false); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 3ad31a18886..56cb55e7a8b 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -27,7 +27,9 @@ import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.Response; import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.stream.AbortedStreamException; +import io.netty.channel.EventLoop; import io.netty.util.AsciiString; /** @@ -37,9 +39,8 @@ * @param the {@link Response} type */ public abstract class AbstractRetryingClient - > + extends SimpleDecoratingClient { - private static final Logger logger = LoggerFactory.getLogger(AbstractRetryingClient.class); /** @@ -63,58 +64,185 @@ public abstract class AbstractRetryingClient this.retryConfig = retryConfig; } + abstract RetryContext newRetryContext( + Client delegate, + ClientRequestContext ctx, + I req, + RetryConfig config); + @Override public final O execute(ClientRequestContext ctx, I req) throws Exception { - final RetryConfig retryConfigForReq = + final RetryConfig config = retryConfig != null ? retryConfig : requireNonNull(retryMapping.get(ctx, req), "retryMapping.get() returned null"); + final RetryContext rctx = newRetryContext(unwrap(), ctx, req, config); + + // The request could complete at any moment. + // In that case, let us make sure that we close the scheduler so we + // do not run any retry task unnecessarily. + // That said, running retry() even with a completed response is handled by + // rctx.request().executeAttempt() which throws an AbortedAttemptException which we handle gracefully. + rctx.request().res().whenComplete().handle((unused, cause) -> { + // Make sure we do not unnecessarily run any retry task. + if (rctx.retryEventLoop().inEventLoop()) { + rctx.scheduler().close(); + } else { + rctx.retryEventLoop().execute(rctx.scheduler()::close); + } + return null; + }); - final R rctx = getRetryingContext(ctx, retryConfigForReq, req); - rctx.init().handle((initSuccessful, initCause) -> { - if (!initSuccessful || initCause != null) { - rctx.abort(initCause); - logger.debug("RetryingContext initialization failed, not retrying: {}", rctx, initCause); - return null; + // If something goes wrong with the scheduler, e.g. when the ClientFactory closes and the scheduler is + // unable to schedule retry tasks, we want to make sure we gracefully abort the request with that + // exception. + rctx.scheduler().whenClosed().handle((unused, cause) -> { + assert rctx.retryEventLoop().inEventLoop(); + + if (cause == null) { + cause = new IllegalStateException( + "retry scheduler was closed before the request was completed"); } - executeAttempt(null, rctx); + rctx.request().abort(cause); return null; }); - return rctx.res(); + if (rctx.retryEventLoop().inEventLoop()) { + retry(rctx, null); + } else { + rctx.retryEventLoop().execute(() -> retry(rctx, null)); + } + + return rctx.request().res(); } - abstract R getRetryingContext(ClientRequestContext ctx, RetryConfig config, I req); + // NOTE: + // - Must run on the retryEventLoop. + // - The first call must be done from execute() above. + // - Subsequent calls must only be issued in the retry task scheduled by `rctx.scheduler()`. + // The corresponding `scheduler.schedule()` calls must all be done from ´tryScheduleRetryAfter()´ below. + private void retry( + RetryContext rctx, + @Nullable Backoff previousBackoff + ) { + // First record the execution of the following attempt. This increases the attempt count + // and the attempt count for the backoff. Also see (A) for a correctness argument. + rctx.counter().consumeAttemptFrom(previousBackoff); + + rctx.request() + .executeAttempt(rctx.delegate()) + .handle((executionResult, cause) -> { + assert rctx.retryEventLoop().inEventLoop(); + + if (cause != null) { + if (cause instanceof AbortedAttemptException) { + // The attempt was aborted in the course of executing it. This can happen when + // we execute an attempt on a completed request or when the request is completed while + // RetriedRequest is waiting for the response of the attempt. + return null; + } + // Make sure we complete the request with the exception. We could have been aborted because + // another attempt already completed the request but this is not an issue as the call to + // `request.abort()` has no effect then. + rctx.request().abort( + cause != null ? cause : new IllegalStateException("executionResult is null")); + return null; + } - private void executeAttempt(@Nullable Backoff lastBackoff, R rctx) { - rctx.executeAttempt(lastBackoff, unwrap()) - .handle((decision, decisionCause) -> { - if (decisionCause != null) { - rctx.abort(decisionCause); + // An empty backoff means that the RetryRule to commit this attempt. + if (executionResult.decision().backoff() == null) { + rctx.request().commit(executionResult.attemptNumber()); return null; } - final Backoff backoff = decision != null ? decision.backoff() : null; - final long nextRetryTimeNanos; - if (backoff != null) { - nextRetryTimeNanos = rctx.nextRetryTimeNanos(backoff); - } else { - nextRetryTimeNanos = Long.MAX_VALUE; + // Note that applying the pushback needs to be done before the call to tryScheduleRetryAfter`. + if (executionResult.minimumBackoffMillis() > 0) { + rctx.scheduler().applyMinimumBackoffMillisForNextRetry( + executionResult.minimumBackoffMillis()); } - if (nextRetryTimeNanos < Long.MAX_VALUE) { - rctx.abortAttempt(); - rctx.scheduleNextRetry( - nextRetryTimeNanos, - () -> executeAttempt(backoff, rctx), - rctx::abort - ); + tryScheduleRetryAfter(rctx, executionResult.decision().backoff()); + + if (!rctx.scheduler().hasNextRetryTask()) { + // We are not going to retry again so we are the last attempt. Let us commit it even if + // the response was deemed to be unsatisfactory by the RetryRule (backoff != null). + rctx.request().commit(executionResult.attemptNumber()); } else { - rctx.commit(); + // The responsibility of aborting the RetriedRequest is now with the retry task scheduled + // by `tryScheduleRetryAfter()`. Thus, we can safely abort this attempt now. + rctx.request().abort(executionResult.attemptNumber(), AbortedStreamException.get()); } return null; }); } + + // NOTE: Must run on the retryEventLoop. + private void tryScheduleRetryAfter(RetryContext rctx, Backoff nextBackoff) { + if (rctx.counter().hasReachedMaxAttempts()) { + return; + } + + final long nextRetryDelayMillisFromBackoff = nextBackoff.nextDelayMillis( + // NOTE: `attemptsSoFarWithBackoff` is read-only and in particular it does not increase + // the attempt count for the backoff. + // +1 because nextDelayMillis counts the original request as one attempt for the backoff. + rctx.counter().attemptsSoFarWithBackoff(nextBackoff) + 1 + ); + + if (nextRetryDelayMillisFromBackoff < 0) { + // We exceeded the attempt limit for the backoff. + return; + } + + // (A): We are under `maxAttempts` have also not exceeded the backoff attempt limit so let us + // try to schedule the next retry task. + // NOTE: The scheduler guarantees *if* we run this retry task, there will be no retry task run between + // this call and the execution of the retry task. This guarantees two things: + // 1. The attempt number and the attempt number for the backoff now and upon execution of the retry task + // are the same. + // 2. Based on 1., the delay we use to schedule the retry task is indeed the one we consume in the retry + // task via `counter.consumeAttemptFrom(nextBackoff)`. + rctx.scheduler().trySchedule(/* retry task */ () -> retry(rctx, nextBackoff), + nextRetryDelayMillisFromBackoff); + } + + protected final class RetryContext { + + final EventLoop retryEventLoop; + final RetriedRequest request; + final RetryScheduler scheduler; + final RetryCounter counter; + final Client delegate; + + RetryContext(EventLoop retryEventLoop, RetriedRequest request, + RetryScheduler scheduler, RetryCounter counter, Client delegate) { + this.retryEventLoop = retryEventLoop; + this.request = request; + this.scheduler = scheduler; + this.counter = counter; + this.delegate = delegate; + } + + EventLoop retryEventLoop() { + return retryEventLoop; + } + + RetriedRequest request() { + return request; + } + + RetryScheduler scheduler() { + return scheduler; + } + + RetryCounter counter() { + return counter; + } + + Client delegate() { + return delegate; + } + } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryCounter.java b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryCounter.java new file mode 100644 index 00000000000..c716245d324 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryCounter.java @@ -0,0 +1,85 @@ +/* + * Copyright 2025 LINE Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.linecorp.armeria.client.retry; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +import com.google.common.base.MoreObjects; + +import com.linecorp.armeria.common.annotation.Nullable; + +final class DefaultRetryCounter implements RetryCounter { + private final int maxAttempts; + + private int numberAttemptsSoFar; + @Nullable + private Backoff lastBackoff; + private int numberAttemptsSoFarForLastBackoff; + + DefaultRetryCounter(int maxAttempts) { + checkArgument(maxAttempts > 0, "maxAttempts: %s (expected: > 0)", maxAttempts); + this.maxAttempts = maxAttempts; + numberAttemptsSoFar = 0; + lastBackoff = null; + numberAttemptsSoFarForLastBackoff = 0; + } + + @Override + public void consumeAttemptFrom(@Nullable Backoff backoff) { + checkState(!hasReachedMaxAttempts(), "Exceeded the maximum number of attempts: %s", maxAttempts); + + ++numberAttemptsSoFar; + + if (backoff != null) { + if (lastBackoff != backoff) { + lastBackoff = backoff; + numberAttemptsSoFarForLastBackoff = 0; + } + numberAttemptsSoFarForLastBackoff++; + } else { + assert lastBackoff == null; + } + } + + @Override + public int attemptsSoFarWithBackoff(Backoff backoff) { + requireNonNull(backoff, "backoff"); + if (lastBackoff != backoff) { + return 0; + } else { + return numberAttemptsSoFarForLastBackoff; + } + } + + @Override + public boolean hasReachedMaxAttempts() { + return numberAttemptsSoFar >= maxAttempts; + } + + @Override + public String toString() { + return MoreObjects + .toStringHelper(this) + .add("maxAttempts", maxAttempts) + .add("numberAttemptsSoFar", numberAttemptsSoFar) + .add("lastBackoff", lastBackoff) + .add("numberAttemptsSoFarForLastBackoff", numberAttemptsSoFarForLastBackoff) + .toString(); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java new file mode 100644 index 00000000000..3d16db3d509 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java @@ -0,0 +1,211 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.client.retry; + +import static com.google.common.base.Preconditions.checkState; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.util.UnmodifiableFuture; + +import io.netty.channel.EventLoop; +import io.netty.util.concurrent.ScheduledFuture; + +final class DefaultRetryScheduler implements RetryScheduler { + private static final Logger logger = LoggerFactory.getLogger(DefaultRetryScheduler.class); + + private final EventLoop retryEventLoop; + private final long deadlineTimeNanos; + + private final CompletableFuture closedFuture; + private final RetryTaskWrapper retryTaskWrapper; + + @Nullable + private Runnable nextRetryTask; + @Nullable + private ScheduledFuture nextRetryTaskFuture; + // Long.MIN_VALUE if not set. + private long earliestRetryTimeNanos; + + private final class RetryTaskWrapper implements Runnable { + @Override + public void run() { + assert retryEventLoop.inEventLoop(); + + if (closedFuture.isDone()) { + logger.debug("Tried to run a retry task after the scheduler was closed. Skipping this task."); + return; + } + + if (System.nanoTime() > deadlineTimeNanos) { + logger.debug("Tried to run a retry task after the deadline. Skipping this task."); + } + + assert nextRetryTask != null; + final Runnable retryTaskToRun = nextRetryTask; + nextRetryTask = null; + nextRetryTaskFuture = null; + earliestRetryTimeNanos = Long.MIN_VALUE; + + try { + retryTaskToRun.run(); + } catch (Throwable t) { + closeExceptionally(t); + } + } + } + + DefaultRetryScheduler(EventLoop retryEventLoop, long deadlineTimeNanos) { + this.retryEventLoop = retryEventLoop; + this.deadlineTimeNanos = deadlineTimeNanos; + + retryTaskWrapper = new RetryTaskWrapper(); + closedFuture = new CompletableFuture<>(); + + nextRetryTask = null; + nextRetryTaskFuture = null; + earliestRetryTimeNanos = Long.MIN_VALUE; + } + + @Override + public void trySchedule(Runnable retryTask, long delayMillis) { + checkInRetryEventLoop(); + + if (isClosed()) { + return; + } + + // We are a sequential scheduler there must not be a nextRetryTask already set. + checkState(!hasNextRetryTask(), "cannot schedule a retry task when another is scheduled"); + + assert nextRetryTask == null; + assert nextRetryTaskFuture == null; + + final long delayNanos = TimeUnit.MILLISECONDS.toNanos(delayMillis); + final long retryTimeNanos = Math.max(System.nanoTime() + delayNanos, earliestRetryTimeNanos); + + if (retryTimeNanos > deadlineTimeNanos) { + return; + } + + try { + final long nextRetryDelayMillis = TimeUnit.NANOSECONDS.toMillis( + retryTimeNanos - System.nanoTime()); + + nextRetryTask = retryTask; + if (nextRetryDelayMillis <= 0) { + nextRetryTaskFuture = null; + retryEventLoop.execute(retryTaskWrapper); + } else { + nextRetryTaskFuture = retryEventLoop.schedule( + retryTaskWrapper, nextRetryDelayMillis, + TimeUnit.MILLISECONDS); + nextRetryTaskFuture.addListener(future -> { + if (isClosed()) { + return; + } + + if (future.isCancelled()) { + // future is cancelled when the client factory is closed. + closeExceptionally(new IllegalStateException( + ClientFactory.class.getSimpleName() + " has been closed.")); + } else if (future.cause() != null) { + // Other unexpected exceptions. + closeExceptionally(future.cause()); + } + }); + } + } catch (Throwable t) { + closeExceptionally(t); + } + } + + @Override + public void applyMinimumBackoffMillisForNextRetry(long minimumBackoffMillis) { + checkInRetryEventLoop(); + + if (isClosed()) { + return; + } + + // We explicitly disallow that just to avoid having to implement cancelling and rescheduling + // logic. A scheduler implementing hedging would need to do support that, however. + checkState(!hasNextRetryTask(), + "cannot apply minimum backoff when a retry task is scheduled"); + + earliestRetryTimeNanos = Math.max(earliestRetryTimeNanos, + System.nanoTime() + TimeUnit.MILLISECONDS.toNanos( + minimumBackoffMillis) + ); + } + + @Override + public boolean hasNextRetryTask() { + checkInRetryEventLoop(); + // NOTE: nextRetryTask is null when scheduler is closed. + return nextRetryTask != null; + } + + @Override + public void close() { + checkInRetryEventLoop(); + + if (isClosed()) { + return; + } + + clearRetryTaskIfExists(); + closedFuture.complete(null); + } + + @Override + public CompletableFuture whenClosed() { + return UnmodifiableFuture.wrap(closedFuture); + } + + private void closeExceptionally(Throwable cause) { + if (isClosed()) { + return; + } + + clearRetryTaskIfExists(); + closedFuture.completeExceptionally(cause); + } + + private void clearRetryTaskIfExists() { + if (nextRetryTaskFuture != null) { + nextRetryTaskFuture.cancel(false); + } + nextRetryTaskFuture = null; + earliestRetryTimeNanos = Long.MIN_VALUE; + nextRetryTask = null; + } + + private boolean isClosed() { + return closedFuture.isDone(); + } + + private void checkInRetryEventLoop() { + checkState(retryEventLoop.inEventLoop(), "not in the retryEventLoop: %s but in thread %s", + retryEventLoop, Thread.currentThread().getName()); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java index 5598f9d251e..1764496ace8 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java @@ -17,253 +17,504 @@ package com.linecorp.armeria.client.retry; import static com.google.common.base.Preconditions.checkState; +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; +import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; +import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; -import java.time.Duration; -import java.util.Date; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.function.Function; import com.google.common.base.MoreObjects; +import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.common.AggregatedHttpResponse; -import com.linecorp.armeria.common.HttpHeaderNames; +import com.linecorp.armeria.common.ContextAwareEventLoop; +import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.HttpResponseDuplicator; +import com.linecorp.armeria.common.RequestHeadersBuilder; import com.linecorp.armeria.common.ResponseHeaders; import com.linecorp.armeria.common.SplitHttpResponse; import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.logging.RequestLog; -import com.linecorp.armeria.common.logging.RequestLogAccess; import com.linecorp.armeria.common.logging.RequestLogBuilder; import com.linecorp.armeria.common.logging.RequestLogProperty; -import com.linecorp.armeria.common.stream.AbortedStreamException; import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; +import com.linecorp.armeria.internal.client.ClientRequestContextExtension; import com.linecorp.armeria.internal.client.TruncatingHttpResponse; -import io.netty.handler.codec.DateFormatter; - +/** + * A single attempt for a {@link RetriedHttpRequest}. + * + *

+ * NOTE: All methods of {@link HttpRetryAttempt} must be invoked from the + * {@code retryEventLoop}. + *

+ */ final class HttpRetryAttempt { - private static final Logger logger = LoggerFactory.getLogger(HttpRetryAttempt.class); + /** + * Result of {@code State.PREPARING_AND_DECIDING} which contains the response of the attempt + * and the response or cause to be delivered to the {@link RetryRule} for decision + * processing the response until the point where the {@link RetryRule} can make a decision + * (e.g., receiving headers, content, or trailers). + */ + private static final class PreparationResult { + /** The response of the attempt. */ + final HttpResponse res; + + /** + * Response that is derived from {@link PreparationResult#res} which is delivered to + * the {@link RetryRule} for decision. May be {@link PreparationResult#res} itself. + * It is {@code null} iff {@link PreparationResult#causeToDecide} is not {@code null}. + */ + @Nullable + final HttpResponse resToDecide; + + /** + * Exception that occurred while preparing the {@link PreparationResult#res}. It is delivered to + * the {@link RetryRule} for decision. + * It is not {@code null} iff {@link PreparationResult#resToDecide} is {@code null}. + */ + @Nullable + Throwable causeToDecide; + + private PreparationResult(HttpResponse res, @Nullable HttpResponse resToDecide, + @Nullable Throwable causeToDecide) { + // TODO: remove? + assert res != null; + assert (resToDecide == null) != (causeToDecide == null) + : resToDecide + ", " + causeToDecide; + + this.res = res; + this.resToDecide = resToDecide; + this.causeToDecide = causeToDecide; + } + static PreparationResult ofSuccess(HttpResponse res, HttpResponse resForRule) { + return new PreparationResult(res, resForRule, null); + } + + static PreparationResult ofFailure(Throwable cause) { + return new PreparationResult(HttpResponse.ofFailure(cause), null, cause); + } + } + + /** + * The state of this attempt. + * The following state machine diagram illustrates the possible transitions: + * 
+     *          Start
+     *            |
+     *           IDLE
+     *            |
+     *            v
+     *        EXECUTING----------+
+     *            |              |
+     *            v              |
+     *         DECIDED-----------+
+     *          /   \            |
+     *         /     \           |
+     *        /       \          |
+     *       v         v         |
+     *   COMMITTED  ABORTED<-----+
+     *       \         /
+     *        +---+---+
+     *            |
+     *            v
+     *           End
+     *
+ */ enum State { + /** + * The attempt was not executed yet via {@link #executeAndDecide(Client)}. + */ + IDLE, + /** + * The attempt was executed by {@link #executeAndDecide(Client)}. + * + *

+ * The response is being prepared for and given to the {@link RetryRule} to make a decision. + * In this context, 'preparing' means processing the response until the point where + * the {@link RetryRule} can make a decision (e.g., receiving headers, content, or trailers). + * {@link #res} is available from this state onwards. + *

+ */ EXECUTING, - DECIDING, + /** + * The response was prepared and the {@link RetryRule} made a decision. + * The attempt is ready to be committed via {@link #commit()} or aborted via {@link #abort(Throwable)}. + */ DECIDED, + /** + * The attempt was committed via {@link #commit()}. This is a terminal state. + */ COMMITTED, + /** + * The attempt was aborted via {@link #abort(Throwable)}. The response is *going to be* + * aborted via the same cause. In this state and only in this state {@link #cause} is not-{@code null}. + * This is a terminal state. + */ ABORTED } + private final RetryConfig config; + private final ContextAwareEventLoop retryEventLoop; + private final int attemptNumber; + private final ClientRequestContext ctx; + private final HttpRequest req; + private State state; - private final HttpRetryingContext rctx; - private final ClientRequestContext ctx; - private HttpResponse res; - @Nullable - private Throwable resCause; + /** + * The response of the attempt. It is available in {@link State#EXECUTING}, + * {@link State#DECIDED}, and {@link State#COMMITTED} states. + */ @Nullable - private HttpResponse resWithContent; - private final CompletableFuture<@Nullable RetryDecision> whenDecidedFuture; + private HttpResponse res; - HttpRetryAttempt(HttpRetryingContext rctx, ClientRequestContext ctx, HttpResponse res, - boolean isResponseStreaming) { - this.rctx = rctx; + /** + * The cause of the attempt failure. It is not-{@code null} iff we are in {@link State#ABORTED} state. + */ + @Nullable + Throwable cause; + + HttpRetryAttempt( + RetryConfig config, + ContextAwareEventLoop retryEventLoop, + int attemptNumber, + ClientRequestContext ctx, + HttpRequest req + ) { + this.config = config; + this.retryEventLoop = retryEventLoop; + this.attemptNumber = attemptNumber; this.ctx = ctx; - this.res = res; - resCause = null; - resWithContent = null; - whenDecidedFuture = new CompletableFuture<>(); + this.req = req; - state = State.EXECUTING; + state = State.IDLE; + } - if (!isResponseStreaming || rctx.config().requiresResponseTrailers()) { - handleAggRes(); - } else { - handleStreamingRes(); + /** + * Executes the attempt, prepares and gives the response or the cause to the {@link RetryRule}. + * This method must be called at most once and except for calls to {@link #abort(Throwable)} + * it must be called before any other method, in particular before {@link #commit()}. + * + * @param delegate the next {@link Client} in the decoration chain + * @return a future that will be completed with the {@link RetryDecision} or an exception if failed during + * execution, preparation, or decision. + * In particular, it fails with a {@link AbortedAttemptException} + * if the attempt was aborted by {@link #abort(Throwable)}. + * + * @see #commit() + */ + CompletableFuture executeAndDecide(Client delegate) { + checkState(retryEventLoop.inEventLoop()); + + if (state == State.ABORTED) { + assert cause != null; + return UnmodifiableFuture.exceptionallyCompletedFuture(cause); } + + checkState(state == State.IDLE); + state = State.EXECUTING; + + res = execute(delegate); + + return prepareForDecision(res) + .thenCompose(preparationResult -> { + assert retryEventLoop.inEventLoop(); + // Let us avoid calling decide (which may be expensive if we already know that we were + // aborted. + if (state == State.ABORTED) { + assert cause != null; + assert preparationResult.res.isComplete(); + + if (preparationResult.resToDecide != null && + preparationResult.res != preparationResult.resToDecide) { + preparationResult.resToDecide.abort(cause); + } + + return UnmodifiableFuture.exceptionallyCompletedFuture(cause); + } + + assert state == State.EXECUTING : state; + + return decide(preparationResult.resToDecide, + preparationResult.causeToDecide) + .whenComplete((unusedDecision, unusedDecisionCause) -> { + assert retryEventLoop.inEventLoop(); + // Let us abort the response that was delivered to the rule as we do not need + // it anymore. + if (preparationResult.resToDecide != null && + // In case we delivered the attempt response + preparationResult.res != preparationResult.resToDecide) { + preparationResult.resToDecide.abort(); + } + }) + .thenCompose( + decision -> { + if (state == State.ABORTED) { + assert cause != null; + // Was aborted by {@link #abort()} while we were deciding. + assert preparationResult.res.isComplete(); + return UnmodifiableFuture.exceptionallyCompletedFuture(cause); + } else { + assert state == State.EXECUTING : state; + assert cause == null : cause; // sanity check + assert res != null; + res = preparationResult.res; + state = State.DECIDED; + return UnmodifiableFuture.completedFuture(decision); + } + } + ); + }) + .handle( + (decision, preparationOrDecisionCause) -> { + if (preparationOrDecisionCause == null) { + return (CompletableFuture) UnmodifiableFuture.completedFuture( + decision); + } + + if (state == State.ABORTED) { + assert cause != null; + if (cause != preparationOrDecisionCause) { + cause.addSuppressed(preparationOrDecisionCause); + } + return UnmodifiableFuture.exceptionallyCompletedFuture( + preparationOrDecisionCause); + } + assert state == State.EXECUTING : state; + + abort(preparationOrDecisionCause); + return UnmodifiableFuture.exceptionallyCompletedFuture( + preparationOrDecisionCause); + } + ) + .thenCompose(Function.identity()); } - private void decide() { - assert state == State.EXECUTING; - state = State.DECIDING; + private HttpResponse execute(Client delegate) { + final boolean isInitialAttempt = attemptNumber <= 1; - final CompletionStage<@Nullable RetryDecision> retryRuleDecisionFuture; - if (rctx.config().needsContentInRule()) { - final RetryRuleWithContent retryRuleWithContent = - rctx.config().retryRuleWithContent(); - assert retryRuleWithContent != null; - retryRuleDecisionFuture = shouldBeRetriedWith(retryRuleWithContent); + if (!isInitialAttempt) { + ctx.mutateAdditionalRequestHeaders( + mutator -> mutator.add(ARMERIA_RETRY_COUNT, Integer.toString(attemptNumber - 1))); + } + + final RequestHeadersBuilder attemptHeadersBuilder = req.headers().toBuilder(); + attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, attemptNumber - 1); + + final ClientRequestContextExtension ctxExt = + ctx.as(ClientRequestContextExtension.class); + if (!isInitialAttempt && ctxExt != null && ctx.endpoint() == null) { + // clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(ctx); + // if the endpoint hasn't been selected, + // try to initialize the attempCtx with a new endpoint/event loop + return initContextAndExecuteWithFallback( + delegate, ctxExt, HttpResponse::of, + (context, cause) -> + HttpResponse.ofFailure(cause), req, false); } else { - final RetryRule retryRule = rctx.config().retryRule(); - assert retryRule != null; - retryRuleDecisionFuture = shouldBeRetriedWith(retryRule); + return executeWithFallback(delegate, ctx, + (context, cause) -> + HttpResponse.ofFailure(cause), req, false); } + } - retryRuleDecisionFuture.whenComplete((result, exception) -> { - state = State.DECIDED; - if (exception != null) { - whenDecidedFuture.completeExceptionally(exception); - } else { - whenDecidedFuture.complete(result); - } - }); + /** + * Prepares the response for decision by the {@link RetryRule}. + * + *

+ * In the context of this class, preparation means processing the response until the point where the + * {@link RetryRule} can make a decision (e.g., receiving headers, content, or trailers). + * The returned future is completed with a {@link PreparationResult} which contains the response + * for the {@link RetryRule} ({@link PreparationResult#resToDecide}) to make a decision + * and the new response of the attempt (@link PreparationResult#res}). + *

+ * + *

+ * Note that {@link PreparationResult#res} might be different from the + * initial response stored in {@link #res} as we might need to transform it during preparation + * (e.g. by aggregation). + *

+ * + * @param res the response of the attempt + * @return a future that will be completed with the {@link PreparationResult} + */ + private CompletableFuture prepareForDecision(HttpResponse res) { + final boolean aggregateResponse = + !ctx.exchangeType().isResponseStreaming() || config.requiresResponseTrailers(); + return withCompletionOnRetryEventLoop( + aggregateResponse ? prepareAggregatedResponse(res) : prepareStreamingResponse(res)); } + /** + * Marks this attempt as committed and returns the response of the attempt. + * The attempt must be decided, i.e. {@link #executeAndDecide(Client)} must have been called and completed + * successfully before calling this method. After this call, the attempt cannot be aborted anymore. + * This method is idempotent once the first call returns successfully. + * + * @return the response of the attempt + * + * @see #executeAndDecide(Client) + */ public HttpResponse commit() { + checkState(retryEventLoop.inEventLoop()); + if (state == State.COMMITTED) { + assert res != null; return res; } checkState(state == State.DECIDED); + assert res != null; + assert cause == null : cause; state = State.COMMITTED; - - if (resWithContent != null) { - resWithContent.abort(); - } - return res; } - public void abort() { - abort(AbortedStreamException.get()); - } - + /** + * Aborts this attempt with the specified {@code cause}. + * {@link #executeAndDecide(Client)} must be called before this method is called. + * After this call, the attempt cannot be committed anymore. + * This method is idempotent once the first call returns successfully. + * + * @param cause the cause of the abortion + */ public void abort(Throwable cause) { + checkState(retryEventLoop.inEventLoop()); + if (state == State.ABORTED) { return; } - checkState(state == State.EXECUTING || state == State.DECIDING || state == State.DECIDED); + assert state == State.IDLE || + state == State.EXECUTING || + state == State.DECIDED : state; + assert this.cause == null : state; state = State.ABORTED; - if (resWithContent != null) { - resWithContent.abort(); - } - final RequestLogBuilder logBuilder = ctx.logBuilder(); // Set response content with null to make sure that the log is complete. logBuilder.responseContent(null, null); logBuilder.responseContentPreview(null); - res.abort(cause); - - whenDecidedFuture.completeExceptionally(cause); - } - CompletableFuture<@Nullable RetryDecision> whenDecided() { - return whenDecidedFuture; + if (res != null) { + res.abort(cause); + } else { + // If we are in EXECUTING or PREPARING_AND_DECIDING, the execution/preparation must have failed. + } } + /** + * Returns the {@link ClientRequestContext} of this attempt. + * + * @return the {@link ClientRequestContext} of this attempt + */ ClientRequestContext ctx() { return ctx; } + /** + * Returns the {@link State} of this attempt. + * + * @return the {@link State} of this attempt + */ State state() { return state; } - long retryAfterMillis() { - assert state == State.DECIDED; - - final RequestLogAccess attemptLog = ctx.log(); - final String retryAfterValue; - final RequestLog requestLog = attemptLog.getIfAvailable(RequestLogProperty.RESPONSE_HEADERS); - retryAfterValue = requestLog != null ? - requestLog.responseHeaders().get(HttpHeaderNames.RETRY_AFTER) : null; - - if (retryAfterValue != null) { - try { - return Duration.ofSeconds(Integer.parseInt(retryAfterValue)).toMillis(); - } catch (Exception ignored) { - // Not a second value. - } - - try { - @SuppressWarnings("UseOfObsoleteDateTimeApi") - final Date retryAfterDate = DateFormatter.parseHttpDate(retryAfterValue); - if (retryAfterDate != null) { - return retryAfterDate.getTime() - System.currentTimeMillis(); - } - } catch (Exception ignored) { - // `parseHttpDate()` can raise an exception rather than returning `null` - // when the given value has more than 64 characters. - } + private CompletableFuture prepareAggregatedResponse(HttpResponse res) { + return res.aggregate() + .thenCompose(aggRes -> { + // NOTE: Might not run on the retry event loop. + completeLogIfBytesNotTransferred(aggRes); + return ctx.log() + .whenAvailable(RequestLogProperty.RESPONSE_END_TIME) + .thenApply(unused -> + // NOTE: Might not run on the retry event loop. + PreparationResult.ofSuccess( + aggRes.toHttpResponse(), + aggRes.toHttpResponse() + ) + ); + }).handle((preparationResult, resCause) -> { + // NOTE: Might not run on the retry event loop. + if (resCause == null) { + return UnmodifiableFuture.completedFuture(preparationResult); + } - logger.debug("The retryAfter: {}, from the server is neither an HTTP date nor a second.", - retryAfterValue); - } + resCause = Exceptions.peel(resCause); - return -1; + ctx.logBuilder().endRequest(resCause); + ctx.logBuilder().endResponse(resCause); + return UnmodifiableFuture.completedFuture(PreparationResult.ofFailure(resCause)); + }) + .thenCompose(Function.identity()); } - private void handleAggRes() { - assert state == State.EXECUTING; - - res.aggregate().handle((aggRes, resCause) -> { - if (resCause != null) { - ctx.logBuilder().endRequest(resCause); - ctx.logBuilder().endResponse(resCause); - complete(HttpResponse.ofFailure(resCause), resCause); - } else { - completeLogIfBytesNotTransferred(aggRes); - ctx.log().whenAvailable(RequestLogProperty.RESPONSE_END_TIME).thenRun(() -> { - completeWithContent(aggRes.toHttpResponse(), aggRes.toHttpResponse()); - }); - } - return null; - }); - } - - private void handleStreamingRes() { - assert state == State.EXECUTING; - + private CompletableFuture prepareStreamingResponse(HttpResponse res) { final SplitHttpResponse splitRes = res.split(); - splitRes.headers().handle((resHeaders, headersCause) -> { - assert state == State.EXECUTING; - - final Throwable resCause; - if (headersCause == null) { - final RequestLog log = ctx.log().getIfAvailable(RequestLogProperty.RESPONSE_CAUSE); - resCause = log != null ? log.responseCause() : null; - } else { - resCause = Exceptions.peel(headersCause); - } - completeLogIfBytesNotTransferred(resHeaders, resCause); - - ctx.log().whenAvailable(RequestLogProperty.RESPONSE_HEADERS).thenRun(() -> { - assert state == State.EXECUTING; - - if (rctx.config().needsContentInRule() && resCause == null) { - final HttpResponse unsplitRes = splitRes.unsplit(); - final HttpResponseDuplicator resDuplicator = - unsplitRes.toDuplicator(ctx.eventLoop().withoutContext(), - ctx.maxResponseLength()); - final HttpResponse duplicatedRes = resDuplicator.duplicate(); - final TruncatingHttpResponse truncatingAttemptRes = - new TruncatingHttpResponse(resDuplicator.duplicate(), - rctx.config().maxContentLength()); - resDuplicator.close(); - completeWithContent(duplicatedRes, truncatingAttemptRes); - } else { - if (resCause != null) { - splitRes.body().abort(resCause); - complete(HttpResponse.ofFailure(resCause), resCause); - } else { - complete(splitRes.unsplit(), null); - } - } - }); - return null; - }); + return splitRes.headers() + .handle((resHeaders, headersCause) -> { + // NOTE: Might not run on the retry event loop. + final Throwable resCause; + if (headersCause == null) { + final RequestLog log = ctx.log().getIfAvailable( + RequestLogProperty.RESPONSE_CAUSE); + resCause = log != null ? log.responseCause() : null; + } else { + resCause = Exceptions.peel(headersCause); + } + + final HttpResponse unsplitRes = splitRes.unsplit(); + completeLogIfBytesNotTransferred(unsplitRes, resHeaders, resCause); + + return ctx.log() + // NOTE: .whenAvailable is guaranteed to never complete exceptionally. + .whenAvailable(RequestLogProperty.RESPONSE_HEADERS) + .thenApply(unused -> { + // NOTE: Might not run on the retry event loop. + if (resCause != null) { + unsplitRes.abort(resCause); + return PreparationResult.ofFailure(resCause); + } + + if (config.needsContentInRule()) { + try ( + HttpResponseDuplicator resDuplicator = + unsplitRes.toDuplicator( + ctx.eventLoop().withoutContext(), + ctx.maxResponseLength() + ) + ) { + return PreparationResult.ofSuccess( + resDuplicator.duplicate(), + new TruncatingHttpResponse( + resDuplicator.duplicate(), + config.maxContentLength() + ) + ); + } + } else { + return PreparationResult.ofSuccess( + unsplitRes, + unsplitRes + ); + } + }); + }) + .thenCompose(Function.identity()); } private void completeLogIfBytesNotTransferred(AggregatedHttpResponse aggRes) { - assert state == State.EXECUTING; - if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { final RequestLogBuilder attemptLogBuilder = ctx.logBuilder(); attemptLogBuilder.endRequest(); @@ -276,10 +527,9 @@ private void completeLogIfBytesNotTransferred(AggregatedHttpResponse aggRes) { } private void completeLogIfBytesNotTransferred( + HttpResponse res, @Nullable ResponseHeaders headers, @Nullable Throwable resCause) { - assert state == State.EXECUTING; - if (!ctx.log().isAvailable(RequestLogProperty.REQUEST_FIRST_BYTES_TRANSFERRED_TIME)) { final RequestLogBuilder logBuilder = ctx.logBuilder(); if (resCause != null) { @@ -302,77 +552,89 @@ private void completeLogIfBytesNotTransferred( } } - private void completeWithContent(HttpResponse res, HttpResponse resWithContent) { - assert state == State.EXECUTING; - - this.res = res; - this.resWithContent = resWithContent; - resCause = null; - decide(); - } - - private void complete(HttpResponse res, @Nullable Throwable resCause) { - assert state == State.EXECUTING; - - if (resCause != null) { - resCause = Exceptions.peel(resCause); + /** + * Calls {@link RetryRule#shouldRetry(ClientRequestContext, Throwable)} to receive a {@link RetryDecision}. + * + * @param resForRule the response to be delivered to the {@link RetryRule} for decision. + * @param causeForRule the cause to be delivered to the {@link RetryRule} for decision. + * @return a future that will be completed with the {@link RetryDecision} or an exception if failed during + * the decision. + */ + private CompletableFuture decide(@Nullable HttpResponse resForRule, + @Nullable Throwable causeForRule) { + if (causeForRule != null) { + resForRule = null; + causeForRule = Exceptions.peel(causeForRule); + } else { + assert resForRule != null; + causeForRule = null; } - this.res = res; - resWithContent = null; - this.resCause = resCause; - decide(); - } - - private CompletionStage<@Nullable RetryDecision> shouldBeRetriedWith(RetryRule retryRule) { - assert state == State.DECIDING; + final CompletionStage<@Nullable RetryDecision> maybeDecision; - try { - return retryRule.shouldRetry(ctx, resCause); - } catch (Throwable t) { - return UnmodifiableFuture.exceptionallyCompletedFuture(t); + if (config.needsContentInRule()) { + final RetryRuleWithContent retryRuleWithContent = + config.retryRuleWithContent(); + assert retryRuleWithContent != null; + try { + maybeDecision = retryRuleWithContent.shouldRetry(ctx, resForRule, causeForRule); + } catch (Throwable t) { + return UnmodifiableFuture.exceptionallyCompletedFuture(t); + } + } else { + final RetryRule retryRule = config.retryRule(); + assert retryRule != null; + try { + maybeDecision = retryRule.shouldRetry(ctx, causeForRule); + } catch (Throwable t) { + return UnmodifiableFuture.exceptionallyCompletedFuture(t); + } } - } - - private CompletionStage<@Nullable RetryDecision> shouldBeRetriedWith( - RetryRuleWithContent retryRuleWithContent) { - assert state == State.DECIDING; - @Nullable - final HttpResponse resForRule; - @Nullable - final Throwable causeForRule; + // Remember that RetryRule.shouldRetry could be client code so we do have any guarantees + // on which thread we are completing. Let us make sure we are completing on running on the + // retry event loop again. + return withCompletionOnRetryEventLoop( + maybeDecision.thenApply( + decision -> decision == null ? RetryDecision.noRetry() : decision + ) + ); + } - if (resCause != null) { - resForRule = null; - causeForRule = resCause; - } else { - if (resWithContent == null) { - resForRule = res; + private CompletableFuture withCompletionOnRetryEventLoop(CompletionStage future) { + final CompletableFuture futureOnTheRetryEventLoop = new CompletableFuture<>(); + future.whenComplete((result, cause) -> { + if (retryEventLoop.inEventLoop()) { + if (cause != null) { + futureOnTheRetryEventLoop.completeExceptionally(cause); + } else { + futureOnTheRetryEventLoop.complete(result); + } } else { - resForRule = resWithContent; + retryEventLoop.execute(() -> { + if (cause != null) { + futureOnTheRetryEventLoop.completeExceptionally(cause); + } else { + futureOnTheRetryEventLoop.complete(result); + } + }); } - - causeForRule = null; - } - - try { - return retryRuleWithContent.shouldRetry(ctx, resForRule, causeForRule); - } catch (Throwable t) { - return UnmodifiableFuture.exceptionallyCompletedFuture(t); - } + }); + return futureOnTheRetryEventLoop; } @Override public String toString() { - // We omit `rctx` to keep the representation compact. + checkState(retryEventLoop.inEventLoop()); + return MoreObjects .toStringHelper(this) .add("state", state) + .add("attemptNumber", attemptNumber) .add("ctx", ctx) + .add("req", req) .add("res", res) - .add("resCause", resCause) - .add("resWithContent", resWithContent) + .add("cause", cause) .toString(); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java deleted file mode 100644 index 1fd6c0816b6..00000000000 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryingContext.java +++ /dev/null @@ -1,404 +0,0 @@ -/* - * Copyright 2025 LY Corporation - * - * LY Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.client.retry; - -import static com.google.common.base.Preconditions.checkState; -import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; -import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; -import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.MoreObjects; - -import com.linecorp.armeria.client.Client; -import com.linecorp.armeria.client.ClientRequestContext; -import com.linecorp.armeria.client.ResponseTimeoutException; -import com.linecorp.armeria.common.AggregationOptions; -import com.linecorp.armeria.common.HttpRequest; -import com.linecorp.armeria.common.HttpRequestDuplicator; -import com.linecorp.armeria.common.HttpResponse; -import com.linecorp.armeria.common.RequestHeadersBuilder; -import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.stream.AbortedStreamException; -import com.linecorp.armeria.common.util.TimeoutMode; -import com.linecorp.armeria.common.util.UnmodifiableFuture; -import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; -import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; -import com.linecorp.armeria.internal.client.ClientRequestContextExtension; -import com.linecorp.armeria.internal.client.ClientUtil; - -final class HttpRetryingContext implements RetryingContext { - private static final Logger logger = LoggerFactory.getLogger(HttpRetryingContext.class); - - private enum State { - UNINITIALIZED, - INITIALIZING, - INITIALIZED, - EXECUTING, - COMPLETED - } - - private final Object lock = new Object(); - - private State state; - private final ClientRequestContext ctx; - private final RetryConfig config; - private final HttpResponse res; - private final CompletableFuture resFuture; - private final HttpRequest req; - @Nullable - private HttpRequestDuplicator reqDuplicator; - private final RetryCounter counter; - private final RetryScheduler scheduler; - - private final long deadlineTimeNanos; - private final boolean hasDeadline; - - private final boolean useRetryAfter; - @Nullable - HttpRetryAttempt currentAttempt; - - HttpRetryingContext(ClientRequestContext ctx, - RetryConfig config, - HttpResponse res, - CompletableFuture resFuture, - HttpRequest req, - boolean useRetryAfter) { - - state = State.UNINITIALIZED; - this.ctx = ctx; - this.config = config; - this.resFuture = resFuture; - this.res = res; - this.req = req; - reqDuplicator = null; // will be initialized in init(). - counter = new RetryCounter(config.maxTotalAttempts()); - scheduler = new RetryScheduler(ctx); - - final long responseTimeoutMillis = ctx.responseTimeoutMillis(); - if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { - deadlineTimeNanos = 0; - hasDeadline = false; - } else { - deadlineTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); - hasDeadline = true; - } - - this.useRetryAfter = useRetryAfter; - currentAttempt = null; - } - - @Override - public CompletableFuture init() { - synchronized (lock) { - checkState(state == State.UNINITIALIZED); - state = State.INITIALIZING; - - final CompletableFuture initFuture = new CompletableFuture<>(); - - if (ctx.exchangeType().isRequestStreaming()) { - reqDuplicator = - req.toDuplicator(ctx.eventLoop().withoutContext(), 0); - state = State.INITIALIZED; - initFuture.complete(true); - } else { - req.aggregate(AggregationOptions.usePooledObjects(ctx.alloc(), ctx.eventLoop())) - .handle((agg, reqCause) -> { - synchronized (lock) { - assert state == State.INITIALIZING; - - if (reqCause != null) { - resFuture.completeExceptionally(reqCause); - ctx.logBuilder().endRequest(reqCause); - ctx.logBuilder().endResponse(reqCause); - state = State.COMPLETED; - initFuture.complete(false); - } else { - reqDuplicator = new AggregatedHttpRequestDuplicator(agg); - state = State.INITIALIZED; - initFuture.complete(true); - } - } - return null; - }); - } - - initFuture.whenComplete((initSuccessful, initCause) -> { - synchronized (lock) { - assert state == State.INITIALIZED || state == State.COMPLETED; - if (!initSuccessful || initCause != null) { - return; - } - - req.whenComplete().handle((unused, cause) -> { - if (cause != null) { - abort(cause); - } - return null; - }); - - res.whenComplete().handle((result, cause) -> { - final Throwable abortCause; - if (cause != null) { - abortCause = cause; - } else { - abortCause = AbortedStreamException.get(); - } - abort(abortCause); - return null; - }); - } - }); - - return initFuture; - } - } - - @Override - public CompletableFuture<@Nullable RetryDecision> executeAttempt( - @Nullable Backoff backoff, - Client delegate) { - synchronized (lock) { - checkState(state == State.INITIALIZED); - assert reqDuplicator != null; - // We are not supporting concurrent attempts (yet). - // As such, we expect the previous attempt (if any) to be aborted before - // (abortion sets this field to null). - // assert state != State.EXECUTING; - assert currentAttempt == null; - - if (!setResponseTimeout()) { - return UnmodifiableFuture.exceptionallyCompletedFuture(ResponseTimeoutException.get()); - } - - state = State.EXECUTING; - counter.recordAttemptWith(backoff); - - final int attemptNumber = counter.attemptNumber(); - final boolean isInitialAttempt = attemptNumber <= 1; - - final HttpRequest attemptReq; - final ClientRequestContext attemptCtx; - if (isInitialAttempt) { - attemptReq = reqDuplicator.duplicate(); - } else { - final RequestHeadersBuilder attemptHeadersBuilder = req.headers().toBuilder(); - attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, attemptNumber - 1); - attemptReq = reqDuplicator.duplicate(attemptHeadersBuilder.build()); - } - - attemptCtx = ClientUtil.newDerivedContext(ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); - - final HttpResponse attemptRes; - final ClientRequestContextExtension attemptCtxExt = - attemptCtx.as(ClientRequestContextExtension.class); - if (!isInitialAttempt && attemptCtxExt != null && attemptCtx.endpoint() == null) { - // clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); - // if the endpoint hasn't been selected, - // try to initialize the attempCtx with a new endpoint/event loop - attemptRes = initContextAndExecuteWithFallback( - delegate, attemptCtxExt, HttpResponse::of, - (context, cause) -> - HttpResponse.ofFailure(cause), attemptReq, false); - } else { - attemptRes = executeWithFallback(delegate, attemptCtx, - (context, cause) -> - HttpResponse.ofFailure(cause), attemptReq, false); - } - - final HttpRetryAttempt attempt = new HttpRetryAttempt(this, attemptCtx, attemptRes, - ctx.exchangeType().isResponseStreaming()); - currentAttempt = attempt; - return attempt.whenDecided(); - } - } - - // returns Long.MAX_VALUE if no retry is possible. - @Override - public long nextRetryTimeNanos(Backoff backoff) { - synchronized (lock) { - checkState(state == State.EXECUTING); - assert currentAttempt != null; - checkState(currentAttempt.state() == HttpRetryAttempt.State.DECIDED); - - if (counter.hasReachedMaxAttempts()) { - logger.debug("Exceeded the default number of max attempt: {}", config.maxTotalAttempts()); - return Long.MAX_VALUE; - } - - final long nextRetryDelayForBackoffMillis = backoff.nextDelayMillis( - counter.attemptNumberForBackoff(backoff) + 1); - - if (nextRetryDelayForBackoffMillis < 0) { - logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); - return Long.MAX_VALUE; - } - - final long nextRetryDelayMillis = Math.max(nextRetryDelayForBackoffMillis, - useRetryAfter ? currentAttempt.retryAfterMillis() : -1); - final long nextDelayTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos( - nextRetryDelayMillis); - - if (hasDeadline && nextDelayTimeNanos > deadlineTimeNanos) { - // The next retry will be after the response deadline. So return just Long.MAX_VALUE. - return Long.MAX_VALUE; - } - - return nextDelayTimeNanos; - } - } - - @Override - public void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, - Consumer exceptionHandler) { - synchronized (lock) { - checkState(state == State.INITIALIZED); - assert currentAttempt == null; - scheduler.scheduleNextRetry(nextRetryTimeNanos, retryTask, exceptionHandler); - } - } - - @Override - public void commit() { - synchronized (lock) { - if (state == State.COMPLETED) { - // Already completed. - return; - } - checkState(state == State.EXECUTING); - assert reqDuplicator != null; - assert currentAttempt != null; - checkState(currentAttempt.state() == HttpRetryAttempt.State.DECIDED); - - state = State.COMPLETED; - - final HttpResponse attemptRes = currentAttempt.commit(); - ctx.logBuilder().endResponseWithChild(currentAttempt.ctx().log()); - resFuture.complete(attemptRes); - reqDuplicator.close(); - } - } - - @Override - public void abortAttempt() { - synchronized (lock) { - checkState(state == State.EXECUTING); - checkState(currentAttempt != null, "No active attempt to abort"); - currentAttempt.abort(); - state = State.INITIALIZED; - currentAttempt = null; - } - } - - @Override - public void abort(Throwable cause) { - synchronized (lock) { - if (state == State.COMPLETED) { - // Already completed. - return; - } - - if (state == State.EXECUTING) { - assert currentAttempt != null; - currentAttempt.abort(); - } - - state = State.COMPLETED; - - if (reqDuplicator != null) { - reqDuplicator.abort(cause); - } - - if (!ctx.log().isRequestComplete()) { - ctx.logBuilder().endRequest(cause); - } - ctx.logBuilder().endResponse(cause); - - resFuture.completeExceptionally(cause); - } - } - - @Override - public HttpResponse res() { - return res; - } - - RetryConfig config() { - return config; - } - - private boolean setResponseTimeout() { - final long responseTimeoutMillis = responseTimeoutMillis(); - if (responseTimeoutMillis < 0) { - return false; - } else if (responseTimeoutMillis == 0) { - ctx.clearResponseTimeout(); - return true; - } else { - ctx.setResponseTimeoutMillis(TimeoutMode.SET_FROM_NOW, responseTimeoutMillis); - return true; - } - } - - private long responseTimeoutMillis() { - if (!hasDeadline) { - return config.responseTimeoutMillisForEachAttempt(); - } - - final long remainingTimeUntilDeadlineMillis = TimeUnit.NANOSECONDS.toMillis( - deadlineTimeNanos - System.nanoTime()); - - // Consider 0 or less than 0 of actualResponseTimeoutMillis as timed out. - if (remainingTimeUntilDeadlineMillis <= 0) { - return -1; - } - - if (config.responseTimeoutMillisForEachAttempt() > 0) { - return Math.min(config.responseTimeoutMillisForEachAttempt(), - remainingTimeUntilDeadlineMillis); - } - - return remainingTimeUntilDeadlineMillis; - } - - @Override - public String toString() { - synchronized (lock) { - return MoreObjects - .toStringHelper(this) - .add("state", state) - .add("ctx", ctx) - .add("config", config) - .add("req", req) - .add("res", res) - .add("useRetryAfter", useRetryAfter) - .add("deadlineTimeNanos", deadlineTimeNanos) - .add("hasDeadline", hasDeadline) - .add("counter", counter) - .add("scheduler", scheduler) - .add("currentAttempt", currentAttempt) - .toString(); - } - } -} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java new file mode 100644 index 00000000000..300a60f1208 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java @@ -0,0 +1,402 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.client.retry; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; + +import java.util.ArrayList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeUnit; + +import com.linecorp.armeria.client.Client; +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.client.ResponseTimeoutException; +import com.linecorp.armeria.common.AggregationOptions; +import com.linecorp.armeria.common.ContextAwareEventLoop; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.common.HttpRequestDuplicator; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.RequestHeadersBuilder; +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.stream.AbortedStreamException; +import com.linecorp.armeria.common.util.TimeoutMode; +import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; +import com.linecorp.armeria.internal.client.ClientUtil; + +class RetriedHttpRequest implements RetriedRequest { + private enum State { + /** + * Initial state. {@link RetriedHttpRequest} is instantiated but not yet initialized. + */ + IDLE, + /** + * {@link RetriedHttpRequest} is being initialized in {@link #init()} with the first call + * to {@link #executeAttempt(Client)}. See the javadoc of {@link #init()} for details. + */ + INITIALIZING, + /** + * {@link RetriedHttpRequest} is initialized and is ready to execute/ is executing attempts. + */ + PENDING, + /** + * Terminal state. Either {@link #commit(int)} or {@link #abort(Throwable)} was called. + * {@link RetriedHttpRequest} will not execute any more attempts in that it completes every call + * to {@link #executeAttempt(Client)} exceptionally with an {@link AbortedAttemptException}. + */ + COMPLETED + } + + private final ContextAwareEventLoop retryEventLoop; + private final RetryConfig config; + private final ClientRequestContext ctx; + private final HttpResponse res; + private final CompletableFuture resFuture; + private final HttpRequest req; + + private final long deadlineTimeNanos; + private final boolean useRetryAfter; + + private State state; + + /** + * Future that completes when the initialization is done. It always completes successfully. + */ + private final CompletableFuture<@Nullable Void> initFuture; + + int previousAttemptNumber; + + /** + * Duplicator for the original request. It is set from {@code State.PENDING} onwards. + */ + @Nullable + private HttpRequestDuplicator reqDuplicator; + + final ArrayList attempts; + + RetriedHttpRequest( + ContextAwareEventLoop retryEventLoop, + RetryConfig config, + ClientRequestContext ctx, + HttpRequest req, + HttpResponse res, CompletableFuture resFuture, boolean useRetryAfter + ) { + this.retryEventLoop = retryEventLoop; + this.ctx = ctx; + this.res = res; + this.resFuture = resFuture; + this.req = req; + this.config = config; + this.useRetryAfter = useRetryAfter; + initFuture = new CompletableFuture<>(); + + final long responseTimeoutMillis = ctx.responseTimeoutMillis(); + if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { + deadlineTimeNanos = Long.MAX_VALUE; + } else { + deadlineTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); + } + + state = State.IDLE; + previousAttemptNumber = 0; + // We can assume that most of the time we will succeed with the first attempt. + attempts = new ArrayList<>(1); + reqDuplicator = null; + } + + @Override + public CompletionStage executeAttempt( + Client delegate) { + checkState(ctx.eventLoop().inEventLoop()); + return init().thenCompose(unused -> executeAttemptAfterInit(delegate)); + } + + /** + * Initializes the {@link RetriedHttpRequest}. This method is idempotent. + * Initialization includes + * - subscribing to {@link #req} to initiate abortion in case its fails, + * - subscribing to {@link #res} to initiate abortion in case it fails or completes, and + * - building and setting {@link #reqDuplicator}. + * + * @return a future that completes when the initialization is done. After the future completes + * the {@link RetriedHttpRequest} is either in {@code PENDING} or {@code COMPLETED} state. + * */ + private CompletableFuture init() { + assert ctx.eventLoop().inEventLoop(); + + if (state == State.INITIALIZING || state == State.PENDING || state == State.COMPLETED) { + return initFuture; + } + + assert state == State.IDLE : state; + state = State.INITIALIZING; + + // We are not putting this in the constructor as we might get aborted before + // we are properly initialized. + req.whenComplete().handle((unused, cause) -> { + if (cause != null) { + if (retryEventLoop.inEventLoop()) { + abort(cause); + } else { + retryEventLoop.execute(() -> abort(cause)); + } + } + return null; + }); + + res.whenComplete().handle((result, cause) -> { + final Throwable abortCause; + if (cause != null) { + abortCause = cause; + } else { + abortCause = AbortedStreamException.get(); + } + if (retryEventLoop.inEventLoop()) { + abort(abortCause); + } else { + retryEventLoop.execute(() -> abort(abortCause)); + } + return null; + }); + + // Guaranteed to be completed on the retryEventLoop. + final CompletableFuture reqDuplicatorFuture; + + if (ctx.exchangeType().isRequestStreaming()) { + reqDuplicatorFuture = UnmodifiableFuture.completedFuture( + req.toDuplicator(retryEventLoop.withoutContext(), 0)); + } else { + reqDuplicatorFuture = req.aggregate( + // TODO: Do we need to run this with the context? + AggregationOptions.usePooledObjects(ctx.alloc(), retryEventLoop)) + .thenApply(AggregatedHttpRequestDuplicator::new); + } + + reqDuplicatorFuture + .thenAccept( + reqDuplicator0 -> { + assert retryEventLoop.inEventLoop(); + assert reqDuplicator == null : reqDuplicator; + + if (state == State.COMPLETED) { + reqDuplicator0.close(); + } else { + assert state == State.INITIALIZING : state; + reqDuplicator = reqDuplicator0; + state = State.PENDING; + } + } + ) + .exceptionally(cause -> { + abort(cause); + return null; + }) + .thenRun(() -> initFuture.complete(null)); + + return initFuture; + } + + private CompletableFuture executeAttemptAfterInit( + Client delegate) { + assert retryEventLoop.inEventLoop(); + + if (state == State.COMPLETED) { + return UnmodifiableFuture.exceptionallyCompletedFuture(AbortedAttemptException.get()); + } + + // We need to be initialized/ the reqDuplicator must be present. + checkState(state == State.PENDING); + assert reqDuplicator != null; + checkState(previousAttemptNumber < config.maxTotalAttempts()); + + final int attemptNumber = ++previousAttemptNumber; + + assert attemptNumber == attempts.size() + 1 : attemptNumber + ", " + attempts.size(); + assert attemptNumber <= config.maxTotalAttempts() : attemptNumber + ", " + config.maxTotalAttempts(); + + if (!checkAndSetResponseTimeout()) { + final ResponseTimeoutException timeoutException = ResponseTimeoutException.get(); + abort(timeoutException); + return UnmodifiableFuture.exceptionallyCompletedFuture(timeoutException); + } + + final HttpRetryAttempt attempt = newAttempt(attemptNumber); + attempts.add(attempt); + return attempt + .executeAndDecide(delegate) + .thenApply(decision -> { + assert retryEventLoop.inEventLoop(); + return new AttemptExecutionResult( + attemptNumber, + decision, + useRetryAfter ? + ClientUtil.retryAfterMillis(attempt.ctx().log()) + : -1L + ); + }) + .exceptionally(cause -> { + assert retryEventLoop.inEventLoop(); + abort(cause); + return null; + }); + } + + private HttpRetryAttempt newAttempt(int attemptNumber) { + assert reqDuplicator != null; + + final boolean isInitialAttempt = attemptNumber <= 1; + + final HttpRequest attemptReq; + final ClientRequestContext attemptCtx; + if (isInitialAttempt) { + attemptReq = reqDuplicator.duplicate(); + } else { + final RequestHeadersBuilder attemptHeadersBuilder = req.headers().toBuilder(); + attemptHeadersBuilder.setInt(ARMERIA_RETRY_COUNT, attemptNumber - 1); + attemptReq = reqDuplicator.duplicate(attemptHeadersBuilder.build()); + } + + attemptCtx = ClientUtil.newDerivedContext(ctx, attemptReq, ctx.rpcRequest(), isInitialAttempt); + + return new HttpRetryAttempt(config, retryEventLoop, attemptNumber, attemptCtx, attemptReq); + } + + @Override + public void commit(int attemptNumber) { + checkState(retryEventLoop.inEventLoop()); + + if (state == State.COMPLETED) { + return; + } + checkState(state == State.PENDING); + assert reqDuplicator != null; + + checkArgument(attemptNumber >= 1); + checkArgument(attemptNumber <= attempts.size()); + + final HttpRetryAttempt attemptToCommit = attempts.get(attemptNumber - 1); + + checkState(attemptToCommit != null); + checkState(attemptToCommit.state() == HttpRetryAttempt.State.DECIDED); + + state = State.COMPLETED; + + abortAllExcept(attemptToCommit); + + reqDuplicator.close(); + final HttpResponse attemptRes = attemptToCommit.commit(); + ctx.logBuilder().endResponseWithChild(attemptToCommit.ctx().log()); + resFuture.complete(attemptRes); + } + + @Override + public void abort(Throwable cause) { + checkState(retryEventLoop.inEventLoop()); + + if (state == State.COMPLETED) { + return; + } + + state = State.COMPLETED; + + abortAllExcept(null); + + if (reqDuplicator != null) { + reqDuplicator.close(); + } + + if (!ctx.log().isRequestComplete()) { + ctx.logBuilder().endRequest(cause); + } + ctx.logBuilder().endResponse(cause); + resFuture.completeExceptionally(cause); + } + + @Override + public void abort(int attemptNumber, Throwable cause) { + checkState(retryEventLoop.inEventLoop()); + checkArgument(attemptNumber >= 1); + + final HttpRetryAttempt attemptToAbort; + if (state == State.COMPLETED) { + return; + } + + checkState(state == State.PENDING); + checkArgument(attemptNumber >= 1); + checkArgument(attemptNumber <= attempts.size()); + + attemptToAbort = attempts.get(attemptNumber - 1); + + checkState(attemptToAbort != null); + + attemptToAbort.abort(cause); + } + + private void abortAllExcept(@Nullable HttpRetryAttempt winningAttempt) { + assert retryEventLoop.inEventLoop(); + assert state == State.COMPLETED : state; + + for (final HttpRetryAttempt attempt : attempts) { + if (attempt == winningAttempt) { + continue; + } + + attempt.abort(AbortedAttemptException.get()); + } + } + + @Override + public HttpResponse res() { + return res; + } + + public long deadlineTimeNanos() { + return deadlineTimeNanos; + } + + private boolean checkAndSetResponseTimeout() { + long remainingTimeUntilDeadlineMillis = Long.MAX_VALUE; + boolean hasTimeout = false; + + if (deadlineTimeNanos < Long.MAX_VALUE) { + hasTimeout = true; + remainingTimeUntilDeadlineMillis = TimeUnit.NANOSECONDS.toMillis( + deadlineTimeNanos - System.nanoTime()); + } + + if (config.responseTimeoutMillisForEachAttempt() != 0) { + hasTimeout = true; + remainingTimeUntilDeadlineMillis = + Math.min(remainingTimeUntilDeadlineMillis, config.responseTimeoutMillisForEachAttempt()); + } + + if (hasTimeout) { + if (remainingTimeUntilDeadlineMillis > 0) { + ctx.setResponseTimeoutMillis(TimeoutMode.SET_FROM_NOW, remainingTimeUntilDeadlineMillis); + return true; + } else { + return false; + } + } else { + ctx.clearResponseTimeout(); + return true; + } + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java new file mode 100644 index 00000000000..04496f4d9af --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java @@ -0,0 +1,150 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.client.retry; + +import java.util.concurrent.CompletionStage; + +import com.linecorp.armeria.client.Client; +import com.linecorp.armeria.common.Request; +import com.linecorp.armeria.common.Response; +import com.linecorp.armeria.common.logging.RequestLog; + +/** + * A retried request that manages multiple retry attempts. + * + *

+ * NOTE: All methods of {@link RetryScheduler} must be invoked from the + * {@code retryEventLoop}. Implementations of {@link RetriedRequest} therefore do not need to be thread-safe. + *

+ * + * @param the {@link Request} type + * @param the {@link Response} type + */ +interface RetriedRequest { + /** + * The result of an attempt execution. + */ + final class AttemptExecutionResult { + private final int attemptNumber; + private final RetryDecision decision; + private final long minimumBackoffMillis; + + AttemptExecutionResult(int attemptNumber, RetryDecision decision, + long minimumBackoffMillis) { + this.attemptNumber = attemptNumber; + this.decision = decision; + this.minimumBackoffMillis = minimumBackoffMillis; + } + + /** + * Returns the attempt number. It can be used to {@link RetriedRequest#abort(int, Throwable)} or + * {@link RetriedRequest#commit(int)} the attempt. + * + * @return the attempt number. + */ + public int attemptNumber() { + return attemptNumber; + } + + /** + * Returns the decision made by {@link RetryRule} from the {@link RetryConfig} on the attempt response. + * The attempt response was prepared enough for the {@link RetryRule} to make a decision. This could + * include awaiting receiving headers, content, or trailers for example. + * + * @return the decision made by the {@link RetryRule} + */ + public RetryDecision decision() { + return decision; + } + + /** + * Returns the minimum backoff in milliseconds for the next retry attempt + * returned by the remote peer that received the attempt. + * If negative, the minimum backoff is either not available or not requested. + * + * @return the minimum backoff in milliseconds. If negative, no minimum backoff should be applied for + * the next retry attempt. + */ + public long minimumBackoffMillis() { + return minimumBackoffMillis; + } + } + + /** + * Tries to execute a new attempt on {@code delegate} based on the {@link RetryConfig}. + * + *

+ * It completes with a {@link AttemptExecutionResult} if the attempt was successfully executed and decided + * by the {@link RetryRule} from the {@link RetryConfig}. + *

+ * + *

+ * It completed exceptionally in case something unexpected happened during the attempt execution. + * In particular, it completes exceptionally with a {@link AbortedAttemptException} if the attempt + * was aborted by the {@link RetriedRequest}. This could happen if the {@link RetriedRequest} completes + * concurrently while the attempt is being executed. + *

+ * + * @param delegate the next {@link Client} in the decoration chain + * @return a future completed with the result of the attempt execution. It completes with a + * {@link AbortedAttemptException} in case the attempt was aborted in a controlled way. + */ + CompletionStage executeAttempt(Client delegate); + + /** + * Commits the attempt with the specified {@code attemptNumber}. Committing an attempt includes: + *
    + *
  • completing the original request with the attempt's response,
    • + *
  • aborting all other pending attempts, and
    • + *
  • marking the {@link RetriedRequest} as completed.
    • + *
+ * Does nothing if the {@link RetriedRequest} is already completed. + * + * @param attemptNumber the attempt number to commit + */ + void commit(int attemptNumber); + + /** + * Aborts the attempt identified by the given {@code attemptNumber}. + * + *

The attempt must have been previously started via + * {@link #executeAttempt(Client)}. Aborting an attempt ensures that its + * response and {@link RequestLog} + * complete, but that it will not be committed as the final result of the + * {@link RetriedRequest}.

+ * + *

If the {@link RetriedRequest} is already completed, this method has no effect. + * If the specified attempt has not been started, an {@link IllegalStateException} is thrown.

+ * + * @param attemptNumber the identifier of the attempt to abort + * @throws IllegalStateException if the attempt has not been executed + */ + void abort(int attemptNumber, Throwable cause); + + /** + * Aborts all attempts and completes the {@link RetriedRequest} exceptionally with the specified + * {@code cause}. Marks the {@link RetriedRequest} as completed. Does nothing if the {@link RetriedRequest} + * was already completed before. + */ + void abort(Throwable cause); + + /** + * Returns the original {@link Response} that is completed when the {@link RetriedRequest} is completed. + * @return the {@link Response} with either a committed attempt's response or with an exception from a call + * to {@link #abort(Throwable)} or from an unexpected error during the processing of an attempt. + */ + O res(); +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java index e949bd53341..f3360267974 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryCounter.java @@ -16,71 +16,40 @@ package com.linecorp.armeria.client.retry; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static java.util.Objects.requireNonNull; - -import com.google.common.base.MoreObjects; - import com.linecorp.armeria.common.annotation.Nullable; -final class RetryCounter { - private final int maxAttempts; - - private int numberAttemptsSoFar; - @Nullable - private Backoff lastBackoff; - private int numberAttemptsSoFarForLastBackoff; - - RetryCounter(int maxAttempts) { - checkArgument(maxAttempts > 0, "maxAttempts: %s (expected: > 0)", maxAttempts); - this.maxAttempts = maxAttempts; - numberAttemptsSoFar = 0; - lastBackoff = null; - numberAttemptsSoFarForLastBackoff = 0; - } - - void recordAttemptWith(@Nullable Backoff backoff) { - checkState(!hasReachedMaxAttempts(), "Exceeded the maximum number of attempts: %s", maxAttempts); - - ++numberAttemptsSoFar; - - if (backoff != null) { - if (lastBackoff != backoff) { - lastBackoff = backoff; - numberAttemptsSoFarForLastBackoff = 0; - } - numberAttemptsSoFarForLastBackoff++; - } else { - assert lastBackoff == null; - } - } - - int attemptNumber() { - return numberAttemptsSoFar; - } - - int attemptNumberForBackoff(Backoff backoff) { - requireNonNull(backoff, "backoff"); - if (lastBackoff != backoff) { - return 0; - } else { - return numberAttemptsSoFarForLastBackoff; - } - } - - boolean hasReachedMaxAttempts() { - return numberAttemptsSoFar >= maxAttempts; - } - - @Override - public String toString() { - return MoreObjects - .toStringHelper(this) - .add("maxAttempts", maxAttempts) - .add("numberAttemptsSoFar", numberAttemptsSoFar) - .add("lastBackoff", lastBackoff) - .add("numberAttemptsSoFarForLastBackoff", numberAttemptsSoFarForLastBackoff) - .toString(); - } +/** + * A counter that keeps track of the number of attempts made so far for a {@link RetryingClient}. + * In particular, it keeps track of the total number of attempts but also + * the number of attempts made with each {@link Backoff}. + * + *

+ * Implementors do not need to be thread-safe. + *

+ */ +interface RetryCounter { + /** + * Records an attempt in that it increases the total number of attempts by one. If {@code backoff} is not + * {@code null}, the number of attempts for that {@code backoff} is increased as well. + * + * @param backoff the backoff used for the attempt, or {@code null} if no backoff was used. + */ + void consumeAttemptFrom(@Nullable Backoff backoff); + + /** + * Returns the number of attempts executed so far with {@code backoff}. + * + * @param backoff the backoff whose number of attempts is requested + * @return the number of attempts executed so far with {@code backoff} + */ + int attemptsSoFarWithBackoff(Backoff backoff); + + /** + * Returns {@code true} if the total number of attempts has reached the maximum number of attempts + * configured in the {@link RetryConfig}. + * + * @return {@code true} if the total number of attempts has reached the maximum number of attempts + * configured in the {@link RetryConfig} + */ + boolean hasReachedMaxAttempts(); } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index e69e99b69fc..03e5410e08a 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -15,50 +15,72 @@ */ package com.linecorp.armeria.client.retry; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -import com.linecorp.armeria.client.ClientFactory; -import com.linecorp.armeria.client.ClientRequestContext; - -import io.netty.util.concurrent.ScheduledFuture; +import java.util.concurrent.CompletableFuture; /** - * Schedules retry tasks using the {@link ClientRequestContext}'s event loop. + * A scheduler that runs retry tasks sequentially on a designated event loop, + * while enforcing a fixed request deadline and minimum backoff between attempts. + * + *

+ * Sequential means that {@link #trySchedule(Runnable, long)} can only be called with not other retry task + * scheduled (i.e. with {@link #hasNextRetryTask()} returning {@code false}). + *

+ * + *

+ * NOTE: All methods of {@link RetryScheduler} must be invoked from the + * {@code retryEventLoop}. Implementations of {@link RetryScheduler} therefore do not need to be thread-safe. + *

*/ -final class RetryScheduler { - private final ClientRequestContext ctx; +interface RetryScheduler { + /** + * Tries to schedule the given {@code retryTask} to be run after the given {@code delayMillis} + * or as early as the minimum backoff delays applied for the next schedule allow. Scheduling might + * not succeed which can be checked by calling {@link #hasNextRetryTask()}. + * + * @param retryTask the task to run to perform a retry + * @param delayMillis the delay in milliseconds after which the {@code retryTask} should be run + */ + void trySchedule(Runnable retryTask, long delayMillis); + + /** + * Returns {@code true} if there is a retry task scheduled. If this method returns {@code true}, + * the scheduler guarantees that it is going to run a retry task at some point in the future + * on the retryEventLoop. If it is not possible to run a retry task, the scheduler guarantees + * to complete the future from {@link #whenClosed()} exceptionally. + * + * @return {@code true} if there is a retry task scheduled + */ + boolean hasNextRetryTask(); - RetryScheduler(ClientRequestContext ctx) { - this.ctx = ctx; - } + /** + * Closes the scheduler which is cancelling the next retry task if any and + * completing the future from {@link #whenClosed()}. + */ + void close(); - void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, - Consumer exceptionHandler) { - try { - final long nextRetryDelayMillis = TimeUnit.NANOSECONDS.toMillis( - nextRetryTimeNanos - System.nanoTime()); - if (nextRetryDelayMillis <= 0) { - ctx.eventLoop().execute(retryTask); - } else { - @SuppressWarnings("unchecked") - final ScheduledFuture scheduledFuture = (ScheduledFuture) ctx - .eventLoop().schedule(retryTask, nextRetryDelayMillis, TimeUnit.MILLISECONDS); - scheduledFuture.addListener(future -> { - if (future.isCancelled()) { - // future is cancelled when the client factory is closed. - exceptionHandler.accept(new IllegalStateException( - ClientFactory.class.getSimpleName() + " has been closed.")); - } else if (future.cause() != null) { - // Other unexpected exceptions. - exceptionHandler.accept(future.cause()); - } - }); - } - } catch (Throwable t) { - exceptionHandler.accept(t); - } - } + /** + * Returns a future that is completed + *
    + *
  • normally, when the scheduler is closed after a successful call to {@link #close()}, or
    • + *
  • exceptionally, when it is not possible to run a retry task once it was meant to be scheduled
    • + *
. + * The future is guaranteed to be completed on the {@code retryEventLoop}. + * + * @return a future that is completed when the scheduler is closed + */ + CompletableFuture whenClosed(); - // TODO: toString() method with info about current retry task. + /** + * Applies the given {@code minimumBackoffMillis} for the next retry scheduling. + * This means that the next retry task will not be scheduled before the given + * {@code minimumBackoffMillis} has elapsed since the call to this method. + * + *

+ * This method must not be called when {@link #hasNextRetryTask()} returns {@code true} as the scheduler + * only supports sequential retrying at the moment. + *

+ * + * @param minimumBackoffMillis the minimum backoff in milliseconds to apply for the next retry scheduling + */ + void applyMinimumBackoffMillisForNextRetry(long minimumBackoffMillis); } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 35864c9d303..67ea173c31c 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -21,8 +21,10 @@ import java.util.concurrent.CompletableFuture; import java.util.function.Function; +import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.HttpClient; +import com.linecorp.armeria.common.ContextAwareEventLoop; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.Request; @@ -31,8 +33,7 @@ /** * An {@link HttpClient} decorator that handles failures of an invocation and retries HTTP requests. */ -public final class RetryingClient - extends AbstractRetryingClient +public final class RetryingClient extends AbstractRetryingClient implements HttpClient { /** * Returns a new {@link RetryingClientBuilder} with the specified {@link RetryConfig}. @@ -210,10 +211,27 @@ public static Function newDecorator(RetryRul } @Override - HttpRetryingContext getRetryingContext(ClientRequestContext ctx, RetryConfig config, - HttpRequest req) { + RetryContext newRetryContext( + Client delegate, + ClientRequestContext ctx, + HttpRequest req, + RetryConfig config) { final CompletableFuture resFuture = new CompletableFuture<>(); final HttpResponse res = HttpResponse.of(resFuture, ctx.eventLoop()); - return new HttpRetryingContext(ctx, config, res, resFuture, req, useRetryAfter); + + final ContextAwareEventLoop retryEventLoop = ctx.eventLoop(); + + final RetriedHttpRequest request = new RetriedHttpRequest( + retryEventLoop, config, ctx, req, res, resFuture, useRetryAfter + ); + + final RetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + request.deadlineTimeNanos() + ); + + final RetryCounter counter = new DefaultRetryCounter(config.maxTotalAttempts()); + + return new RetryContext(ctx.eventLoop(), request, scheduler, counter, delegate); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 050c577ef72..284a9496969 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -15,9 +15,9 @@ */ package com.linecorp.armeria.client.retry; -import java.util.concurrent.CompletableFuture; import java.util.function.Function; +import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.RpcClient; import com.linecorp.armeria.common.Request; @@ -28,7 +28,7 @@ * An {@link RpcClient} decorator that handles failures of an invocation and retries RPC requests. */ public final class RetryingRpcClient - extends AbstractRetryingClient + extends AbstractRetryingClient implements RpcClient { /** @@ -132,10 +132,11 @@ public static RetryingRpcClientBuilder builder(RetryConfigMapping m } @Override - RpcRetryingContext getRetryingContext( - ClientRequestContext ctx, RetryConfig retryConfig, RpcRequest req) { - final CompletableFuture resFuture = new CompletableFuture<>(); - final RpcResponse res = RpcResponse.from(resFuture); - return new RpcRetryingContext(ctx, retryConfig, resFuture, res, req); + RetryContext newRetryContext( + Client delegate, + ClientRequestContext ctx, + RpcRequest req, + RetryConfig config) { + throw new UnsupportedOperationException(); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java deleted file mode 100644 index b3c0876ed99..00000000000 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2025 LY Corporation - * - * LY Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.client.retry; - -import static com.google.common.base.Preconditions.checkState; - -import java.util.concurrent.CompletableFuture; - -import com.google.common.base.MoreObjects; - -import com.linecorp.armeria.client.ClientRequestContext; -import com.linecorp.armeria.common.RpcResponse; -import com.linecorp.armeria.common.annotation.Nullable; - -final class RpcRetryAttempt { - enum State { - EXECUTING, - DECIDING, - DECIDED, - COMMITTED, - ABORTED - } - - State state; - - private final RpcRetryingContext rctx; - private final ClientRequestContext ctx; - private final RpcResponse res; - @Nullable - private Throwable resCause; - private final CompletableFuture<@Nullable RetryDecision> whenDecidedFuture; - - RpcRetryAttempt(RpcRetryingContext rctx, ClientRequestContext ctx, - RpcResponse res) { - this.rctx = rctx; - this.ctx = ctx; - this.res = res; - resCause = null; - whenDecidedFuture = new CompletableFuture<>(); - - state = State.EXECUTING; - - res.handle((unused, cause) -> { - resCause = cause; - decide(); - return null; - }); - } - - private void decide() { - assert state == State.EXECUTING; - state = State.DECIDING; - - final RetryRuleWithContent retryRule = - rctx.config().needsContentInRule() ? rctx.config().retryRuleWithContent() - : rctx.config().fromRetryRule(); - - assert retryRule != null; - try { - retryRule.shouldRetry(ctx, res, resCause) - .handle((decision, cause) -> { - state = State.DECIDED; - - if (cause == null) { - whenDecidedFuture.complete(decision); - } else { - whenDecidedFuture.completeExceptionally(cause); - } - return null; - }); - } catch (Throwable t) { - state = State.DECIDED; - whenDecidedFuture.completeExceptionally(t); - } - } - - RpcResponse commit() { - if (state == State.COMMITTED) { - return res; - } - - checkState(state == State.DECIDED); - state = State.COMMITTED; - return res; - } - - void abort() { - if (state == State.ABORTED) { - return; - } - - checkState(state == State.EXECUTING || state == State.DECIDING || state == State.DECIDED); - state = State.ABORTED; - } - - CompletableFuture<@Nullable RetryDecision> whenDecided() { - return whenDecidedFuture; - } - - ClientRequestContext ctx() { - return ctx; - } - - State state() { - return state; - } - - @Override - public String toString() { - // We omit `rctx` to keep the representation compact. - return MoreObjects - .toStringHelper(this) - .add("state", state) - .add("ctx", ctx) - .add("res", res) - .add("resCause", resCause) - .toString(); - } -} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java deleted file mode 100644 index 6b9022947d8..00000000000 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryingContext.java +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Copyright 2025 LY Corporation - * - * LY Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.client.retry; - -import static com.google.common.base.Preconditions.checkState; -import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; -import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; -import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.MoreObjects; - -import com.linecorp.armeria.client.Client; -import com.linecorp.armeria.client.ClientRequestContext; -import com.linecorp.armeria.client.ResponseTimeoutException; -import com.linecorp.armeria.client.endpoint.EndpointGroup; -import com.linecorp.armeria.common.HttpRequest; -import com.linecorp.armeria.common.RpcRequest; -import com.linecorp.armeria.common.RpcResponse; -import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.stream.AbortedStreamException; -import com.linecorp.armeria.common.util.TimeoutMode; -import com.linecorp.armeria.common.util.UnmodifiableFuture; -import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; -import com.linecorp.armeria.internal.client.ClientRequestContextExtension; -import com.linecorp.armeria.internal.client.ClientUtil; - -final class RpcRetryingContext implements RetryingContext { - private static final Logger logger = LoggerFactory.getLogger(RpcRetryingContext.class); - - private enum State { - INITIALIZED, - EXECUTING, - COMPLETED - } - - private final Object lock = new Object(); - - private State state; - private final ClientRequestContext ctx; - private final RetryConfig config; - private final CompletableFuture resFuture; - private final RpcResponse res; - private final RpcRequest req; - private final RetryCounter counter; - private final RetryScheduler scheduler; - - private final long deadlineTimeNanos; - private final boolean hasDeadline; - - @Nullable - RpcRetryAttempt currentAttempt; - - RpcRetryingContext(ClientRequestContext ctx, - RetryConfig config, - CompletableFuture resFuture, - RpcResponse res, - RpcRequest req) { - state = State.INITIALIZED; - this.ctx = ctx; - this.config = config; - this.resFuture = resFuture; - this.res = res; - this.req = req; - counter = new RetryCounter(config.maxTotalAttempts()); - scheduler = new RetryScheduler(ctx); - - final long responseTimeoutMillis = ctx.responseTimeoutMillis(); - if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { - deadlineTimeNanos = 0; - hasDeadline = false; - } else { - deadlineTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); - hasDeadline = true; - } - - currentAttempt = null; - } - - @Override - public CompletableFuture init() { - synchronized (lock) { - checkState(state == State.INITIALIZED); - - res.whenComplete().handle((result, cause) -> { - final Throwable abortCause; - if (cause != null) { - abortCause = cause; - } else { - abortCause = AbortedStreamException.get(); - } - abort(abortCause); - return null; - }); - - return UnmodifiableFuture.completedFuture(true); - } - } - - @Override - public CompletableFuture<@Nullable RetryDecision> executeAttempt(@Nullable Backoff backoff, - Client delegate) { - synchronized (lock) { - checkState(state == State.INITIALIZED); - assert currentAttempt == null; - - if (!setResponseTimeout()) { - return UnmodifiableFuture.exceptionallyCompletedFuture(ResponseTimeoutException.get()); - } - - state = State.EXECUTING; - - counter.recordAttemptWith(backoff); - - final int attemptNumber = counter.attemptNumber(); - final boolean isInitialAttempt = attemptNumber <= 1; - - final ClientRequestContext attemptCtx = ClientUtil.newDerivedContext(ctx, null, req, - isInitialAttempt); - - if (!isInitialAttempt) { - attemptCtx.mutateAdditionalRequestHeaders( - mutator -> mutator.add(ARMERIA_RETRY_COUNT, Integer.toString(attemptNumber - 1))); - } - - final RpcResponse attemptRes; - final ClientRequestContextExtension attemptCtxExtension = - attemptCtx.as(ClientRequestContextExtension.class); - final EndpointGroup endpointGroup = attemptCtx.endpointGroup(); - if (!isInitialAttempt && attemptCtxExtension != null && - endpointGroup != null && attemptCtx.endpoint() == null) { - // Clear the pending throwable to retry endpoint selection - ClientPendingThrowableUtil.removePendingThrowable(attemptCtx); - // Initialize the context with a new endpoint/event loop if not selected yet - attemptRes = initContextAndExecuteWithFallback(delegate, attemptCtxExtension, RpcResponse::from, - (unused, cause) -> RpcResponse.ofFailure(cause), - req, true); - } else { - attemptRes = executeWithFallback(delegate, attemptCtx, - (unused, cause) -> RpcResponse.ofFailure(cause), - req, true); - } - - final RpcRetryAttempt attempt = new RpcRetryAttempt(this, attemptCtx, attemptRes); - currentAttempt = attempt; - return attempt.whenDecided(); - } - } - - @Override - public long nextRetryTimeNanos(Backoff backoff) { - synchronized (lock) { - if (state != State.EXECUTING) { - return Long.MAX_VALUE; - } - - if (counter.hasReachedMaxAttempts()) { - logger.debug("Exceeded the default number of max attempt: {}", config.maxTotalAttempts()); - return Long.MAX_VALUE; - } - - final long nextRetryDelayMillis = backoff.nextDelayMillis( - counter.attemptNumberForBackoff(backoff) + 1); - - if (nextRetryDelayMillis < 0) { - logger.debug("Exceeded the number of max attempts in the backoff: {}", backoff); - return Long.MAX_VALUE; - } - - final long nextDelayTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos( - nextRetryDelayMillis); - if (hasDeadline && nextDelayTimeNanos > deadlineTimeNanos) { - // The next retry will be after the response deadline. So return just Long.MAX_VALUE. - return Long.MAX_VALUE; - } - return nextDelayTimeNanos; - } - } - - @Override - public void scheduleNextRetry(long nextRetryTimeNanos, Runnable retryTask, - Consumer exceptionHandler) { - synchronized (lock) { - checkState(state == State.INITIALIZED); - assert currentAttempt == null; - scheduler.scheduleNextRetry(nextRetryTimeNanos, retryTask, exceptionHandler); - } - } - - @Override - public void commit() { - synchronized (lock) { - if (state == State.COMPLETED) { - // Already completed, so just return. - return; - } - checkState(state == State.EXECUTING); - assert currentAttempt != null; - checkState(currentAttempt.state() == RpcRetryAttempt.State.DECIDED); - - state = State.COMPLETED; - - final RpcResponse attemptRes = currentAttempt.commit(); - ctx.logBuilder().endResponseWithChild(currentAttempt.ctx().log()); - final HttpRequest attemptReq = currentAttempt.ctx().request(); - if (attemptReq != null) { - ctx.updateRequest(attemptReq); - } - resFuture.complete(attemptRes); - } - } - - @Override - public void abortAttempt() { - synchronized (lock) { - checkState(state == State.EXECUTING); - assert currentAttempt != null; - // Can be called in any state. - currentAttempt.abort(); - currentAttempt = null; - state = State.INITIALIZED; - } - } - - @Override - public void abort(Throwable cause) { - synchronized (lock) { - if (state == State.COMPLETED) { - return; - } - - if (state == State.EXECUTING) { - assert currentAttempt != null; - currentAttempt.abort(); - } - - state = State.COMPLETED; - - if (!ctx.log().isRequestComplete()) { - ctx.logBuilder().endRequest(cause); - } - ctx.logBuilder().endResponse(cause); - resFuture.completeExceptionally(cause); - } - } - - private boolean setResponseTimeout() { - final long responseTimeoutMillis = responseTimeoutMillis(); - if (responseTimeoutMillis < 0) { - return false; - } else if (responseTimeoutMillis == 0) { - ctx.clearResponseTimeout(); - return true; - } else { - ctx.setResponseTimeoutMillis(TimeoutMode.SET_FROM_NOW, responseTimeoutMillis); - return true; - } - } - - private long responseTimeoutMillis() { - if (!hasDeadline) { - return config.responseTimeoutMillisForEachAttempt(); - } - - final long remaining = TimeUnit.NANOSECONDS.toMillis(deadlineTimeNanos - System.nanoTime()); - if (remaining <= 0) { - return -1; - } - if (config.responseTimeoutMillisForEachAttempt() > 0) { - return Math.min(config.responseTimeoutMillisForEachAttempt(), remaining); - } - return remaining; - } - - @Override - public RpcResponse res() { - return res; - } - - RetryConfig config() { - return config; - } - - @Override - public String toString() { - synchronized (lock) { - return MoreObjects - .toStringHelper(this) - .add("ctx", ctx) - .add("state", state) - .add("req", req) - .add("res", res) - .add("deadlineTimeNanos", deadlineTimeNanos) - .add("hasDeadline", hasDeadline) - .add("counter", counter) - .add("scheduler", scheduler) - .add("currentAttempt", currentAttempt) - .toString(); - } - } -} diff --git a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java index a4f65d598fc..a63efc7bba4 100644 --- a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java +++ b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java @@ -18,10 +18,15 @@ import static java.util.Objects.requireNonNull; import java.net.URI; +import java.time.Duration; +import java.util.Date; import java.util.concurrent.CompletableFuture; import java.util.function.BiFunction; import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.Endpoint; @@ -30,6 +35,7 @@ import com.linecorp.armeria.client.UnprocessedRequestException; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.client.endpoint.EndpointGroup; +import com.linecorp.armeria.common.HttpHeaderNames; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.RequestId; @@ -45,7 +51,10 @@ import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.armeria.common.util.SafeCloseable; +import io.netty.handler.codec.DateFormatter; + public final class ClientUtil { + private static final Logger logger = LoggerFactory.getLogger(ClientUtil.class); /** * An undefined {@link URI} to create {@link WebClient} without specifying {@link URI}. @@ -281,5 +290,45 @@ public static ClientRequestContext newDerivedContext(ClientRequestContext ctx, return derived; } + /** + * Tries to parse the {@code "Retry-After"} response header from the given {@link RequestLogAccess}. + * If the header is present and valid, returns the number of milliseconds after which a retry + * should be attempted. Otherwise, returns {@code -1}. + * + * @param log the {@link RequestLogAccess} which may contain the {@code "Retry-After"} header + * @return the number of milliseconds after which a retry should be attempted, or {@code -1} if + * the header is not present or invalid + */ + public static long retryAfterMillis(RequestLogAccess log) { + final String retryAfterValue; + final RequestLog requestLog = log.getIfAvailable(RequestLogProperty.RESPONSE_HEADERS); + retryAfterValue = requestLog != null ? + requestLog.responseHeaders().get(HttpHeaderNames.RETRY_AFTER) : null; + + if (retryAfterValue != null) { + try { + return Duration.ofSeconds(Integer.parseInt(retryAfterValue)).toMillis(); + } catch (Exception ignored) { + // Not a second value. + } + + try { + @SuppressWarnings("UseOfObsoleteDateTimeApi") + final Date retryAfterDate = DateFormatter.parseHttpDate(retryAfterValue); + if (retryAfterDate != null) { + return retryAfterDate.getTime() - System.currentTimeMillis(); + } + } catch (Exception ignored) { + // `parseHttpDate()` can raise an exception rather than returning `null` + // when the given value has more than 64 characters. + } + + logger.debug("The retryAfter: {}, from the server is neither an HTTP date nor a second.", + retryAfterValue); + } + + return -1; + } + private ClientUtil() {} } From 0f83d696486f4c92b0ecc1ef0a4b9c033d093719 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 7 Sep 2025 17:43:51 +0200 Subject: [PATCH 26/42] fix: refactor deadline setting to ClientUtil - also patch some inaccuracies in RetryingClient and RetriedHttpRequest --- .../client/retry/HttpRetryAttempt.java | 11 ++-- .../client/retry/RetriedHttpRequest.java | 37 +------------- .../armeria/client/retry/RetriedRequest.java | 28 +++++------ .../armeria/client/retry/RetryingClient.java | 2 +- .../armeria/internal/client/ClientUtil.java | 50 +++++++++++++++++++ 5 files changed, 74 insertions(+), 54 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java index 1764496ace8..e31466f09f9 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/HttpRetryAttempt.java @@ -150,11 +150,12 @@ enum State { */ DECIDED, /** - * The attempt was committed via {@link #commit()}. This is a terminal state. + * The attempt was committed via {@link #commit()}. This is a terminal state and the response + * is final. */ COMMITTED, /** - * The attempt was aborted via {@link #abort(Throwable)}. The response is *going to be* + * The attempt was aborted via {@link #abort(Throwable)}. The response is * aborted via the same cause. In this state and only in this state {@link #cause} is not-{@code null}. * This is a terminal state. */ @@ -558,7 +559,7 @@ private void completeLogIfBytesNotTransferred( * @param resForRule the response to be delivered to the {@link RetryRule} for decision. * @param causeForRule the cause to be delivered to the {@link RetryRule} for decision. * @return a future that will be completed with the {@link RetryDecision} or an exception if failed during - * the decision. + * the decision. The future will be completed on the retry event loop. */ private CompletableFuture decide(@Nullable HttpResponse resForRule, @Nullable Throwable causeForRule) { @@ -601,6 +602,10 @@ private CompletableFuture decide(@Nullable HttpResponse resForRul ); } + /** + * Ensures the given {@link CompletionStage} completes on the retry event loop. + * If already on the retry event loop, completes directly; otherwise schedules completion. + */ private CompletableFuture withCompletionOnRetryEventLoop(CompletionStage future) { final CompletableFuture futureOnTheRetryEventLoop = new CompletableFuture<>(); future.whenComplete((result, cause) -> { diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java index 300a60f1208..0623a07d7ab 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java @@ -35,7 +35,6 @@ import com.linecorp.armeria.common.RequestHeadersBuilder; import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.stream.AbortedStreamException; -import com.linecorp.armeria.common.util.TimeoutMode; import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; import com.linecorp.armeria.internal.client.ClientUtil; @@ -230,7 +229,8 @@ private CompletableFuture executeAttemptAfterInit( assert attemptNumber == attempts.size() + 1 : attemptNumber + ", " + attempts.size(); assert attemptNumber <= config.maxTotalAttempts() : attemptNumber + ", " + config.maxTotalAttempts(); - if (!checkAndSetResponseTimeout()) { + if (!ClientUtil.checkAndSetResponseTimeout(ctx, deadlineTimeNanos, + config.responseTimeoutMillisForEachAttempt())) { final ResponseTimeoutException timeoutException = ResponseTimeoutException.get(); abort(timeoutException); return UnmodifiableFuture.exceptionallyCompletedFuture(timeoutException); @@ -339,13 +339,9 @@ public void abort(int attemptNumber, Throwable cause) { } checkState(state == State.PENDING); - checkArgument(attemptNumber >= 1); checkArgument(attemptNumber <= attempts.size()); attemptToAbort = attempts.get(attemptNumber - 1); - - checkState(attemptToAbort != null); - attemptToAbort.abort(cause); } @@ -370,33 +366,4 @@ public HttpResponse res() { public long deadlineTimeNanos() { return deadlineTimeNanos; } - - private boolean checkAndSetResponseTimeout() { - long remainingTimeUntilDeadlineMillis = Long.MAX_VALUE; - boolean hasTimeout = false; - - if (deadlineTimeNanos < Long.MAX_VALUE) { - hasTimeout = true; - remainingTimeUntilDeadlineMillis = TimeUnit.NANOSECONDS.toMillis( - deadlineTimeNanos - System.nanoTime()); - } - - if (config.responseTimeoutMillisForEachAttempt() != 0) { - hasTimeout = true; - remainingTimeUntilDeadlineMillis = - Math.min(remainingTimeUntilDeadlineMillis, config.responseTimeoutMillisForEachAttempt()); - } - - if (hasTimeout) { - if (remainingTimeUntilDeadlineMillis > 0) { - ctx.setResponseTimeoutMillis(TimeoutMode.SET_FROM_NOW, remainingTimeUntilDeadlineMillis); - return true; - } else { - return false; - } - } else { - ctx.clearResponseTimeout(); - return true; - } - } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java index 04496f4d9af..beab4300721 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java @@ -26,8 +26,8 @@ * A retried request that manages multiple retry attempts. * *

- * NOTE: All methods of {@link RetryScheduler} must be invoked from the - * {@code retryEventLoop}. Implementations of {@link RetriedRequest} therefore do not need to be thread-safe. + * NOTE: All methods of {@link RetriedRequest} must be invoked from a single-threaded event loop. + * Implementations of {@link RetriedRequest} therefore do not need to be thread-safe. *

* * @param the {@link Request} type @@ -92,9 +92,8 @@ public long minimumBackoffMillis() { *

* *

- * It completed exceptionally in case something unexpected happened during the attempt execution. - * In particular, it completes exceptionally with a {@link AbortedAttemptException} if the attempt - * was aborted by the {@link RetriedRequest}. This could happen if the {@link RetriedRequest} completes + * It completes exceptionally with a {@link AbortedAttemptException} if the attempt + * was aborted by the {@link RetriedRequest}. This can happen when the {@link RetriedRequest} completes * concurrently while the attempt is being executed. *

* @@ -107,7 +106,7 @@ public long minimumBackoffMillis() { /** * Commits the attempt with the specified {@code attemptNumber}. Committing an attempt includes: *
    - *
  • completing the original request with the attempt's response,
    • + *
  • completing the original request future with the attempt's response,
    • *
  • aborting all other pending attempts, and
    • *
  • marking the {@link RetriedRequest} as completed.
    • *
@@ -118,13 +117,13 @@ public long minimumBackoffMillis() { void commit(int attemptNumber); /** - * Aborts the attempt identified by the given {@code attemptNumber}. + * Aborts a specific attempt that was previously started via {@link #executeAttempt(Client)}. * - *

The attempt must have been previously started via - * {@link #executeAttempt(Client)}. Aborting an attempt ensures that its - * response and {@link RequestLog} - * complete, but that it will not be committed as the final result of the - * {@link RetriedRequest}.

+ *

+ * The attempt must have been previously started via + * {@link #executeAttempt(Client)}. Aborting an attempt ensures that the attempt's + * response and {@link RequestLog} are completed but not set as the original request's response or log. + *

* *

If the {@link RetriedRequest} is already completed, this method has no effect. * If the specified attempt has not been started, an {@link IllegalStateException} is thrown.

@@ -142,9 +141,8 @@ public long minimumBackoffMillis() { void abort(Throwable cause); /** - * Returns the original {@link Response} that is completed when the {@link RetriedRequest} is completed. - * @return the {@link Response} with either a committed attempt's response or with an exception from a call - * to {@link #abort(Throwable)} or from an unexpected error during the processing of an attempt. + * Returns the original {@link Response}. In case of a call to {@link #abort(Throwable)}, the returned + * {@link Response} is completed exceptionally with the same {@code cause}. */ O res(); } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index 67ea173c31c..a29267d7749 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -232,6 +232,6 @@ RetryContext newRetryContext( final RetryCounter counter = new DefaultRetryCounter(config.maxTotalAttempts()); - return new RetryContext(ctx.eventLoop(), request, scheduler, counter, delegate); + return new RetryContext(retryEventLoop, request, scheduler, counter, delegate); } } diff --git a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java index a63efc7bba4..7fac3b8545e 100644 --- a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java +++ b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java @@ -21,6 +21,7 @@ import java.time.Duration; import java.util.Date; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import java.util.function.Function; @@ -50,6 +51,7 @@ import com.linecorp.armeria.common.stream.StreamMessage; import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.armeria.common.util.SafeCloseable; +import com.linecorp.armeria.common.util.TimeoutMode; import io.netty.handler.codec.DateFormatter; @@ -330,5 +332,53 @@ public static long retryAfterMillis(RequestLogAccess log) { return -1; } + /** + * Sets the response timeout on a {@link ClientRequestContext} based on the minimum of a deadline + * and response timeout, or clears the timeout if neither is specified. + * + *

This method respects both absolute deadlines (in nanoseconds) and relative response timeouts + * (in milliseconds), using whichever is more restrictive. If the calculated timeout has already + * expired, the method returns {@code false} to indicate the request should be considered timed out. + * + * @param ctx the {@link ClientRequestContext} to configure + * @param deadlineTimeNanos absolute deadline in nanoseconds from {@link System#nanoTime()}, + * or {@link Long#MAX_VALUE} if no deadline is set + * @param responseTimeoutMillis relative timeout in milliseconds, or {@code 0} if no timeout is set + * @return {@code true} if the timeout was successfully set/cleared and the request is still valid, + * {@code false} if the request has already exceeded the calculated timeout + * + * @see ClientRequestContext#setResponseTimeoutMillis(TimeoutMode, long) + * @see ClientRequestContext#clearResponseTimeout() + */ + public static boolean checkAndSetResponseTimeout(ClientRequestContext ctx, long deadlineTimeNanos, + long responseTimeoutMillis) { + long remainingTimeUntilDeadlineMillis = Long.MAX_VALUE; + boolean hasTimeout = false; + + if (deadlineTimeNanos < Long.MAX_VALUE) { + hasTimeout = true; + remainingTimeUntilDeadlineMillis = TimeUnit.NANOSECONDS.toMillis( + deadlineTimeNanos - System.nanoTime()); + } + + if (responseTimeoutMillis != 0) { + hasTimeout = true; + remainingTimeUntilDeadlineMillis = + Math.min(remainingTimeUntilDeadlineMillis, responseTimeoutMillis); + } + + if (hasTimeout) { + if (remainingTimeUntilDeadlineMillis > 0) { + ctx.setResponseTimeoutMillis(TimeoutMode.SET_FROM_NOW, remainingTimeUntilDeadlineMillis); + return true; + } else { + return false; + } + } else { + ctx.clearResponseTimeout(); + return true; + } + } + private ClientUtil() {} } From f7655ad3ae32e10c2cad8563527c15f207fdd87f Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 7 Sep 2025 18:27:51 +0200 Subject: [PATCH 27/42] feat: refactor RetryingRpcClient --- .../client/retry/RetriedRpcRequest.java | 283 ++++++++++++ .../client/retry/RetryingRpcClient.java | 20 +- .../armeria/client/retry/RpcRetryAttempt.java | 407 ++++++++++++++++++ .../client/retry/RetryingRpcClientTest.java | 79 +++- 4 files changed, 781 insertions(+), 8 deletions(-) create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetriedRpcRequest.java create mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRpcRequest.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRpcRequest.java new file mode 100644 index 00000000000..7dc3c080b24 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRpcRequest.java @@ -0,0 +1,283 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.client.retry; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; + +import java.util.ArrayList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeUnit; + +import com.linecorp.armeria.client.Client; +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.client.ResponseTimeoutException; +import com.linecorp.armeria.common.ContextAwareEventLoop; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.common.RpcRequest; +import com.linecorp.armeria.common.RpcResponse; +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.stream.AbortedStreamException; +import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.armeria.internal.client.ClientUtil; + +/** + * Manages retry attempts for an RPC request, coordinating execution and decision-making + * across multiple attempts until one succeeds or all retry options are exhausted. + * + *

All state mutations must occur on the {@code retryEventLoop} thread. + */ +class RetriedRpcRequest implements RetriedRequest { + private enum State { + /** + * Initial state. {@link RetriedRpcRequest} is instantiated but not yet initialized. + */ + IDLE, + /** + * {@link RetriedRpcRequest} is initialized and is ready to execute/ is executing attempts. + */ + PENDING, + /** + * Terminal state. Either {@link #commit(int)} or {@link #abort(Throwable)} was called. + * {@link RetriedRpcRequest} will not execute any more attempts in that it completes every call + * to {@link #executeAttempt(Client)} with an {@link AbortedAttemptException}. + */ + COMPLETED + } + + private final ContextAwareEventLoop retryEventLoop; + private final RetryConfig config; + private final ClientRequestContext ctx; + private final RpcResponse res; + private final CompletableFuture resFuture; + private final RpcRequest req; + + private final long deadlineTimeNanos; + + private State state; + + int previousAttemptNumber; + + final ArrayList attempts; + + RetriedRpcRequest( + ContextAwareEventLoop retryEventLoop, + RetryConfig config, + ClientRequestContext ctx, + RpcRequest req, + RpcResponse res, CompletableFuture resFuture + ) { + this.retryEventLoop = retryEventLoop; + this.ctx = ctx; + this.res = res; + this.resFuture = resFuture; + this.req = req; + this.config = config; + + final long responseTimeoutMillis = ctx.responseTimeoutMillis(); + if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { + deadlineTimeNanos = Long.MAX_VALUE; + } else { + deadlineTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); + } + + state = State.IDLE; + previousAttemptNumber = 0; + // We can assume that most of the time we will succeed with the first attempt. + attempts = new ArrayList<>(1); + } + + @Override + public CompletionStage executeAttempt( + Client delegate) { + checkState(ctx.eventLoop().inEventLoop()); + init(); + return executeAttemptAfterInit(delegate); + } + + /** + * Initializes the {@link RetriedRpcRequest}. This method is idempotent. + * Initialization mean subscribing to the original {@link RpcResponse} so that we can + * abort all attempts when the original {@link RpcResponse} is completed. + */ + private void init() { + assert ctx.eventLoop().inEventLoop(); + + if (state == State.PENDING || state == State.COMPLETED) { + return; + } + + assert state == State.IDLE : state; + state = State.PENDING; + + res.whenComplete().handle((result, cause) -> { + final Throwable abortCause; + if (cause != null) { + abortCause = cause; + } else { + abortCause = AbortedStreamException.get(); + } + if (retryEventLoop.inEventLoop()) { + abort(abortCause); + } else { + retryEventLoop.execute(() -> abort(abortCause)); + } + return null; + }); + } + + private CompletionStage executeAttemptAfterInit( + Client delegate) { + assert retryEventLoop.inEventLoop(); + + if (state == State.COMPLETED) { + return UnmodifiableFuture.exceptionallyCompletedFuture(AbortedAttemptException.get()); + } + + // We need to be initialized/ the reqDuplicator must be present. + checkState(state == State.PENDING); + checkState(previousAttemptNumber < config.maxTotalAttempts()); + + final int attemptNumber = ++previousAttemptNumber; + + assert attemptNumber == attempts.size() + 1 : attemptNumber + ", " + attempts.size(); + assert attemptNumber <= config.maxTotalAttempts() : attemptNumber + ", " + config.maxTotalAttempts(); + + if (!ClientUtil.checkAndSetResponseTimeout(ctx, deadlineTimeNanos, + config.responseTimeoutMillisForEachAttempt())) { + final ResponseTimeoutException timeoutException = ResponseTimeoutException.get(); + abort(timeoutException); + return UnmodifiableFuture.exceptionallyCompletedFuture(timeoutException); + } + + final RpcRetryAttempt attempt = newAttempt(attemptNumber); + attempts.add(attempt); + return attempt + .executeAndDecide(delegate) + .thenApply(decision -> { + assert retryEventLoop.inEventLoop(); + return new AttemptExecutionResult( + attemptNumber, + decision, + ClientUtil.retryAfterMillis(attempt.ctx().log()) + ); + }) + .exceptionally(cause -> { + assert retryEventLoop.inEventLoop(); + abort(cause); + return null; + }); + } + + private RpcRetryAttempt newAttempt(int attemptNumber) { + final boolean isInitialAttempt = attemptNumber <= 1; + + final ClientRequestContext attemptCtx = ClientUtil.newDerivedContext(ctx, null, req, + isInitialAttempt); + + return new RpcRetryAttempt(config, retryEventLoop, attemptNumber, attemptCtx, req); + } + + @Override + public void commit(int attemptNumber) { + checkState(retryEventLoop.inEventLoop()); + + if (state == State.COMPLETED) { + return; + } + checkState(state == State.PENDING); + + checkArgument(attemptNumber >= 1); + checkArgument(attemptNumber <= attempts.size()); + + final RpcRetryAttempt attemptToCommit = attempts.get(attemptNumber - 1); + + checkState(attemptToCommit != null); + checkState(attemptToCommit.state() == RpcRetryAttempt.State.DECIDED); + + state = State.COMPLETED; + + abortAllExcept(attemptToCommit); + + final RpcResponse attemptRes = attemptToCommit.commit(); + + ctx.logBuilder().endResponseWithChild(attemptToCommit.ctx().log()); + final HttpRequest attemptReq = attemptToCommit.ctx().request(); + if (attemptReq != null) { + ctx.updateRequest(attemptReq); + } + resFuture.complete(attemptRes); + } + + @Override + public void abort(Throwable cause) { + checkState(retryEventLoop.inEventLoop()); + + if (state == State.COMPLETED) { + return; + } + + state = State.COMPLETED; + + abortAllExcept(null); + + if (!ctx.log().isRequestComplete()) { + ctx.logBuilder().endRequest(cause); + } + ctx.logBuilder().endResponse(cause); + resFuture.completeExceptionally(cause); + } + + @Override + public void abort(int attemptNumber, Throwable cause) { + checkState(retryEventLoop.inEventLoop()); + checkArgument(attemptNumber >= 1); + + final RpcRetryAttempt attemptToAbort; + if (state == State.COMPLETED) { + return; + } + + checkState(state == State.PENDING); + checkArgument(attemptNumber <= attempts.size()); + + attemptToAbort = attempts.get(attemptNumber - 1); + attemptToAbort.abort(cause); + } + + private void abortAllExcept(@Nullable RpcRetryAttempt winningAttempt) { + assert retryEventLoop.inEventLoop(); + assert state == State.COMPLETED : state; + + for (final RpcRetryAttempt attempt : attempts) { + if (attempt == winningAttempt) { + continue; + } + + attempt.abort(AbortedAttemptException.get()); + } + } + + @Override + public RpcResponse res() { + return res; + } + + public long deadlineTimeNanos() { + return deadlineTimeNanos; + } +} diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 284a9496969..1ae0f4c2ac5 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -15,11 +15,13 @@ */ package com.linecorp.armeria.client.retry; +import java.util.concurrent.CompletableFuture; import java.util.function.Function; import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.RpcClient; +import com.linecorp.armeria.common.ContextAwareEventLoop; import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; @@ -137,6 +139,22 @@ RetryContext newRetryContext( ClientRequestContext ctx, RpcRequest req, RetryConfig config) { - throw new UnsupportedOperationException(); + final CompletableFuture resFuture = new CompletableFuture<>(); + final RpcResponse res = RpcResponse.from(resFuture); + + final ContextAwareEventLoop retryEventLoop = ctx.eventLoop(); + + final RetriedRpcRequest request = new RetriedRpcRequest( + retryEventLoop, config, ctx, req, res, resFuture + ); + + final RetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + request.deadlineTimeNanos() + ); + + final RetryCounter counter = new DefaultRetryCounter(config.maxTotalAttempts()); + + return new RetryContext(retryEventLoop, request, scheduler, counter, delegate); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java new file mode 100644 index 00000000000..dfc06ebdea0 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java @@ -0,0 +1,407 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.retry; + +import static com.google.common.base.Preconditions.checkState; +import static com.linecorp.armeria.client.retry.AbstractRetryingClient.ARMERIA_RETRY_COUNT; +import static com.linecorp.armeria.internal.client.ClientUtil.executeWithFallback; +import static com.linecorp.armeria.internal.client.ClientUtil.initContextAndExecuteWithFallback; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Function; + +import com.google.common.base.MoreObjects; + +import com.linecorp.armeria.client.Client; +import com.linecorp.armeria.client.ClientRequestContext; +import com.linecorp.armeria.common.ContextAwareEventLoop; +import com.linecorp.armeria.common.RpcRequest; +import com.linecorp.armeria.common.RpcResponse; +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.util.Exceptions; +import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.armeria.internal.client.ClientPendingThrowableUtil; +import com.linecorp.armeria.internal.client.ClientRequestContextExtension; + +/** + * A single attempt for a {@link RetriedRpcRequest}. + * + *

+ * NOTE: All methods of {@link RpcRetryAttempt} must be invoked from the + * {@code retryEventLoop}. + *

+ */ +final class RpcRetryAttempt { + /** + * The state of this attempt. + * + *

This state machine controls the retry attempt lifecycle. All state transitions + * must occur on the {@code retryEventLoop} thread to ensure consistency. + * + *

The following state machine diagram illustrates the possible transitions: + * 

+     *          Start
+     *            |
+     *           IDLE
+     *            |
+     *            v
+     *        EXECUTING----------+
+     *            |              |
+     *            v              |
+     *         DECIDED-----------+
+     *          /   \            |
+     *         /     \           |
+     *        /       \          |
+     *       v         v         |
+     *   COMMITTED  ABORTED<-----+
+     *       \         /
+     *        +---+---+
+     *            |
+     *            v
+     *           End
+     *
+ */ + enum State { + /** + * The attempt has not been executed yet. + * Call {@link #executeAndDecide(Client)} to transition to {@link #EXECUTING}. + */ + IDLE, + + /** + * The attempt is executing and waiting for a response or exception. + * The {@link #res} field becomes available in this state. + * Can transition to {@link #DECIDED} on completion or {@link #ABORTED} if aborted. + */ + EXECUTING, + + /** + * The {@link RetryRule} has made a decision about whether to retry. + * The attempt is ready to be either committed via {@link #commit()} + * or aborted via {@link #abort(Throwable)}. + */ + DECIDED, + + /** + * The attempt was successfully committed via {@link #commit()}. + * This is a terminal state and the response is final. + */ + COMMITTED, + + /** + * The attempt was aborted via {@link #abort(Throwable)}. + * The {@link #cause} field is non-null only in this state. + * This is a terminal state. + */ + ABORTED + } + + private final RetryConfig config; + private final ContextAwareEventLoop retryEventLoop; + private final int attemptNumber; + private final ClientRequestContext ctx; + private final RpcRequest req; + + private State state; + + /** + * The response of the attempt. It is available in {@link State#EXECUTING}, + * {@link State#DECIDED}, and {@link State#COMMITTED} states. + */ + @Nullable + private RpcResponse res; + + /** + * The cause of the attempt failure. It is not-{@code null} iff we are in {@link State#ABORTED} state. + */ + @Nullable + Throwable cause; + + RpcRetryAttempt( + RetryConfig config, + ContextAwareEventLoop retryEventLoop, + int attemptNumber, + ClientRequestContext ctx, + RpcRequest req + ) { + this.config = config; + this.retryEventLoop = retryEventLoop; + this.attemptNumber = attemptNumber; + this.ctx = ctx; + this.req = req; + + state = State.IDLE; + } + + /** + * Executes the attempt, prepares and gives the response or the cause to the {@link RetryRule}. + * This method must be called at most once and except for calls to {@link #abort(Throwable)} + * it must be called before any other method, in particular before {@link #commit()}. + * + * @param delegate the next {@link Client} in the decoration chain + * @return a future that will be completed with the {@link RetryDecision} or an exception if failed during + * execution, preparation, or decision. + * In particular, it fails with a {@link AbortedAttemptException} + * if the attempt was aborted by {@link #abort(Throwable)}. + * + * @see #commit() + */ + CompletionStage executeAndDecide(Client delegate) { + checkState(retryEventLoop.inEventLoop()); + + if (state == State.ABORTED) { + assert cause != null; + return UnmodifiableFuture.exceptionallyCompletedFuture(cause); + } + + checkState(state == State.IDLE); + state = State.EXECUTING; + + res = execute(delegate); + + return res + .handle((unused, causeToDecide) -> { + assert retryEventLoop.inEventLoop(); + // Let us avoid calling decide (which may be expensive if we already know that we were + // aborted. + if (state == State.ABORTED) { + assert cause != null; + return UnmodifiableFuture.exceptionallyCompletedFuture(cause); + } + + assert state == State.EXECUTING : state; + + return decide(res, causeToDecide) + .thenCompose( + decision -> { + if (state == State.ABORTED) { + assert cause != null; + return UnmodifiableFuture.exceptionallyCompletedFuture( + cause); + } else { + assert state == State.EXECUTING : state; + assert cause == null : cause; // sanity check + assert res != null; + state = State.DECIDED; + return UnmodifiableFuture.completedFuture(decision); + } + } + ); + }) + .thenCompose(Function.identity()) + .handle( + (decision, preparationOrDecisionCause) -> { + if (preparationOrDecisionCause == null) { + return (CompletableFuture) UnmodifiableFuture.completedFuture( + decision); + } + + if (state == State.ABORTED) { + assert cause != null; + if (cause != preparationOrDecisionCause) { + cause.addSuppressed(preparationOrDecisionCause); + } + return UnmodifiableFuture.exceptionallyCompletedFuture( + preparationOrDecisionCause); + } + assert state == State.EXECUTING : state; + + abort(preparationOrDecisionCause); + return UnmodifiableFuture.exceptionallyCompletedFuture( + preparationOrDecisionCause); + } + ) + .thenCompose(Function.identity()); + } + + private RpcResponse execute(Client delegate) { + final boolean isInitialAttempt = attemptNumber <= 1; + + if (!isInitialAttempt) { + ctx.mutateAdditionalRequestHeaders( + mutator -> mutator.add(ARMERIA_RETRY_COUNT, Integer.toString(attemptNumber - 1))); + } + + final ClientRequestContextExtension ctxExt = + ctx.as(ClientRequestContextExtension.class); + if (!isInitialAttempt && ctxExt != null && ctx.endpoint() == null) { + // clear the pending throwable to retry endpoint selection + ClientPendingThrowableUtil.removePendingThrowable(ctx); + // if the endpoint hasn't been selected, + // try to initialize the attempCtx with a new endpoint/event loop + return initContextAndExecuteWithFallback( + delegate, ctxExt, RpcResponse::of, + (context, cause) -> + RpcResponse.ofFailure(cause), req, false); + } else { + return executeWithFallback(delegate, ctx, + (context, cause) -> + RpcResponse.ofFailure(cause), req, false); + } + } + + /** + * Marks this attempt as committed and returns the response of the attempt. + * The attempt must be decided, i.e. {@link #executeAndDecide(Client)} must have been called and completed + * successfully before calling this method. After this call, the attempt cannot be aborted anymore. + * This method is idempotent once the first call returns successfully. + * + * @return the response of the attempt + * + * @see #executeAndDecide(Client) + */ + public RpcResponse commit() { + checkState(retryEventLoop.inEventLoop()); + + if (state == State.COMMITTED) { + assert res != null; + return res; + } + + checkState(state == State.DECIDED); + assert res != null; + assert cause == null : cause; + state = State.COMMITTED; + return res; + } + + /** + * Aborts this attempt with the specified {@code cause}. + * {@link #executeAndDecide(Client)} must be called before this method is called. + * After this call, the attempt cannot be committed anymore. + * This method is idempotent once the first call returns successfully. + * + * @param cause the cause of the abortion + */ + public void abort(Throwable cause) { + checkState(retryEventLoop.inEventLoop()); + + if (state == State.ABORTED) { + return; + } + + assert state == State.IDLE || + state == State.EXECUTING || + state == State.DECIDED : state; + assert this.cause == null : this.cause; + state = State.ABORTED; + this.cause = cause; + ctx.cancel(); + } + + /** + * Returns the {@link ClientRequestContext} of this attempt. + * + * @return the {@link ClientRequestContext} of this attempt + */ + ClientRequestContext ctx() { + return ctx; + } + + /** + * Returns the {@link State} of this attempt. + * + * @return the {@link State} of this attempt + */ + State state() { + return state; + } + + /** + * Calls {@link RetryRule#shouldRetry(ClientRequestContext, Throwable)} to receive a {@link RetryDecision}. + * + * @param resForRule the response to be delivered to the {@link RetryRule} for decision. + * @param causeForRule the cause to be delivered to the {@link RetryRule} for decision. + * @return a future that will be completed with the {@link RetryDecision} or an exception if failed during + * the decision. The future will be completed on the retry event loop. + */ + private CompletableFuture decide(@Nullable RpcResponse resForRule, + @Nullable Throwable causeForRule) { + if (causeForRule != null) { + resForRule = null; + causeForRule = Exceptions.peel(causeForRule); + } else { + assert resForRule != null; + causeForRule = null; + } + + final RetryRuleWithContent retryRule = + config.needsContentInRule() ? config.retryRuleWithContent() + : config.fromRetryRule(); + assert retryRule != null; + + try { + final CompletionStage<@Nullable RetryDecision> maybeDecision = retryRule + .shouldRetry(ctx, + resForRule, + causeForRule); + + // Remember that RetryRule.shouldRetry could be client code so we do have any guarantees + // on which thread we are completing. Let us make sure we are completing on running on the + // retry event loop again. + return withCompletionOnRetryEventLoop( + maybeDecision.thenApply( + decision -> decision == null ? RetryDecision.noRetry() : decision + ) + ); + } catch (Throwable t) { + return UnmodifiableFuture.exceptionallyCompletedFuture(t); + } + } + + /** + * Ensures the given {@link CompletionStage} completes on the retry event loop. + * If already on the retry event loop, completes directly; otherwise schedules completion. + */ + private CompletableFuture withCompletionOnRetryEventLoop(CompletionStage future) { + final CompletableFuture futureOnTheRetryEventLoop = new CompletableFuture<>(); + future.whenComplete((result, cause) -> { + if (retryEventLoop.inEventLoop()) { + if (cause != null) { + futureOnTheRetryEventLoop.completeExceptionally(cause); + } else { + futureOnTheRetryEventLoop.complete(result); + } + } else { + retryEventLoop.execute(() -> { + if (cause != null) { + futureOnTheRetryEventLoop.completeExceptionally(cause); + } else { + futureOnTheRetryEventLoop.complete(result); + } + }); + } + }); + return futureOnTheRetryEventLoop; + } + + @Override + public String toString() { + checkState(retryEventLoop.inEventLoop()); + + return MoreObjects + .toStringHelper(this) + .add("state", state) + .add("attemptNumber", attemptNumber) + .add("ctx", ctx) + .add("req", req) + .add("res", res) + .add("cause", cause) + .toString(); + } +} diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java index a4b3d5df441..c9ab99f1e8f 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java @@ -23,6 +23,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.only; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -73,6 +74,7 @@ class RetryingRpcClientTest { private final HelloService.Iface serviceHandler = mock(HelloService.Iface.class); private final DevNullService.Iface devNullServiceHandler = mock(DevNullService.Iface.class); private final AtomicInteger serviceRetryCount = new AtomicInteger(); + private final AtomicInteger serviceSleepMillis = new AtomicInteger(); @RegisterExtension final ServerExtension server = new ServerExtension() { @@ -327,37 +329,100 @@ void shouldGetExceptionWhenFactoryIsClosed() throws Exception { } @Test - void doNotRetryWhenResponseIsCancelled() throws Exception { + void doNotRetryWhenResponseIsCancelledAfterRequestComplete() throws Exception { + final AtomicInteger subscriberCancelServiceCallCounterWhenCancelled = new AtomicInteger(-1); + try (ClientFactory factory = ClientFactory.builder().build()) { final AtomicReference context = new AtomicReference<>(); final HelloService.Iface client = ThriftClients.builder(server.httpUri()) .path("/thrift") .factory(factory) - .rpcDecorator(RetryingRpcClient.builder(retryAlways).newDecorator()) + .rpcDecorator(RetryingRpcClient + .builder((ctx, response, cause) -> + UnmodifiableFuture.completedFuture( + RetryDecision.retry( + Backoff.fixed(5) + ) + ) + ) + .maxTotalAttempts(2 * (1000 / 5)) + .newDecorator()) .rpcDecorator((delegate, ctx, req) -> { - context.set(ctx); final RpcResponse res = delegate.execute(ctx, req); - res.cancel(true); + ctx.eventLoop().schedule(() -> { + // (A) + subscriberCancelServiceCallCounterWhenCancelled.set( + serviceRetryCount.get()); + res.cancel(true); + }, 1000, TimeUnit.MILLISECONDS); + context.set(ClientRequestContext.current()); return res; }) .build(HelloService.Iface.class); when(serviceHandler.hello(anyString())).thenThrow(new IllegalArgumentException()); + // We retry very often until (A) gets scheduled and we cancel the request. assertThatThrownBy(() -> client.hello("hello")).isInstanceOf(CancellationException.class); await().untilAsserted(() -> { - verify(serviceHandler, only()).hello("hello"); + assertThat(serviceRetryCount.get()).isGreaterThanOrEqualTo(1); + // After we cancelled we do not expect further service calls. + assertThat(subscriberCancelServiceCallCounterWhenCancelled.get()).isIn( + serviceRetryCount.get(), serviceRetryCount.get() + 1); + verify(serviceHandler, times(serviceRetryCount.get())).hello("hello"); }); + + final RequestLog log = context.get().log().whenComplete().join(); + + // The request was made successfully. + assertThat(log.requestCause()).isNull(); + assertThat(log.responseCause()).isExactlyInstanceOf(CancellationException.class); + + // Sleep 1 second more to check if there was another retry. + TimeUnit.SECONDS.sleep(1); + verify(serviceHandler, times(serviceRetryCount.get())).hello("hello"); + } + } + + @Test + void doNotRetryWhenResponseIsCancelledBeforeRequestComplete() throws Exception { + final AtomicInteger subscriberCancelServiceCallCounterWhenCancelled = new AtomicInteger(-1); + + try (ClientFactory factory = ClientFactory.builder().build()) { + final AtomicReference context = new AtomicReference<>(); + final HelloService.Iface client = + ThriftClients.builder(server.httpUri()) + .path("/thrift") + .factory(factory) + .rpcDecorator(RetryingRpcClient + .builder(retryAlways) + .maxTotalAttempts(10_000) + .newDecorator()) + .rpcDecorator((delegate, ctx, req) -> { + final RpcResponse res = delegate.execute(ctx, req); + subscriberCancelServiceCallCounterWhenCancelled.set( + serviceRetryCount.get()); + res.cancel(true); + context.set(ClientRequestContext.current()); + return res; + }) + .build(HelloService.Iface.class); + when(serviceHandler.hello(anyString())).thenThrow(new IllegalArgumentException()); + + assertThatThrownBy(() -> client.hello("hello")).isInstanceOf(CancellationException.class); + + verify(serviceHandler, never()).hello("hello"); + final RequestLog log = context.get().log().whenComplete().join(); - // ClientUtil.completeLogIfIncomplete() records exceptions caused by response cancellations. + // The request was cancelled midst processing in the decorator. assertThat(log.requestCause()).isExactlyInstanceOf(CancellationException.class); assertThat(log.responseCause()).isExactlyInstanceOf(CancellationException.class); // Sleep 1 second more to check if there was another retry. TimeUnit.SECONDS.sleep(1); - verify(serviceHandler, only()).hello("hello"); + verify(serviceHandler, never()).hello("hello"); } } } From 126729ac28e1b1ea580c0143296c4eb05af327d0 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sun, 7 Sep 2025 23:55:23 +0200 Subject: [PATCH 28/42] fix: RpcRetryAttempt.execute --- .../linecorp/armeria/client/retry/RpcRetryAttempt.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java index dfc06ebdea0..0d0d617e753 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RpcRetryAttempt.java @@ -239,19 +239,19 @@ private RpcResponse execute(Client delegate) { final ClientRequestContextExtension ctxExt = ctx.as(ClientRequestContextExtension.class); - if (!isInitialAttempt && ctxExt != null && ctx.endpoint() == null) { + if (!isInitialAttempt && ctxExt != null && ctx.endpointGroup() != null && ctx.endpoint() == null) { // clear the pending throwable to retry endpoint selection ClientPendingThrowableUtil.removePendingThrowable(ctx); // if the endpoint hasn't been selected, // try to initialize the attempCtx with a new endpoint/event loop return initContextAndExecuteWithFallback( - delegate, ctxExt, RpcResponse::of, + delegate, ctxExt, RpcResponse::from, (context, cause) -> - RpcResponse.ofFailure(cause), req, false); + RpcResponse.ofFailure(cause), req, true); } else { return executeWithFallback(delegate, ctx, (context, cause) -> - RpcResponse.ofFailure(cause), req, false); + RpcResponse.ofFailure(cause), req, true); } } From 402ed1c754efb7ee36a82c04ccfc5d44a3dcc4a6 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Mon, 8 Sep 2025 11:22:44 +0200 Subject: [PATCH 29/42] fix: fix offset mishap in RetryingRpcClientTest.doNotRetryWhenResponseIsCancelledAfterRequestCompletes --- .../armeria/it/client/retry/RetryingRpcClientTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java index c9ab99f1e8f..26b31e3010b 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java @@ -329,7 +329,7 @@ void shouldGetExceptionWhenFactoryIsClosed() throws Exception { } @Test - void doNotRetryWhenResponseIsCancelledAfterRequestComplete() throws Exception { + void doNotRetryWhenResponseIsCancelledAfterRequestCompletes() throws Exception { final AtomicInteger subscriberCancelServiceCallCounterWhenCancelled = new AtomicInteger(-1); try (ClientFactory factory = ClientFactory.builder().build()) { @@ -368,8 +368,9 @@ void doNotRetryWhenResponseIsCancelledAfterRequestComplete() throws Exception { await().untilAsserted(() -> { assertThat(serviceRetryCount.get()).isGreaterThanOrEqualTo(1); // After we cancelled we do not expect further service calls. + // We allow an additional request in case we cancelled concurrently with an outgoing request. assertThat(subscriberCancelServiceCallCounterWhenCancelled.get()).isIn( - serviceRetryCount.get(), serviceRetryCount.get() + 1); + serviceRetryCount.get() - 1, serviceRetryCount.get()); verify(serviceHandler, times(serviceRetryCount.get())).hello("hello"); }); @@ -386,7 +387,7 @@ void doNotRetryWhenResponseIsCancelledAfterRequestComplete() throws Exception { } @Test - void doNotRetryWhenResponseIsCancelledBeforeRequestComplete() throws Exception { + void doNotRetryWhenResponseIsCancelledBeforeRequestCompletes() throws Exception { final AtomicInteger subscriberCancelServiceCallCounterWhenCancelled = new AtomicInteger(-1); try (ClientFactory factory = ClientFactory.builder().build()) { From 1b397b75836980d61872ab1980a96216214a4c9d Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Tue, 9 Sep 2025 10:31:57 +0200 Subject: [PATCH 30/42] docs: fix docs for RetryScheduler --- .../armeria/client/retry/RetryScheduler.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index 03e5410e08a..65a2c1731df 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -59,12 +59,17 @@ interface RetryScheduler { void close(); /** - * Returns a future that is completed + * Returns a future that is completed when the scheduler is closed. It completes *
    *
  • normally, when the scheduler is closed after a successful call to {@link #close()}, or
    • - *
  • exceptionally, when it is not possible to run a retry task once it was meant to be scheduled
    • - *
. - * The future is guaranteed to be completed on the {@code retryEventLoop}. + *
  • exceptionally, when it is not possible to run a retry task once it was meant to be scheduled. + *
    • + * + * + *

    + * The future is guaranteed to be completed on the {@code retryEventLoop}. + * This method must not throw an {@link Exception}. + *

    * * @return a future that is completed when the scheduler is closed */ From 6c6f7e887274fe48ba71bc6987d232d134f741e6 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Tue, 9 Sep 2025 10:35:32 +0200 Subject: [PATCH 31/42] refactor: remove legacy RetryingContext --- .../armeria/client/retry/RetryingContext.java | 75 ------------------- 1 file changed, 75 deletions(-) delete mode 100644 core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java deleted file mode 100644 index 4ea5f14b75b..00000000000 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingContext.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2025 LY Corporation - * - * LY Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.client.retry; - -import java.util.concurrent.CompletableFuture; -import java.util.function.Consumer; - -import com.linecorp.armeria.client.Client; -import com.linecorp.armeria.client.ClientRequestContext; -import com.linecorp.armeria.common.Request; -import com.linecorp.armeria.common.Response; -import com.linecorp.armeria.common.annotation.Nullable; - -/** - * Handle used by {@link AbstractRetryingClient} to drive the retrying process of a request. - * In particular, it provides methods to - * - *
      - *
    • initialize the retrying process ({@link #init()}),
      • - *
    • execute a request attempt ({@link #executeAttempt(Backoff, Client)}),
      • - *
    • decide to abort the attempt and schedule a new one ({@link #abortAttempt()} and - * {@link #scheduleNextRetry(long, Runnable, Consumer)}) or to
      • - *
    • accept the response attempt as the final response of the request ({@link #commit()}).
      • - *
    - * - *

    - * Notes: - *

      - *
    • - * Calls to methods of this interface must be in a certain order. The method call order - * can be "seen" in {@link AbstractRetryingClient#execute(ClientRequestContext, Request)}.
      - * Note that {@link #abort(Throwable)} and {@link #res()} - * can be called at any point, even before a call to {@link #init()}. - *
      • - *
    • Implementors of this interface must be thread-safe.
      • - *
    - *

    - * - * @param the {@link Request} type - * @param the {@link Response} type - * @see HttpRetryingContext - * @see RpcRetryingContext - */ -interface RetryingContext { - CompletableFuture init(); - - CompletableFuture<@Nullable RetryDecision> executeAttempt(@Nullable Backoff backoff, Client delegate); - - long nextRetryTimeNanos(Backoff backoff); - - void scheduleNextRetry(long retryTimeNanos, Runnable retryTask, - Consumer exceptionHandler); - - void commit(); - - void abortAttempt(); - - void abort(Throwable cause); - - O res(); -} From 2f4b69823c898bf623377913a9c4d15edbe2ce97 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Tue, 9 Sep 2025 11:22:53 +0200 Subject: [PATCH 32/42] refactor: remove responsibility of completing original res from RetriedRequest to AbstractRetryingClient - also move out setting the deadline, saving deadlineTimeNanos() from RetriedRequest --- .../client/retry/AbstractRetryingClient.java | 200 ++++++++++++------ .../client/retry/DefaultRetryScheduler.java | 2 + .../client/retry/RetriedHttpRequest.java | 56 ++--- .../armeria/client/retry/RetriedRequest.java | 17 +- .../client/retry/RetriedRpcRequest.java | 79 ++----- .../armeria/client/retry/RetryScheduler.java | 1 - .../armeria/client/retry/RetryingClient.java | 10 +- .../client/retry/RetryingRpcClient.java | 11 +- .../armeria/internal/client/ClientUtil.java | 16 ++ 9 files changed, 208 insertions(+), 184 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 56cb55e7a8b..255f7de17a1 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -17,8 +17,7 @@ import static java.util.Objects.requireNonNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.concurrent.CompletableFuture; import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; @@ -41,8 +40,6 @@ public abstract class AbstractRetryingClient extends SimpleDecoratingClient { - private static final Logger logger = LoggerFactory.getLogger(AbstractRetryingClient.class); - /** * The header which indicates the retry count of a {@link Request}. * The server might use this value to reject excessive retries, etc. @@ -78,18 +75,61 @@ public final O execute(ClientRequestContext ctx, I req) throws Exception { "retryMapping.get() returned null"); final RetryContext rctx = newRetryContext(unwrap(), ctx, req, config); + if (rctx.retryEventLoop().inEventLoop()) { + prepareAndRetry(rctx); + } else { + rctx.retryEventLoop().execute(() -> prepareAndRetry(rctx)); + } + + return rctx.res(); + } + + private void prepareAndRetry(RetryContext rctx) { + assert rctx.retryEventLoop().inEventLoop(); + + try { + prepareRetry(rctx); + } catch (Throwable cause) { + handleUnexpectedException(rctx, cause); + return; + } + + // retry() does not throw. + retry(rctx, null); + } + + private void prepareRetry(RetryContext rctx) { + rctx.res().whenComplete().handle((result, cause) -> { + final Throwable abortCause; + if (cause != null) { + abortCause = cause; + } else { + abortCause = AbortedStreamException.get(); + } + if (rctx.retryEventLoop().inEventLoop()) { + rctx.req().abort(abortCause); + } else { + rctx.retryEventLoop().execute(() -> rctx.req().abort(abortCause)); + } + return null; + }); + // The request could complete at any moment. // In that case, let us make sure that we close the scheduler so we // do not run any retry task unnecessarily. // That said, running retry() even with a completed response is handled by // rctx.request().executeAttempt() which throws an AbortedAttemptException which we handle gracefully. - rctx.request().res().whenComplete().handle((unused, cause) -> { + rctx.req().whenComplete().handle((res, cause) -> { + assert rctx.retryEventLoop().inEventLoop(); // Make sure we do not unnecessarily run any retry task. - if (rctx.retryEventLoop().inEventLoop()) { - rctx.scheduler().close(); + rctx.scheduler().close(); + + if (cause != null) { + rctx.resFuture().completeExceptionally(cause); } else { - rctx.retryEventLoop().execute(rctx.scheduler()::close); + rctx.resFuture().complete(res); } + return null; }); @@ -104,20 +144,13 @@ public final O execute(ClientRequestContext ctx, I req) throws Exception { "retry scheduler was closed before the request was completed"); } - rctx.request().abort(cause); + rctx.req().abort(cause); return null; }); - - if (rctx.retryEventLoop().inEventLoop()) { - retry(rctx, null); - } else { - rctx.retryEventLoop().execute(() -> retry(rctx, null)); - } - - return rctx.request().res(); } // NOTE: + // - Does not throw. // - Must run on the retryEventLoop. // - The first call must be done from execute() above. // - Subsequent calls must only be issued in the retry task scheduled by `rctx.scheduler()`. @@ -126,56 +159,64 @@ private void retry( RetryContext rctx, @Nullable Backoff previousBackoff ) { - // First record the execution of the following attempt. This increases the attempt count - // and the attempt count for the backoff. Also see (A) for a correctness argument. - rctx.counter().consumeAttemptFrom(previousBackoff); - - rctx.request() - .executeAttempt(rctx.delegate()) - .handle((executionResult, cause) -> { - assert rctx.retryEventLoop().inEventLoop(); - - if (cause != null) { - if (cause instanceof AbortedAttemptException) { - // The attempt was aborted in the course of executing it. This can happen when - // we execute an attempt on a completed request or when the request is completed while - // RetriedRequest is waiting for the response of the attempt. + try { + // First record the execution of the following attempt. This increases the attempt count + // and the attempt count for the backoff. Also see (A) for a correctness argument. + rctx.counter().consumeAttemptFrom(previousBackoff); + + rctx.req() + .executeAttempt(rctx.delegate()) + .handle((executionResult, cause) -> { + assert rctx.retryEventLoop().inEventLoop(); + + if (cause != null) { + rctx.req().abort( + new IllegalStateException("expected RetriedRequest to be completed")); + + if (cause instanceof AbortedAttemptException) { + // The attempt was aborted in the course of executing it. This can happen when + // we execute an attempt on a completed request or when the request is completed + // while RetriedRequest is waiting for the response of the attempt. + return null; + } + return null; } - // Make sure we complete the request with the exception. We could have been aborted because - // another attempt already completed the request but this is not an issue as the call to - // `request.abort()` has no effect then. - rctx.request().abort( - cause != null ? cause : new IllegalStateException("executionResult is null")); - return null; - } - // An empty backoff means that the RetryRule to commit this attempt. - if (executionResult.decision().backoff() == null) { - rctx.request().commit(executionResult.attemptNumber()); + // An empty backoff means that the RetryRule to commit this attempt. + if (executionResult.decision().backoff() == null) { + rctx.req().commit(executionResult.attemptNumber()); + return null; + } + + // Note that applying the pushback needs to be done before the call + // to tryScheduleRetryAfter`. + if (executionResult.minimumBackoffMillis() > 0) { + rctx.scheduler().applyMinimumBackoffMillisForNextRetry( + executionResult.minimumBackoffMillis()); + } + + tryScheduleRetryAfter(rctx, executionResult.decision().backoff()); + + if (!rctx.scheduler().hasNextRetryTask()) { + // We are not going to retry again so we are the last attempt. Let us commit it even if + // the response was deemed to be unsatisfactory by the RetryRule (backoff != null). + rctx.req().commit(executionResult.attemptNumber()); + } else { + // The responsibility of aborting the RetriedRequest is now with the retry + // scheduled by `tryScheduleRetryAfter()`. Thus, we can safely abort this attempt now. + rctx.req().abort(executionResult.attemptNumber(), AbortedStreamException.get()); + } + return null; - } - - // Note that applying the pushback needs to be done before the call to tryScheduleRetryAfter`. - if (executionResult.minimumBackoffMillis() > 0) { - rctx.scheduler().applyMinimumBackoffMillisForNextRetry( - executionResult.minimumBackoffMillis()); - } - - tryScheduleRetryAfter(rctx, executionResult.decision().backoff()); - - if (!rctx.scheduler().hasNextRetryTask()) { - // We are not going to retry again so we are the last attempt. Let us commit it even if - // the response was deemed to be unsatisfactory by the RetryRule (backoff != null). - rctx.request().commit(executionResult.attemptNumber()); - } else { - // The responsibility of aborting the RetriedRequest is now with the retry task scheduled - // by `tryScheduleRetryAfter()`. Thus, we can safely abort this attempt now. - rctx.request().abort(executionResult.attemptNumber(), AbortedStreamException.get()); - } - - return null; - }); + }) + .exceptionally(cause -> { + handleUnexpectedException(rctx, cause); + return null; + }); + } catch (Throwable cause) { + handleUnexpectedException(rctx, cause); + } } // NOTE: Must run on the retryEventLoop. @@ -208,29 +249,48 @@ private void tryScheduleRetryAfter(RetryContext rctx, Backoff nextBackoff) { nextRetryDelayMillisFromBackoff); } - protected final class RetryContext { + private void handleUnexpectedException(RetryContext rctx, Throwable cause) { + assert rctx.retryEventLoop().inEventLoop(); + // Aborting the request will trigger the cleanup logic in prepareRetry(). + rctx.req().abort(new IllegalStateException("Unexpected exception during retrying", cause)); + } + protected final class RetryContext { final EventLoop retryEventLoop; - final RetriedRequest request; + final RetriedRequest req; final RetryScheduler scheduler; final RetryCounter counter; final Client delegate; + final O res; + final CompletableFuture resFuture; - RetryContext(EventLoop retryEventLoop, RetriedRequest request, - RetryScheduler scheduler, RetryCounter counter, Client delegate) { + RetryContext(EventLoop retryEventLoop, RetriedRequest req, + RetryScheduler scheduler, RetryCounter counter, Client delegate, + O res, + CompletableFuture resFuture) { this.retryEventLoop = retryEventLoop; - this.request = request; + this.req = req; this.scheduler = scheduler; this.counter = counter; this.delegate = delegate; + this.res = res; + this.resFuture = resFuture; } EventLoop retryEventLoop() { return retryEventLoop; } - RetriedRequest request() { - return request; + RetriedRequest req() { + return req; + } + + O res() { + return res; + } + + CompletableFuture resFuture() { + return resFuture; } RetryScheduler scheduler() { diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java index 3d16db3d509..63f6b79c167 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java @@ -69,6 +69,8 @@ public void run() { try { retryTaskToRun.run(); } catch (Throwable t) { + // Normally we are running retry() which does not throw an exception + // but let us be defensive here. closeExceptionally(t); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java index 0623a07d7ab..7d02094acfa 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedHttpRequest.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; -import java.util.concurrent.TimeUnit; import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; @@ -34,7 +33,6 @@ import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.RequestHeadersBuilder; import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.stream.AbortedStreamException; import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.internal.client.AggregatedHttpRequestDuplicator; import com.linecorp.armeria.internal.client.ClientUtil; @@ -58,6 +56,7 @@ private enum State { * Terminal state. Either {@link #commit(int)} or {@link #abort(Throwable)} was called. * {@link RetriedHttpRequest} will not execute any more attempts in that it completes every call * to {@link #executeAttempt(Client)} exceptionally with an {@link AbortedAttemptException}. + * {@link #completedFuture} is completed. */ COMPLETED } @@ -65,12 +64,11 @@ private enum State { private final ContextAwareEventLoop retryEventLoop; private final RetryConfig config; private final ClientRequestContext ctx; - private final HttpResponse res; - private final CompletableFuture resFuture; private final HttpRequest req; private final long deadlineTimeNanos; private final boolean useRetryAfter; + private final CompletableFuture completedFuture; private State state; @@ -94,23 +92,18 @@ private enum State { RetryConfig config, ClientRequestContext ctx, HttpRequest req, - HttpResponse res, CompletableFuture resFuture, boolean useRetryAfter + long deadlineTimeNanos, + boolean useRetryAfter ) { this.retryEventLoop = retryEventLoop; this.ctx = ctx; - this.res = res; - this.resFuture = resFuture; this.req = req; this.config = config; this.useRetryAfter = useRetryAfter; - initFuture = new CompletableFuture<>(); + this.deadlineTimeNanos = deadlineTimeNanos; - final long responseTimeoutMillis = ctx.responseTimeoutMillis(); - if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { - deadlineTimeNanos = Long.MAX_VALUE; - } else { - deadlineTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); - } + completedFuture = new CompletableFuture<>(); + initFuture = new CompletableFuture<>(); state = State.IDLE; previousAttemptNumber = 0; @@ -130,7 +123,6 @@ public CompletionStage executeAttempt( * Initializes the {@link RetriedHttpRequest}. This method is idempotent. * Initialization includes * - subscribing to {@link #req} to initiate abortion in case its fails, - * - subscribing to {@link #res} to initiate abortion in case it fails or completes, and * - building and setting {@link #reqDuplicator}. * * @return a future that completes when the initialization is done. After the future completes @@ -159,21 +151,6 @@ private CompletableFuture init() { return null; }); - res.whenComplete().handle((result, cause) -> { - final Throwable abortCause; - if (cause != null) { - abortCause = cause; - } else { - abortCause = AbortedStreamException.get(); - } - if (retryEventLoop.inEventLoop()) { - abort(abortCause); - } else { - retryEventLoop.execute(() -> abort(abortCause)); - } - return null; - }); - // Guaranteed to be completed on the retryEventLoop. final CompletableFuture reqDuplicatorFuture; @@ -286,23 +263,25 @@ public void commit(int attemptNumber) { } checkState(state == State.PENDING); assert reqDuplicator != null; + assert !completedFuture.isDone(); checkArgument(attemptNumber >= 1); checkArgument(attemptNumber <= attempts.size()); final HttpRetryAttempt attemptToCommit = attempts.get(attemptNumber - 1); - checkState(attemptToCommit != null); checkState(attemptToCommit.state() == HttpRetryAttempt.State.DECIDED); state = State.COMPLETED; abortAllExcept(attemptToCommit); + final HttpResponse res = attemptToCommit.commit(); + reqDuplicator.close(); - final HttpResponse attemptRes = attemptToCommit.commit(); ctx.logBuilder().endResponseWithChild(attemptToCommit.ctx().log()); - resFuture.complete(attemptRes); + + completedFuture.complete(res); } @Override @@ -312,6 +291,7 @@ public void abort(Throwable cause) { if (state == State.COMPLETED) { return; } + assert !completedFuture.isDone(); state = State.COMPLETED; @@ -325,7 +305,7 @@ public void abort(Throwable cause) { ctx.logBuilder().endRequest(cause); } ctx.logBuilder().endResponse(cause); - resFuture.completeExceptionally(cause); + completedFuture.completeExceptionally(cause); } @Override @@ -359,11 +339,7 @@ private void abortAllExcept(@Nullable HttpRetryAttempt winningAttempt) { } @Override - public HttpResponse res() { - return res; - } - - public long deadlineTimeNanos() { - return deadlineTimeNanos; + public CompletableFuture whenComplete() { + return completedFuture; } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java index beab4300721..967074076d1 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRequest.java @@ -15,6 +15,7 @@ */ package com.linecorp.armeria.client.retry; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import com.linecorp.armeria.client.Client; @@ -26,7 +27,8 @@ * A retried request that manages multiple retry attempts. * *

    - * NOTE: All methods of {@link RetriedRequest} must be invoked from a single-threaded event loop. + * NOTE: All methods of {@link RetriedRequest} must be invoked from a single-threaded event loop ("retry event + * loop"). * Implementations of {@link RetriedRequest} therefore do not need to be thread-safe. *

    * @@ -141,8 +143,15 @@ public long minimumBackoffMillis() { void abort(Throwable cause); /** - * Returns the original {@link Response}. In case of a call to {@link #abort(Throwable)}, the returned - * {@link Response} is completed exceptionally with the same {@code cause}. + * Returns a future that completes when this {@link RetriedRequest} is completed. + *
      + *
    • Completes successfully with the response of the attempt committed by {@link #commit(int)}
      • + *
    • Completes exceptionally with the cause from {@link #abort(Throwable)}
      • + *
    + * Note: {@link #abort(Throwable)} may be called internally when unexpected errors occur. + * + * @return a future that completes successfully with the committed attempt's response or + * exceptionally with the abort cause. It is guaranteed to be completed on the retry event loop. */ - O res(); + CompletableFuture whenComplete(); } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRpcRequest.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRpcRequest.java index 7dc3c080b24..303fc3878a6 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRpcRequest.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetriedRpcRequest.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; -import java.util.concurrent.TimeUnit; import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; @@ -31,7 +30,6 @@ import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.stream.AbortedStreamException; import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.internal.client.ClientUtil; @@ -55,6 +53,7 @@ private enum State { * Terminal state. Either {@link #commit(int)} or {@link #abort(Throwable)} was called. * {@link RetriedRpcRequest} will not execute any more attempts in that it completes every call * to {@link #executeAttempt(Client)} with an {@link AbortedAttemptException}. + * {@link #completedFuture} is completed. */ COMPLETED } @@ -62,9 +61,8 @@ private enum State { private final ContextAwareEventLoop retryEventLoop; private final RetryConfig config; private final ClientRequestContext ctx; - private final RpcResponse res; - private final CompletableFuture resFuture; private final RpcRequest req; + private final CompletableFuture completedFuture; private final long deadlineTimeNanos; @@ -79,21 +77,14 @@ private enum State { RetryConfig config, ClientRequestContext ctx, RpcRequest req, - RpcResponse res, CompletableFuture resFuture + long deadlineTimeNanos ) { this.retryEventLoop = retryEventLoop; this.ctx = ctx; - this.res = res; - this.resFuture = resFuture; this.req = req; this.config = config; - - final long responseTimeoutMillis = ctx.responseTimeoutMillis(); - if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { - deadlineTimeNanos = Long.MAX_VALUE; - } else { - deadlineTimeNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); - } + this.deadlineTimeNanos = deadlineTimeNanos; + completedFuture = new CompletableFuture<>(); state = State.IDLE; previousAttemptNumber = 0; @@ -104,50 +95,16 @@ private enum State { @Override public CompletionStage executeAttempt( Client delegate) { - checkState(ctx.eventLoop().inEventLoop()); - init(); - return executeAttemptAfterInit(delegate); - } - - /** - * Initializes the {@link RetriedRpcRequest}. This method is idempotent. - * Initialization mean subscribing to the original {@link RpcResponse} so that we can - * abort all attempts when the original {@link RpcResponse} is completed. - */ - private void init() { - assert ctx.eventLoop().inEventLoop(); - - if (state == State.PENDING || state == State.COMPLETED) { - return; - } - - assert state == State.IDLE : state; - state = State.PENDING; - - res.whenComplete().handle((result, cause) -> { - final Throwable abortCause; - if (cause != null) { - abortCause = cause; - } else { - abortCause = AbortedStreamException.get(); - } - if (retryEventLoop.inEventLoop()) { - abort(abortCause); - } else { - retryEventLoop.execute(() -> abort(abortCause)); - } - return null; - }); - } - - private CompletionStage executeAttemptAfterInit( - Client delegate) { - assert retryEventLoop.inEventLoop(); + checkState(retryEventLoop.inEventLoop()); if (state == State.COMPLETED) { return UnmodifiableFuture.exceptionallyCompletedFuture(AbortedAttemptException.get()); } + if (state == State.IDLE) { + state = State.PENDING; + } + // We need to be initialized/ the reqDuplicator must be present. checkState(state == State.PENDING); checkState(previousAttemptNumber < config.maxTotalAttempts()); @@ -220,7 +177,8 @@ public void commit(int attemptNumber) { if (attemptReq != null) { ctx.updateRequest(attemptReq); } - resFuture.complete(attemptRes); + + completedFuture.complete(attemptRes); } @Override @@ -231,6 +189,8 @@ public void abort(Throwable cause) { return; } + assert !completedFuture.isDone(); + state = State.COMPLETED; abortAllExcept(null); @@ -239,7 +199,8 @@ public void abort(Throwable cause) { ctx.logBuilder().endRequest(cause); } ctx.logBuilder().endResponse(cause); - resFuture.completeExceptionally(cause); + + completedFuture.completeExceptionally(cause); } @Override @@ -273,11 +234,7 @@ private void abortAllExcept(@Nullable RpcRetryAttempt winningAttempt) { } @Override - public RpcResponse res() { - return res; - } - - public long deadlineTimeNanos() { - return deadlineTimeNanos; + public CompletableFuture whenComplete() { + return completedFuture; } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index 65a2c1731df..6902d3edc14 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -68,7 +68,6 @@ interface RetryScheduler { * *

    * The future is guaranteed to be completed on the {@code retryEventLoop}. - * This method must not throw an {@link Exception}. *

    * * @return a future that is completed when the scheduler is closed diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java index a29267d7749..47c2d8a41ed 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingClient.java @@ -29,6 +29,7 @@ import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.internal.client.ClientUtil; /** * An {@link HttpClient} decorator that handles failures of an invocation and retries HTTP requests. @@ -221,17 +222,18 @@ RetryContext newRetryContext( final ContextAwareEventLoop retryEventLoop = ctx.eventLoop(); - final RetriedHttpRequest request = new RetriedHttpRequest( - retryEventLoop, config, ctx, req, res, resFuture, useRetryAfter + final long deadlineTimeNanos = ClientUtil.deadlineTimeNanos(ctx); + final RetriedHttpRequest retriedReq = new RetriedHttpRequest( + retryEventLoop, config, ctx, req, deadlineTimeNanos, useRetryAfter ); final RetryScheduler scheduler = new DefaultRetryScheduler( retryEventLoop, - request.deadlineTimeNanos() + deadlineTimeNanos ); final RetryCounter counter = new DefaultRetryCounter(config.maxTotalAttempts()); - return new RetryContext(retryEventLoop, request, scheduler, counter, delegate); + return new RetryContext(retryEventLoop, retriedReq, scheduler, counter, delegate, res, resFuture); } } diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java index 1ae0f4c2ac5..9e40a826b1e 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryingRpcClient.java @@ -25,6 +25,7 @@ import com.linecorp.armeria.common.Request; import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; +import com.linecorp.armeria.internal.client.ClientUtil; /** * An {@link RpcClient} decorator that handles failures of an invocation and retries RPC requests. @@ -144,17 +145,19 @@ RetryContext newRetryContext( final ContextAwareEventLoop retryEventLoop = ctx.eventLoop(); - final RetriedRpcRequest request = new RetriedRpcRequest( - retryEventLoop, config, ctx, req, res, resFuture + final long deadlineTimeNanos = ClientUtil.deadlineTimeNanos(ctx); + + final RetriedRpcRequest retriedReq = new RetriedRpcRequest( + retryEventLoop, config, ctx, req, deadlineTimeNanos ); final RetryScheduler scheduler = new DefaultRetryScheduler( retryEventLoop, - request.deadlineTimeNanos() + deadlineTimeNanos ); final RetryCounter counter = new DefaultRetryCounter(config.maxTotalAttempts()); - return new RetryContext(retryEventLoop, request, scheduler, counter, delegate); + return new RetryContext(retryEventLoop, retriedReq, scheduler, counter, delegate, res, resFuture); } } diff --git a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java index 7fac3b8545e..44427266121 100644 --- a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java +++ b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java @@ -332,6 +332,22 @@ public static long retryAfterMillis(RequestLogAccess log) { return -1; } + /** + * Returns the deadline in nanoseconds for the given {@link ClientRequestContext} based on its + * response timeout. If no response timeout is set, {@link Long#MAX_VALUE} will be returned. + * + * @param ctx the {@link ClientRequestContext} to get the response timeout from + * @return the deadline in nanoseconds, or {@link Long#MAX_VALUE} if no response timeout is set + */ + public static long deadlineTimeNanos(ClientRequestContext ctx) { + final long responseTimeoutMillis = ctx.responseTimeoutMillis(); + if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { + return Long.MAX_VALUE; + } else { + return System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); + } + } + /** * Sets the response timeout on a {@link ClientRequestContext} based on the minimum of a deadline * and response timeout, or clears the timeout if neither is specified. From df10849d166ee6c78d8ddd1637b52de9f0e2ddf6 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Tue, 9 Sep 2025 13:32:13 +0200 Subject: [PATCH 33/42] fix: fix RetryingRpcClientTest.doNotRetryWhenResponseIsCancelled --- .../client/retry/RetryingRpcClientTest.java | 117 ++++++++++++++++-- 1 file changed, 107 insertions(+), 10 deletions(-) diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java index 99ec07c9752..ca72a897248 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; +import static org.assertj.core.api.Fail.fail; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; @@ -28,8 +29,10 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.time.Duration; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CancellationException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.LinkedTransferQueue; import java.util.concurrent.TimeUnit; @@ -39,6 +42,8 @@ import org.apache.thrift.TApplicationException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientRequestContext; @@ -53,6 +58,7 @@ import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.RpcResponse; import com.linecorp.armeria.common.logging.RequestLog; +import com.linecorp.armeria.common.util.TimeoutMode; import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.thrift.THttpService; @@ -62,10 +68,10 @@ import testing.thrift.main.HelloService; class RetryingRpcClientTest { - + private static final Backoff fixedBackoff = Backoff.fixed(500); private static final RetryRuleWithContent retryAlways = (ctx, response, cause) -> - UnmodifiableFuture.completedFuture(RetryDecision.retry(Backoff.fixed(500))); + UnmodifiableFuture.completedFuture(RetryDecision.retry(fixedBackoff)); private static final RetryRuleWithContent retryOnException = RetryRuleWithContent.onException(Backoff.withoutDelay()); @@ -326,34 +332,125 @@ void shouldGetExceptionWhenFactoryIsClosed() throws Exception { "(?i).*(factory has been closed|not accepting a task).*")); } - @Test - void doNotRetryWhenResponseIsCancelled() throws Exception { + enum DoNotRetryWhenResponseIsCancelledTestParams { + // Cancel delays for a backoff of 50 milliseconds (quickBackoffMillis). + CANCEL_FIRST_REQUEST_NO_DELAY(true, 0), + CANCEL_FIRST_REQUEST_WITH_DELAY(true, 500), + CANCEL_AFTER_FIRST_REQUEST_NO_DELAY(false, 0), + CANCEL_AFTER_FIRST_REQUEST_WITH_DELAY(false, 500); + + static final int BACKOFF_MILLIS = 50; + final boolean ensureCancelBeforeFirstRequest; + final long cancelDelayMillis; + + DoNotRetryWhenResponseIsCancelledTestParams(boolean ensureCancelBeforeFirstRequest, + long cancelDelayMillis) { + this.ensureCancelBeforeFirstRequest = ensureCancelBeforeFirstRequest; + this.cancelDelayMillis = cancelDelayMillis; + } + } + + @ParameterizedTest + @EnumSource(DoNotRetryWhenResponseIsCancelledTestParams.class) + void doNotRetryWhenResponseIsCancelled(DoNotRetryWhenResponseIsCancelledTestParams param) throws Exception { serviceRetryCount.set(0); + + final RetryRuleWithContent quickRetryAlways = + RetryRuleWithContent.builder() + .onException() + .thenBackoff(Backoff.fixed( + DoNotRetryWhenResponseIsCancelledTestParams.BACKOFF_MILLIS)); + + final int maxExpectedAttempts = + (int) (param.cancelDelayMillis / DoNotRetryWhenResponseIsCancelledTestParams.BACKOFF_MILLIS) + + 5; + final AtomicInteger serviceRetryCountWhenCancelled = new AtomicInteger(); + final CountDownLatch canRetry = new CountDownLatch(1); try (ClientFactory factory = ClientFactory.builder().build()) { final AtomicReference context = new AtomicReference<>(); final HelloService.Iface client = ThriftClients.builder(server.httpUri()) .path("/thrift") .factory(factory) - .rpcDecorator(RetryingRpcClient.builder(retryAlways).newDecorator()) + .rpcDecorator(RetryingRpcClient.builder(quickRetryAlways) + // We want to cancel the request before + // we quit because of reaching max attempts. + .maxTotalAttempts(maxExpectedAttempts) + .newDecorator()) + .rpcDecorator((delegate, ctx, req) -> { + // Clog the retry event loop so we do not retry until canRetry.countDown() + // is called. + // If you see failure of this test, and you altered AbstractRetryingClient, + // make sure you are executing (prepare)Retry() on the retry event loop and + // that the retry event loop is ctx.eventLoop(). + ctx.eventLoop().execute(() -> { + try { + canRetry.await(); + } catch (InterruptedException e) { + fail(e); + } + }); + + return delegate.execute(ctx, req); + }) .rpcDecorator((delegate, ctx, req) -> { - context.set(ctx); final RpcResponse res = delegate.execute(ctx, req); - res.cancel(true); + + if (param.ensureCancelBeforeFirstRequest) { + Thread.sleep(param.cancelDelayMillis); + assertThat(res.isDone()).isFalse(); + res.cancel(true); + serviceRetryCountWhenCancelled.set(serviceRetryCount.get()); + canRetry.countDown(); + } else { + canRetry.countDown(); + Thread.sleep(param.cancelDelayMillis); + assertThat(res.isDone()).isFalse(); + res.cancel(true); + serviceRetryCountWhenCancelled.set(serviceRetryCount.get()); + } + return res; }) + .rpcDecorator((delegate, ctx, req) -> { + context.set(ctx); + ctx.setResponseTimeout( + TimeoutMode.EXTEND, + Duration.ofMillis(param.cancelDelayMillis + 1000) + ); + + return delegate.execute(ctx, req); + }) .build(HelloService.Iface.class); when(serviceHandler.hello(anyString())).thenThrow(new IllegalArgumentException()); assertThatThrownBy(() -> client.hello("hello")).isInstanceOf(CancellationException.class); await().untilAsserted(() -> { - verify(serviceHandler, only()).hello("hello"); + assertThat(serviceRetryCountWhenCancelled.get()).isIn(serviceRetryCount.get(), + serviceRetryCount.get() - 1); + verify(serviceHandler, times(serviceRetryCount.get())).hello("hello"); }); + + final RequestLog log = context.get().log().whenComplete().join(); + if (param.ensureCancelBeforeFirstRequest) { + assertThat(serviceRetryCount.get()).isZero(); + assertThat(log.requestCause()).isExactlyInstanceOf(CancellationException.class); + assertThat(log.responseCause()).isExactlyInstanceOf(CancellationException.class); + } else { + // We still could cancel the before the first request so we do not have a guarantee for + // requestCause() to be null. + assertThat(log.responseCause()).isExactlyInstanceOf(CancellationException.class); + } + // Sleep 1 second more to check if there was another retry. TimeUnit.SECONDS.sleep(1); - verify(serviceHandler, only()).hello("hello"); - assertThat(serviceRetryCount).hasValue(1); + if (param.ensureCancelBeforeFirstRequest) { + assertThat(serviceRetryCount.get()).isZero(); + } + assertThat(serviceRetryCountWhenCancelled.get()).isIn(serviceRetryCount.get(), + serviceRetryCount.get() - 1); + verify(serviceHandler, times(serviceRetryCount.get())).hello("hello"); } } } From c7a607f8f50df9cac3439c8565d0315b76eb6f9b Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 11 Sep 2025 13:00:45 +0200 Subject: [PATCH 34/42] fix: fix blockhound for RetryingRpcClientTest.doNotRetryWhenResponseIsCancelled --- .../armeria/it/client/retry/RetryingRpcClientTest.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java index ca72a897248..f10044b1c14 100644 --- a/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java +++ b/thrift/thrift0.13/src/test/java/com/linecorp/armeria/it/client/retry/RetryingRpcClientTest.java @@ -19,7 +19,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; -import static org.assertj.core.api.Fail.fail; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; @@ -60,6 +59,7 @@ import com.linecorp.armeria.common.logging.RequestLog; import com.linecorp.armeria.common.util.TimeoutMode; import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.armeria.internal.testing.BlockingUtils; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.thrift.THttpService; import com.linecorp.armeria.testing.junit5.server.ServerExtension; @@ -384,11 +384,9 @@ void doNotRetryWhenResponseIsCancelled(DoNotRetryWhenResponseIsCancelledTestPara // make sure you are executing (prepare)Retry() on the retry event loop and // that the retry event loop is ctx.eventLoop(). ctx.eventLoop().execute(() -> { - try { + BlockingUtils.blockingRun(() -> { canRetry.await(); - } catch (InterruptedException e) { - fail(e); - } + }); }); return delegate.execute(ctx, req); From 07f3fd7982e8029f36bb405297b0ecb529eb5995 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 11 Sep 2025 13:10:04 +0200 Subject: [PATCH 35/42] fix: skip retry task when scheduler after deadline in DefaultRetryScheduler --- .../com/linecorp/armeria/client/retry/DefaultRetryScheduler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java index 63f6b79c167..8e3159873e3 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java @@ -58,6 +58,7 @@ public void run() { if (System.nanoTime() > deadlineTimeNanos) { logger.debug("Tried to run a retry task after the deadline. Skipping this task."); + return; } assert nextRetryTask != null; From 1fb42593afc754c53f3c05873c9cb2f4ea8f1ea1 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Thu, 11 Sep 2025 14:13:36 +0200 Subject: [PATCH 36/42] refactor: simplify RetryScheduler for sequential retrying --- .../client/retry/AbstractRetryingClient.java | 21 ++++---- .../client/retry/DefaultRetryScheduler.java | 15 +++--- .../armeria/client/retry/RetryScheduler.java | 51 +++++++++---------- 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java index 255f7de17a1..0e040313341 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/AbstractRetryingClient.java @@ -94,8 +94,8 @@ private void prepareAndRetry(RetryContext rctx) { return; } - // retry() does not throw. - retry(rctx, null); + // Let us execute retry() through the scheduler so that it sees the first retry task. + assert rctx.scheduler().trySchedule(() -> retry(rctx, null), 0); } private void prepareRetry(RetryContext rctx) { @@ -152,7 +152,7 @@ private void prepareRetry(RetryContext rctx) { // NOTE: // - Does not throw. // - Must run on the retryEventLoop. - // - The first call must be done from execute() above. + // - The first call must be done from prepareAndRetry() above. // - Subsequent calls must only be issued in the retry task scheduled by `rctx.scheduler()`. // The corresponding `scheduler.schedule()` calls must all be done from ´tryScheduleRetryAfter()´ below. private void retry( @@ -196,9 +196,10 @@ private void retry( executionResult.minimumBackoffMillis()); } - tryScheduleRetryAfter(rctx, executionResult.decision().backoff()); + final boolean isNextRetryScheduledOrExecuted = + tryScheduleRetryAfter(rctx, executionResult.decision().backoff()); - if (!rctx.scheduler().hasNextRetryTask()) { + if (!isNextRetryScheduledOrExecuted) { // We are not going to retry again so we are the last attempt. Let us commit it even if // the response was deemed to be unsatisfactory by the RetryRule (backoff != null). rctx.req().commit(executionResult.attemptNumber()); @@ -220,9 +221,9 @@ private void retry( } // NOTE: Must run on the retryEventLoop. - private void tryScheduleRetryAfter(RetryContext rctx, Backoff nextBackoff) { + private boolean tryScheduleRetryAfter(RetryContext rctx, Backoff nextBackoff) { if (rctx.counter().hasReachedMaxAttempts()) { - return; + return false; } final long nextRetryDelayMillisFromBackoff = nextBackoff.nextDelayMillis( @@ -234,7 +235,7 @@ private void tryScheduleRetryAfter(RetryContext rctx, Backoff nextBackoff) { if (nextRetryDelayMillisFromBackoff < 0) { // We exceeded the attempt limit for the backoff. - return; + return false; } // (A): We are under `maxAttempts` have also not exceeded the backoff attempt limit so let us @@ -245,8 +246,8 @@ private void tryScheduleRetryAfter(RetryContext rctx, Backoff nextBackoff) { // are the same. // 2. Based on 1., the delay we use to schedule the retry task is indeed the one we consume in the retry // task via `counter.consumeAttemptFrom(nextBackoff)`. - rctx.scheduler().trySchedule(/* retry task */ () -> retry(rctx, nextBackoff), - nextRetryDelayMillisFromBackoff); + return rctx.scheduler().trySchedule(/* retry task */ () -> retry(rctx, nextBackoff), + nextRetryDelayMillisFromBackoff); } private void handleUnexpectedException(RetryContext rctx, Throwable cause) { diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java index 8e3159873e3..1b7c216a791 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java @@ -90,11 +90,11 @@ public void run() { } @Override - public void trySchedule(Runnable retryTask, long delayMillis) { + public boolean trySchedule(Runnable retryTask, long delayMillis) { checkInRetryEventLoop(); if (isClosed()) { - return; + return false; } // We are a sequential scheduler there must not be a nextRetryTask already set. @@ -107,7 +107,7 @@ public void trySchedule(Runnable retryTask, long delayMillis) { final long retryTimeNanos = Math.max(System.nanoTime() + delayNanos, earliestRetryTimeNanos); if (retryTimeNanos > deadlineTimeNanos) { - return; + return false; } try { @@ -116,8 +116,9 @@ public void trySchedule(Runnable retryTask, long delayMillis) { nextRetryTask = retryTask; if (nextRetryDelayMillis <= 0) { + // Run immediately. nextRetryTaskFuture = null; - retryEventLoop.execute(retryTaskWrapper); + retryTaskWrapper.run(); } else { nextRetryTaskFuture = retryEventLoop.schedule( retryTaskWrapper, nextRetryDelayMillis, @@ -137,8 +138,11 @@ public void trySchedule(Runnable retryTask, long delayMillis) { } }); } + + return true; } catch (Throwable t) { closeExceptionally(t); + return false; } } @@ -161,8 +165,7 @@ public void applyMinimumBackoffMillisForNextRetry(long minimumBackoffMillis) { ); } - @Override - public boolean hasNextRetryTask() { + private boolean hasNextRetryTask() { checkInRetryEventLoop(); // NOTE: nextRetryTask is null when scheduler is closed. return nextRetryTask != null; diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java index 6902d3edc14..72d46ec1064 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/RetryScheduler.java @@ -22,11 +22,6 @@ * while enforcing a fixed request deadline and minimum backoff between attempts. * *

    - * Sequential means that {@link #trySchedule(Runnable, long)} can only be called with not other retry task - * scheduled (i.e. with {@link #hasNextRetryTask()} returning {@code false}). - *

    - * - *

    * NOTE: All methods of {@link RetryScheduler} must be invoked from the * {@code retryEventLoop}. Implementations of {@link RetryScheduler} therefore do not need to be thread-safe. *

    @@ -34,23 +29,37 @@ interface RetryScheduler { /** * Tries to schedule the given {@code retryTask} to be run after the given {@code delayMillis} - * or as early as the minimum backoff delays applied for the next schedule allow. Scheduling might - * not succeed which can be checked by calling {@link #hasNextRetryTask()}. + * or as early as the minimum backoff delays applied for the next schedule allow. + * Implementations are advised to execute the {@code retryTask} directly if + * {@code delayMillis} is {@code 0}. + * + *

    + * The method returns {@code true} if the {@code retryTask} was successfully scheduled or + * {@code false} if not. Note that a return value of {@code false} does not necessarily indicate that + * something exceptional happened during scheduling. For example, {@code delayMillis} might be too long + * for the deadline in which case this method returns {@code false} + * but those events would not {@link #close()} the scheduler exceptionally. + *

    * * @param retryTask the task to run to perform a retry * @param delayMillis the delay in milliseconds after which the {@code retryTask} should be run + * + * @return {@code true} if the {@code retryTask} was successfully scheduled or {@code false} if not. */ - void trySchedule(Runnable retryTask, long delayMillis); + boolean trySchedule(Runnable retryTask, long delayMillis); /** - * Returns {@code true} if there is a retry task scheduled. If this method returns {@code true}, - * the scheduler guarantees that it is going to run a retry task at some point in the future - * on the retryEventLoop. If it is not possible to run a retry task, the scheduler guarantees - * to complete the future from {@link #whenClosed()} exceptionally. + * Applies the given {@code minimumBackoffMillis} for the next retry scheduling. + * This means that the next retry task will not be scheduled before the given + * {@code minimumBackoffMillis} has elapsed since the call to this method. * - * @return {@code true} if there is a retry task scheduled + *

    + * This method must not be called while executing another retry task. + *

    + * + * @param minimumBackoffMillis the minimum backoff in milliseconds to apply for the next retry scheduling */ - boolean hasNextRetryTask(); + void applyMinimumBackoffMillisForNextRetry(long minimumBackoffMillis); /** * Closes the scheduler which is cancelling the next retry task if any and @@ -73,18 +82,4 @@ interface RetryScheduler { * @return a future that is completed when the scheduler is closed */ CompletableFuture whenClosed(); - - /** - * Applies the given {@code minimumBackoffMillis} for the next retry scheduling. - * This means that the next retry task will not be scheduled before the given - * {@code minimumBackoffMillis} has elapsed since the call to this method. - * - *

    - * This method must not be called when {@link #hasNextRetryTask()} returns {@code true} as the scheduler - * only supports sequential retrying at the moment. - *

    - * - * @param minimumBackoffMillis the minimum backoff in milliseconds to apply for the next retry scheduling - */ - void applyMinimumBackoffMillisForNextRetry(long minimumBackoffMillis); } From c5b883bdf2d8c48f09fb54e07b5335e1c0c64af9 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 12 Sep 2025 14:45:39 +0200 Subject: [PATCH 37/42] test: add RetrySchedulerTest --- .../client/retry/DefaultRetryScheduler.java | 61 +- .../client/retry/RetrySchedulerTest.java | 719 ++++++++++++++++++ 2 files changed, 757 insertions(+), 23 deletions(-) create mode 100644 core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java index 1b7c216a791..b9c68ce1895 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java @@ -23,7 +23,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.math.LongMath; + import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.ResponseTimeoutException; import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.util.UnmodifiableFuture; @@ -45,19 +48,20 @@ final class DefaultRetryScheduler implements RetryScheduler { private ScheduledFuture nextRetryTaskFuture; // Long.MIN_VALUE if not set. private long earliestRetryTimeNanos; + private boolean isClosed; private final class RetryTaskWrapper implements Runnable { @Override public void run() { assert retryEventLoop.inEventLoop(); - if (closedFuture.isDone()) { + if (isClosed) { logger.debug("Tried to run a retry task after the scheduler was closed. Skipping this task."); return; } if (System.nanoTime() > deadlineTimeNanos) { - logger.debug("Tried to run a retry task after the deadline. Skipping this task."); + closeExceptionally(ResponseTimeoutException.get()); return; } @@ -87,13 +91,14 @@ public void run() { nextRetryTask = null; nextRetryTaskFuture = null; earliestRetryTimeNanos = Long.MIN_VALUE; + isClosed = false; } @Override public boolean trySchedule(Runnable retryTask, long delayMillis) { checkInRetryEventLoop(); - if (isClosed()) { + if (isClosed) { return false; } @@ -103,10 +108,15 @@ public boolean trySchedule(Runnable retryTask, long delayMillis) { assert nextRetryTask == null; assert nextRetryTaskFuture == null; - final long delayNanos = TimeUnit.MILLISECONDS.toNanos(delayMillis); - final long retryTimeNanos = Math.max(System.nanoTime() + delayNanos, earliestRetryTimeNanos); + final long retryTimeNanos = Math.max( + LongMath.saturatedAdd( + System.nanoTime(), + TimeUnit.MILLISECONDS.toNanos(delayMillis) + ), + earliestRetryTimeNanos + ); - if (retryTimeNanos > deadlineTimeNanos) { + if (retryTimeNanos >= deadlineTimeNanos) { return false; } @@ -124,7 +134,7 @@ public boolean trySchedule(Runnable retryTask, long delayMillis) { retryTaskWrapper, nextRetryDelayMillis, TimeUnit.MILLISECONDS); nextRetryTaskFuture.addListener(future -> { - if (isClosed()) { + if (isClosed) { return; } @@ -150,7 +160,7 @@ public boolean trySchedule(Runnable retryTask, long delayMillis) { public void applyMinimumBackoffMillisForNextRetry(long minimumBackoffMillis) { checkInRetryEventLoop(); - if (isClosed()) { + if (isClosed) { return; } @@ -159,10 +169,15 @@ public void applyMinimumBackoffMillisForNextRetry(long minimumBackoffMillis) { checkState(!hasNextRetryTask(), "cannot apply minimum backoff when a retry task is scheduled"); - earliestRetryTimeNanos = Math.max(earliestRetryTimeNanos, - System.nanoTime() + TimeUnit.MILLISECONDS.toNanos( - minimumBackoffMillis) - ); + earliestRetryTimeNanos = + Math.min( + Math.max(earliestRetryTimeNanos, + LongMath.saturatedAdd(System.nanoTime(), + TimeUnit.MILLISECONDS.toNanos( + minimumBackoffMillis)) + ), + deadlineTimeNanos + ); } private boolean hasNextRetryTask() { @@ -175,28 +190,32 @@ private boolean hasNextRetryTask() { public void close() { checkInRetryEventLoop(); - if (isClosed()) { + if (isClosed) { return; } + isClosed = true; clearRetryTaskIfExists(); closedFuture.complete(null); } - @Override - public CompletableFuture whenClosed() { - return UnmodifiableFuture.wrap(closedFuture); - } - private void closeExceptionally(Throwable cause) { - if (isClosed()) { + if (isClosed) { return; } + isClosed = true; clearRetryTaskIfExists(); closedFuture.completeExceptionally(cause); } + @Override + public CompletableFuture whenClosed() { + checkInRetryEventLoop(); + + return UnmodifiableFuture.wrap(closedFuture); + } + private void clearRetryTaskIfExists() { if (nextRetryTaskFuture != null) { nextRetryTaskFuture.cancel(false); @@ -206,10 +225,6 @@ private void clearRetryTaskIfExists() { nextRetryTask = null; } - private boolean isClosed() { - return closedFuture.isDone(); - } - private void checkInRetryEventLoop() { checkState(retryEventLoop.inEventLoop(), "not in the retryEventLoop: %s but in thread %s", retryEventLoop, Thread.currentThread().getName()); diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java new file mode 100644 index 00000000000..f7d49453c56 --- /dev/null +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -0,0 +1,719 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.client.retry; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; +import static org.assertj.core.api.AssertionsForClassTypes.within; +import static org.awaitility.Awaitility.await; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Test; + +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.internal.testing.AnticipatedException; +import com.linecorp.armeria.internal.testing.BlockingUtils; + +import io.netty.channel.DefaultEventLoop; + +class RetrySchedulerTest { + private static final class MockRetryTask implements Runnable { + private final Consumer runnable; + private final int retryTaskNumber; + private final AtomicBoolean executed; + private final AtomicLong executionTimeNanos; + + MockRetryTask(Consumer runnable, int retryTaskNumber) { + this.runnable = runnable; + this.retryTaskNumber = retryTaskNumber; + executed = new AtomicBoolean(false); + executionTimeNanos = new AtomicLong(); + } + + @Override + public void run() { + assertThat(executed.get()).isFalse(); + executed.set(true); + executionTimeNanos.set(System.nanoTime()); + runnable.accept(this); + } + + long executionTimeNanos() { + assertThat(executed.get()).isTrue(); + return executionTimeNanos.get(); + } + + int nextRetryTaskNumber() { + return retryTaskNumber; + } + } + + private static final class ExceptionCatchingEventLoop extends DefaultEventLoop { + private final List exceptionsCaughtOnEventLoop = Collections.synchronizedList( + new ArrayList<>()); + + @Override + protected void run() { + try { + super.run(); + } catch (Throwable t) { + exceptionsCaughtOnEventLoop.add(t); + throw t; + } + } + + List exceptionsCaughtOnEventLoop() { + return exceptionsCaughtOnEventLoop; + } + } + + private final AtomicInteger nextRetryTaskNumber = new AtomicInteger(0); + private final List executedRetryTasks = Collections.synchronizedList(new ArrayList<>()); + private static final long RETRY_TASK_EXECUTION_TIME_TOLERANCE = 50L; + private ExceptionCatchingEventLoop retryEventLoop; + + @BeforeEach + void setUp() { + retryEventLoop = new ExceptionCatchingEventLoop(); + } + + @AfterEach + void tearDown() throws ExecutionException, InterruptedException { + nextRetryTaskNumber.set(0); + executedRetryTasks.clear(); + retryEventLoop.shutdownGracefully(0, 3, TimeUnit.SECONDS).get(); + retryEventLoop = null; + } + + @Test + void schedule_noDelay_executeImmediately() { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + TimeUnit.SECONDS.toNanos(5) + ); + + runOnRetryEventLoop(() -> { + final long now = System.nanoTime(); + assertThat(scheduler.trySchedule(nextRetryTask(), 0)).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + + assertRetryTaskExecutionsAt(now); + + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + }); + + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void schedule_withDelay_executeAfterDelay() { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler(retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + final AtomicLong schedulingTimeNanos = new AtomicLong(); + final long delayMillis = 200L; + + runOnRetryEventLoop(() -> { + schedulingTimeNanos.set(System.nanoTime()); + assertThat(scheduler.trySchedule(nextRetryTask(), delayMillis)).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + await().untilAsserted( + () -> assertRetryTaskExecutionsAt( + schedulingTimeNanos.get() + + TimeUnit.MILLISECONDS.toNanos(delayMillis)) + ); + + runOnRetryEventLoop(() -> { + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + }); + + assertNoExceptionsOnRetryEventLoop(); + } + + // We only support sequential scheduling at the moment. + @Test + void schedule_whileScheduling_throwsIllegalStateException() throws InterruptedException { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + final AtomicLong schedulingTimeNanos = new AtomicLong(); + + runOnRetryEventLoop(() -> { + schedulingTimeNanos.set(System.nanoTime()); + assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + Thread.sleep(200); + + runOnRetryEventLoop(() -> { + assertThatThrownBy(() -> scheduler.trySchedule(nextRetryTask(), 0)) + .isInstanceOf(IllegalStateException.class); + assertThat(scheduler.whenClosed()).isNotDone(); + assertThatThrownBy(() -> scheduler.trySchedule(nextRetryTask(), 100)) + .isInstanceOf(IllegalStateException.class); + assertThat(scheduler.whenClosed()).isNotDone(); + assertThatThrownBy(() -> scheduler.trySchedule(nextRetryTask(), 1_000)) + .isInstanceOf(IllegalStateException.class); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + Thread.sleep(1_000 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + + assertRetryTaskExecutionsAt(schedulingTimeNanos.get() + + TimeUnit.MILLISECONDS.toNanos(1_000)); + + assertNoExceptionsOnRetryEventLoop(); + } + + @RepeatedTest(5) + void schedule_multiple_executeMultipleAfterDelay() throws InterruptedException { + // then retry task each after 100ms + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + final CountDownLatch retryDone = new CountDownLatch(1); + final AtomicLong schedulingTimeNanos = new AtomicLong(); + + final int numberOfAttempts = 10; + final List retryDelaysMillis = Collections.synchronizedList(new ArrayList<>()); + final List tasks = Collections.synchronizedList(new ArrayList<>()); + + final Random random = new Random(); + + for (int i = 0; i < numberOfAttempts; i++) { + if (random.nextBoolean()) { + retryDelaysMillis.add(0L); + } else { + retryDelaysMillis.add((long) random.nextInt(300)); + } + } + + for (int i = 0; i < numberOfAttempts - 1; i++) { + final int attemptIndex = i; + tasks.add(nextRetryTask(() -> { + final MockRetryTask nextRetryTask = tasks.get(attemptIndex + 1); + final long nextDelayMillis = retryDelaysMillis.get(attemptIndex + 1); + final long nextDelayMillisForCall; + + if (random.nextInt(3) < 2) { + // 0, 1 + final long diffNextDelayAndMinimumBackoff = random.nextInt(50); + + if (random.nextBoolean()) { + scheduler.applyMinimumBackoffMillisForNextRetry( + nextDelayMillis - diffNextDelayAndMinimumBackoff); + nextDelayMillisForCall = nextDelayMillis; + } else { + scheduler.applyMinimumBackoffMillisForNextRetry( + nextDelayMillis); + nextDelayMillisForCall = nextDelayMillis - diffNextDelayAndMinimumBackoff; + } + } else { + // 2 + nextDelayMillisForCall = nextDelayMillis; + } + + // In the end we expect to execute that retry task after nextDelayMillis. + assertThat(scheduler.trySchedule(nextRetryTask, nextDelayMillisForCall)).isTrue(); + })); + } + tasks.add(nextRetryTask(retryDone::countDown)); + + runOnRetryEventLoop(() -> { + schedulingTimeNanos.set(System.nanoTime()); + assertThat(scheduler.trySchedule(tasks.get(0), retryDelaysMillis.get(0))).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + await().until(() -> retryDone.getCount() == 0); + + final List expectedRetryTaskExecutionTimes = new ArrayList<>(); + long cumulativeDelayMillis = 0L; + + for (final Long delayMillis : retryDelaysMillis) { + cumulativeDelayMillis += delayMillis; + expectedRetryTaskExecutionTimes.add( + schedulingTimeNanos.get() + + TimeUnit.MILLISECONDS.toNanos(cumulativeDelayMillis) + ); + } + + await().untilAsserted( + () -> { + assertRetryTaskExecutionsAt(expectedRetryTaskExecutionTimes); + }); + + Thread.sleep(RETRY_TASK_EXECUTION_TIME_TOLERANCE + 1_000); + assertRetryTaskExecutionsAt(expectedRetryTaskExecutionTimes); + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void schedule_withDelayAndMinimumBackoff_executeAfterDelay() { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + final AtomicLong schedulingTimeNanos = new AtomicLong(); + final long minimumBackoffMillis = 200L; + final long delayMillis = minimumBackoffMillis + + RETRY_TASK_EXECUTION_TIME_TOLERANCE + + RETRY_TASK_EXECUTION_TIME_TOLERANCE + + 200L; + + runOnRetryEventLoop(() -> { + schedulingTimeNanos.set(System.nanoTime()); + scheduler.applyMinimumBackoffMillisForNextRetry(Long.MIN_VALUE); + scheduler.applyMinimumBackoffMillisForNextRetry(-1); + scheduler.applyMinimumBackoffMillisForNextRetry(0); + scheduler.applyMinimumBackoffMillisForNextRetry(minimumBackoffMillis - 200); + // Should override all the previous calls. + scheduler.applyMinimumBackoffMillisForNextRetry(minimumBackoffMillis); + assertThat(scheduler.trySchedule(nextRetryTask(), delayMillis)).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + await().untilAsserted( + () -> assertRetryTaskExecutionsAt( + schedulingTimeNanos.get() + + TimeUnit.MILLISECONDS.toNanos(delayMillis)) + ); + + runOnRetryEventLoop(() -> { + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + }); + + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void schedule_withDelayAndMinimumBackoff_executeAfterMinimumBackoff() { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + final AtomicLong schedulingTimeNanos = new AtomicLong(); + final long delayMillis = 200L; + final long minimumBackoffMillis = delayMillis + + RETRY_TASK_EXECUTION_TIME_TOLERANCE + + RETRY_TASK_EXECUTION_TIME_TOLERANCE + + 200L; + + runOnRetryEventLoop(() -> { + schedulingTimeNanos.set(System.nanoTime()); + scheduler.applyMinimumBackoffMillisForNextRetry(minimumBackoffMillis); + assertThat(scheduler.trySchedule(nextRetryTask(), delayMillis)).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + await().untilAsserted( + () -> assertRetryTaskExecutionsAt( + schedulingTimeNanos.get() + + TimeUnit.MILLISECONDS.toNanos(minimumBackoffMillis)) + ); + + runOnRetryEventLoop(() -> { + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + }); + + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void schedule_beyondDeadline_returnFalse() throws InterruptedException { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos(1_000) + ); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.trySchedule(nextRetryTask(), 1_001)).isFalse(); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + Thread.sleep(1_001 + 100); + + assertNoRetryTaskExecutions(); + + runOnRetryEventLoop(() -> { + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + }); + + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void schedule_exceptionInRetryTask_closeExceptionally() { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(1) + ); + + final AtomicReference> whenClosedRef = new AtomicReference<>(); + final AtomicLong schedulingTimeNanos = new AtomicLong(); + final long delayMillis = 200L; + + runOnRetryEventLoop(() -> { + schedulingTimeNanos.set(System.nanoTime()); + assertThat(scheduler.trySchedule( + nextRetryTask(() -> { + throw new AnticipatedException("bad task"); + }), + delayMillis)).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + whenClosedRef.set(scheduler.whenClosed()); + }); + + await().untilAsserted(() -> { + try { + assertThat(whenClosedRef.get()).isCompletedExceptionally(); + whenClosedRef.get().get(); + fail(); + } catch (Throwable e) { + assertThat(e.getCause()).isInstanceOf(AnticipatedException.class) + .hasMessage("bad task"); + } + }); + + assertRetryTaskExecutionsAt(schedulingTimeNanos.get() + + TimeUnit.MILLISECONDS.toNanos(delayMillis)); + + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void schedule_closeRetryEventLoop_closeExceptionally() throws InterruptedException { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + // Close the event loop before the task is executed. + retryEventLoop.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.whenClosed()).isCompletedExceptionally(); + try { + BlockingUtils.blockingRun(() -> { + scheduler.whenClosed().get(); + }); + fail(); + } catch (Throwable e) { + assertThat(e.getCause()).isInstanceOf(IllegalStateException.class) + .hasMessageContaining(ClientFactory.class.getName()) + .hasMessageContaining("has been closed."); + } + }); + + Thread.sleep(1_000 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + + assertNoRetryTaskExecutions(); + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void applyMinimumBackoff_whileScheduling_throwIllegalStateException() throws InterruptedException { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + final AtomicLong schedulingTimeNanos = new AtomicLong(); + + runOnRetryEventLoop(() -> { + schedulingTimeNanos.set(System.nanoTime()); + assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + Thread.sleep(200); + + runOnRetryEventLoop(() -> { + assertThatThrownBy(() -> scheduler.applyMinimumBackoffMillisForNextRetry(42)) + .isInstanceOf(IllegalStateException.class); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + Thread.sleep(800 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + + assertRetryTaskExecutionsAt(schedulingTimeNanos.get() + + TimeUnit.MILLISECONDS.toNanos(1_000)); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.whenClosed()).isNotDone(); + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + }); + + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void applyMinimumBackoff_thatExceedsDeadline_rejectEverySchedule() throws InterruptedException { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(1) + ); + + runOnRetryEventLoop(() -> { + scheduler.applyMinimumBackoffMillisForNextRetry(1_001); + assertThat(scheduler.whenClosed()).isNotDone(); + assertThat(scheduler.trySchedule(nextRetryTask(), 0)).isFalse(); + assertThat(scheduler.trySchedule(nextRetryTask(), 100)).isFalse(); + }); + + Thread.sleep(100 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.whenClosed()).isNotDone(); + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + }); + + assertNoRetryTaskExecutions(); + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void close_thenSchedule_returnFalse() throws InterruptedException { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + runOnRetryEventLoop(() -> { + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + assertThat(scheduler.trySchedule(nextRetryTask(), 200)).isFalse(); + }); + + Thread.sleep(200 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + + assertNoRetryTaskExecutions(); + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void close_whileSchedule_cancelRetryTask() throws InterruptedException { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.trySchedule(nextRetryTask(), 1000L)).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + Thread.sleep(250); + assertNoRetryTaskExecutions(); + + runOnRetryEventLoop(() -> { + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + }); + + Thread.sleep(750 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + + assertNoRetryTaskExecutions(); + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void close_thenDoSomething_doesNotScheduleAndThrow() throws InterruptedException { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + runOnRetryEventLoop(() -> { + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + }); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.trySchedule(nextRetryTask(), 0)).isFalse(); + assertThat(scheduler.trySchedule(nextRetryTask(), 200)).isFalse(); + }); + + Thread.sleep(200 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + + runOnRetryEventLoop(() -> { + scheduler.applyMinimumBackoffMillisForNextRetry(Long.MIN_VALUE); + scheduler.close(); + scheduler.applyMinimumBackoffMillisForNextRetry(-1); + scheduler.applyMinimumBackoffMillisForNextRetry(0); + scheduler.applyMinimumBackoffMillisForNextRetry(100); + scheduler.close(); + }); + + assertNoRetryTaskExecutions(); + assertNoExceptionsOnRetryEventLoop(); + } + + @Test + void schedule_applyMinimumBackoff_close_outsideRetryEventLoop_throwsIllegalStateException() { + final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + + TimeUnit.SECONDS.toNanos(5) + ); + + assertThatThrownBy(() -> scheduler.trySchedule(nextRetryTask(), 100)).isInstanceOf( + IllegalStateException.class); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + assertThatThrownBy(() -> scheduler.applyMinimumBackoffMillisForNextRetry(200)) + .isInstanceOf(IllegalStateException.class); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + assertThatThrownBy(scheduler::close).isInstanceOf(IllegalStateException.class); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.whenClosed()).isNotDone(); + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + }); + + assertNoRetryTaskExecutions(); + assertNoExceptionsOnRetryEventLoop(); + } + + private MockRetryTask nextRetryTask() { + return nextRetryTask(() -> { + // Default does nothing. + }); + } + + private MockRetryTask nextRetryTask(Runnable innerTask) { + final int taskNumber = nextRetryTaskNumber.getAndIncrement(); + return new MockRetryTask(task -> { + assertThat(retryEventLoop.inEventLoop()).isTrue(); + executedRetryTasks.add(task); + innerTask.run(); + }, taskNumber); + } + + private void runOnRetryEventLoop(Runnable runnable) { + final CountDownLatch latch = new CountDownLatch(1); + retryEventLoop.execute(() -> { + try { + runnable.run(); + } finally { + latch.countDown(); + } + }); + try { + assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue(); + } catch (InterruptedException e) { + fail(e); + } + + assertNoExceptionsOnRetryEventLoop(); + } + + private void assertNoRetryTaskExecutions() { + assertRetryTaskExecutionsAt(); + } + + private void assertRetryTaskExecutionsAt(long... expectedExecutionTimesNanos) { + assertRetryTaskExecutionsAt( + expectedExecutionTimesNanos == null ? + Collections.emptyList() + : LongStream.of(expectedExecutionTimesNanos).boxed() + .collect(Collectors.toList()) + ); + } + + private void assertRetryTaskExecutionsAt(List expectedExecutionTimesNanos) { + assertThat(executedRetryTasks).hasSize(expectedExecutionTimesNanos.size()); + for (int expectedRetryTaskNumber = 0; expectedRetryTaskNumber < expectedExecutionTimesNanos.size(); + expectedRetryTaskNumber++) { + final MockRetryTask task = executedRetryTasks.get(expectedRetryTaskNumber); + assertThat(task.nextRetryTaskNumber()).isEqualTo(expectedRetryTaskNumber); + final long schedulingDifferenceMillis = TimeUnit.NANOSECONDS.toMillis( + task.executionTimeNanos() - + expectedExecutionTimesNanos.get(expectedRetryTaskNumber)); + assertThat(schedulingDifferenceMillis) + .as("Retry task %d executed at the expected time", expectedRetryTaskNumber) + .isCloseTo(0, within(RETRY_TASK_EXECUTION_TIME_TOLERANCE)); + } + } + + private void assertNoExceptionsOnRetryEventLoop() { + assertThat(retryEventLoop.exceptionsCaughtOnEventLoop()) + .as("No exceptions thrown on retry event loop") + .isEmpty(); + } +} From df62fbe5003cd2edd9d24063d394f25208956834 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 12 Sep 2025 18:02:39 +0200 Subject: [PATCH 38/42] fix: fix overflow when calculating deadline --- .../linecorp/armeria/internal/client/ClientUtil.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java index 44427266121..e88e955329e 100644 --- a/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java +++ b/core/src/main/java/com/linecorp/armeria/internal/client/ClientUtil.java @@ -28,6 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.math.LongMath; + import com.linecorp.armeria.client.Client; import com.linecorp.armeria.client.ClientRequestContext; import com.linecorp.armeria.client.Endpoint; @@ -341,10 +343,11 @@ public static long retryAfterMillis(RequestLogAccess log) { */ public static long deadlineTimeNanos(ClientRequestContext ctx) { final long responseTimeoutMillis = ctx.responseTimeoutMillis(); - if (responseTimeoutMillis <= 0 || responseTimeoutMillis == Long.MAX_VALUE) { + if (responseTimeoutMillis <= 0) { return Long.MAX_VALUE; } else { - return System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis); + return LongMath.saturatedAdd(System.nanoTime(), + TimeUnit.MILLISECONDS.toNanos(responseTimeoutMillis)); } } @@ -374,7 +377,8 @@ public static boolean checkAndSetResponseTimeout(ClientRequestContext ctx, long if (deadlineTimeNanos < Long.MAX_VALUE) { hasTimeout = true; remainingTimeUntilDeadlineMillis = TimeUnit.NANOSECONDS.toMillis( - deadlineTimeNanos - System.nanoTime()); + LongMath.saturatedSubtract(deadlineTimeNanos, System.nanoTime()) + ); } if (responseTimeoutMillis != 0) { From 07853558dacd29352b7a54009c7b1ac6660c92ae Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Fri, 12 Sep 2025 19:20:42 +0200 Subject: [PATCH 39/42] fix: fix RetrySchedulerTest - prevent scheduler error accumulation --- .../client/retry/RetrySchedulerTest.java | 144 ++++++++---------- 1 file changed, 64 insertions(+), 80 deletions(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java index f7d49453c56..62a6303c3be 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -44,7 +44,6 @@ import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.internal.testing.AnticipatedException; -import com.linecorp.armeria.internal.testing.BlockingUtils; import io.netty.channel.DefaultEventLoop; @@ -53,12 +52,14 @@ private static final class MockRetryTask implements Runnable { private final Consumer runnable; private final int retryTaskNumber; private final AtomicBoolean executed; + private final AtomicLong scheduledTimeNanos; private final AtomicLong executionTimeNanos; MockRetryTask(Consumer runnable, int retryTaskNumber) { this.runnable = runnable; this.retryTaskNumber = retryTaskNumber; executed = new AtomicBoolean(false); + scheduledTimeNanos = new AtomicLong(System.nanoTime()); executionTimeNanos = new AtomicLong(); } @@ -70,6 +71,14 @@ public void run() { runnable.accept(this); } + void setScheduledTimeNanos() { + scheduledTimeNanos.set(System.nanoTime()); + } + + long scheduledTimeNanos() { + return scheduledTimeNanos.get(); + } + long executionTimeNanos() { assertThat(executed.get()).isTrue(); return executionTimeNanos.get(); @@ -125,11 +134,10 @@ void schedule_noDelay_executeImmediately() { ); runOnRetryEventLoop(() -> { - final long now = System.nanoTime(); assertThat(scheduler.trySchedule(nextRetryTask(), 0)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); - assertRetryTaskExecutionsAt(now); + assertRetryTaskExecutionsAt(0); scheduler.close(); assertThat(scheduler.whenClosed()).isCompleted(); @@ -145,19 +153,13 @@ void schedule_withDelay_executeAfterDelay() { TimeUnit.SECONDS.toNanos(5) ); - final AtomicLong schedulingTimeNanos = new AtomicLong(); - final long delayMillis = 200L; - runOnRetryEventLoop(() -> { - schedulingTimeNanos.set(System.nanoTime()); - assertThat(scheduler.trySchedule(nextRetryTask(), delayMillis)).isTrue(); + assertThat(scheduler.trySchedule(nextRetryTask(), 200L)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); }); await().untilAsserted( - () -> assertRetryTaskExecutionsAt( - schedulingTimeNanos.get() + - TimeUnit.MILLISECONDS.toNanos(delayMillis)) + () -> assertRetryTaskExecutionsAt(200L) ); runOnRetryEventLoop(() -> { @@ -201,8 +203,7 @@ void schedule_whileScheduling_throwsIllegalStateException() throws InterruptedEx Thread.sleep(1_000 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); - assertRetryTaskExecutionsAt(schedulingTimeNanos.get() + - TimeUnit.MILLISECONDS.toNanos(1_000)); + assertRetryTaskExecutionsAt(1_000); assertNoExceptionsOnRetryEventLoop(); } @@ -220,16 +221,16 @@ void schedule_multiple_executeMultipleAfterDelay() throws InterruptedException { final AtomicLong schedulingTimeNanos = new AtomicLong(); final int numberOfAttempts = 10; - final List retryDelaysMillis = Collections.synchronizedList(new ArrayList<>()); + final List expectedDelaysMillis = Collections.synchronizedList(new ArrayList<>()); final List tasks = Collections.synchronizedList(new ArrayList<>()); final Random random = new Random(); for (int i = 0; i < numberOfAttempts; i++) { if (random.nextBoolean()) { - retryDelaysMillis.add(0L); + expectedDelaysMillis.add(0L); } else { - retryDelaysMillis.add((long) random.nextInt(300)); + expectedDelaysMillis.add((long) random.nextInt(300)); } } @@ -237,7 +238,7 @@ void schedule_multiple_executeMultipleAfterDelay() throws InterruptedException { final int attemptIndex = i; tasks.add(nextRetryTask(() -> { final MockRetryTask nextRetryTask = tasks.get(attemptIndex + 1); - final long nextDelayMillis = retryDelaysMillis.get(attemptIndex + 1); + final long nextDelayMillis = expectedDelaysMillis.get(attemptIndex + 1); final long nextDelayMillisForCall; if (random.nextInt(3) < 2) { @@ -258,6 +259,7 @@ void schedule_multiple_executeMultipleAfterDelay() throws InterruptedException { nextDelayMillisForCall = nextDelayMillis; } + nextRetryTask.setScheduledTimeNanos(); // In the end we expect to execute that retry task after nextDelayMillis. assertThat(scheduler.trySchedule(nextRetryTask, nextDelayMillisForCall)).isTrue(); })); @@ -266,30 +268,20 @@ void schedule_multiple_executeMultipleAfterDelay() throws InterruptedException { runOnRetryEventLoop(() -> { schedulingTimeNanos.set(System.nanoTime()); - assertThat(scheduler.trySchedule(tasks.get(0), retryDelaysMillis.get(0))).isTrue(); + tasks.get(0).setScheduledTimeNanos(); + assertThat(scheduler.trySchedule(tasks.get(0), expectedDelaysMillis.get(0))).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); }); await().until(() -> retryDone.getCount() == 0); - final List expectedRetryTaskExecutionTimes = new ArrayList<>(); - long cumulativeDelayMillis = 0L; - - for (final Long delayMillis : retryDelaysMillis) { - cumulativeDelayMillis += delayMillis; - expectedRetryTaskExecutionTimes.add( - schedulingTimeNanos.get() + - TimeUnit.MILLISECONDS.toNanos(cumulativeDelayMillis) - ); - } - await().untilAsserted( () -> { - assertRetryTaskExecutionsAt(expectedRetryTaskExecutionTimes); + assertRetryTaskExecutionsAt(expectedDelaysMillis); }); Thread.sleep(RETRY_TASK_EXECUTION_TIME_TOLERANCE + 1_000); - assertRetryTaskExecutionsAt(expectedRetryTaskExecutionTimes); + assertRetryTaskExecutionsAt(expectedDelaysMillis); assertNoExceptionsOnRetryEventLoop(); } @@ -301,7 +293,6 @@ void schedule_withDelayAndMinimumBackoff_executeAfterDelay() { TimeUnit.SECONDS.toNanos(5) ); - final AtomicLong schedulingTimeNanos = new AtomicLong(); final long minimumBackoffMillis = 200L; final long delayMillis = minimumBackoffMillis + RETRY_TASK_EXECUTION_TIME_TOLERANCE + @@ -309,7 +300,6 @@ void schedule_withDelayAndMinimumBackoff_executeAfterDelay() { 200L; runOnRetryEventLoop(() -> { - schedulingTimeNanos.set(System.nanoTime()); scheduler.applyMinimumBackoffMillisForNextRetry(Long.MIN_VALUE); scheduler.applyMinimumBackoffMillisForNextRetry(-1); scheduler.applyMinimumBackoffMillisForNextRetry(0); @@ -321,9 +311,7 @@ void schedule_withDelayAndMinimumBackoff_executeAfterDelay() { }); await().untilAsserted( - () -> assertRetryTaskExecutionsAt( - schedulingTimeNanos.get() + - TimeUnit.MILLISECONDS.toNanos(delayMillis)) + () -> assertRetryTaskExecutionsAt(delayMillis) ); runOnRetryEventLoop(() -> { @@ -342,7 +330,6 @@ void schedule_withDelayAndMinimumBackoff_executeAfterMinimumBackoff() { TimeUnit.SECONDS.toNanos(5) ); - final AtomicLong schedulingTimeNanos = new AtomicLong(); final long delayMillis = 200L; final long minimumBackoffMillis = delayMillis + RETRY_TASK_EXECUTION_TIME_TOLERANCE + @@ -350,16 +337,13 @@ void schedule_withDelayAndMinimumBackoff_executeAfterMinimumBackoff() { 200L; runOnRetryEventLoop(() -> { - schedulingTimeNanos.set(System.nanoTime()); scheduler.applyMinimumBackoffMillisForNextRetry(minimumBackoffMillis); assertThat(scheduler.trySchedule(nextRetryTask(), delayMillis)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); }); await().untilAsserted( - () -> assertRetryTaskExecutionsAt( - schedulingTimeNanos.get() + - TimeUnit.MILLISECONDS.toNanos(minimumBackoffMillis)) + () -> assertRetryTaskExecutionsAt(minimumBackoffMillis) ); runOnRetryEventLoop(() -> { @@ -404,16 +388,13 @@ void schedule_exceptionInRetryTask_closeExceptionally() { ); final AtomicReference> whenClosedRef = new AtomicReference<>(); - final AtomicLong schedulingTimeNanos = new AtomicLong(); - final long delayMillis = 200L; runOnRetryEventLoop(() -> { - schedulingTimeNanos.set(System.nanoTime()); assertThat(scheduler.trySchedule( nextRetryTask(() -> { throw new AnticipatedException("bad task"); }), - delayMillis)).isTrue(); + 200L)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); whenClosedRef.set(scheduler.whenClosed()); }); @@ -429,41 +410,39 @@ void schedule_exceptionInRetryTask_closeExceptionally() { } }); - assertRetryTaskExecutionsAt(schedulingTimeNanos.get() + - TimeUnit.MILLISECONDS.toNanos(delayMillis)); + assertRetryTaskExecutionsAt(200L); assertNoExceptionsOnRetryEventLoop(); } @Test - void schedule_closeRetryEventLoop_closeExceptionally() throws InterruptedException { + void schedule_closeRetryEventLoop_closeExceptionally() throws Exception { final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( retryEventLoop, System.nanoTime() + TimeUnit.SECONDS.toNanos(5) ); + final AtomicReference> whenClosedRef = new AtomicReference<>(); + runOnRetryEventLoop(() -> { assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); + whenClosedRef.set(scheduler.whenClosed()); }); // Close the event loop before the task is executed. - retryEventLoop.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS); + retryEventLoop.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).get(); - runOnRetryEventLoop(() -> { - assertThat(scheduler.whenClosed()).isCompletedExceptionally(); - try { - BlockingUtils.blockingRun(() -> { - scheduler.whenClosed().get(); - }); - fail(); - } catch (Throwable e) { - assertThat(e.getCause()).isInstanceOf(IllegalStateException.class) - .hasMessageContaining(ClientFactory.class.getName()) - .hasMessageContaining("has been closed."); - } - }); + assertThat(whenClosedRef.get()).isCompletedExceptionally(); + try { + whenClosedRef.get().get(); + fail(); + } catch (Throwable e) { + assertThat(e.getCause()).isInstanceOf(IllegalStateException.class) + .hasMessageContaining(ClientFactory.class.getSimpleName()) + .hasMessageContaining("has been closed"); + } Thread.sleep(1_000 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); @@ -479,10 +458,7 @@ void applyMinimumBackoff_whileScheduling_throwIllegalStateException() throws Int TimeUnit.SECONDS.toNanos(5) ); - final AtomicLong schedulingTimeNanos = new AtomicLong(); - runOnRetryEventLoop(() -> { - schedulingTimeNanos.set(System.nanoTime()); assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); }); @@ -497,8 +473,7 @@ void applyMinimumBackoff_whileScheduling_throwIllegalStateException() throws Int Thread.sleep(800 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); - assertRetryTaskExecutionsAt(schedulingTimeNanos.get() + - TimeUnit.MILLISECONDS.toNanos(1_000)); + assertRetryTaskExecutionsAt(1_000L); runOnRetryEventLoop(() -> { assertThat(scheduler.whenClosed()).isNotDone(); @@ -687,27 +662,36 @@ private void assertNoRetryTaskExecutions() { assertRetryTaskExecutionsAt(); } - private void assertRetryTaskExecutionsAt(long... expectedExecutionTimesNanos) { + private void assertRetryTaskExecutionsAt(long... expectedDelaysMillis) { assertRetryTaskExecutionsAt( - expectedExecutionTimesNanos == null ? + expectedDelaysMillis == null ? Collections.emptyList() - : LongStream.of(expectedExecutionTimesNanos).boxed() - .collect(Collectors.toList()) + : LongStream.of(expectedDelaysMillis).boxed() + .collect(Collectors.toList()) ); } - private void assertRetryTaskExecutionsAt(List expectedExecutionTimesNanos) { - assertThat(executedRetryTasks).hasSize(expectedExecutionTimesNanos.size()); - for (int expectedRetryTaskNumber = 0; expectedRetryTaskNumber < expectedExecutionTimesNanos.size(); + private void assertRetryTaskExecutionsAt(List expectedDelaysMillis) { + assertThat(executedRetryTasks).hasSize(expectedDelaysMillis.size()); + + if (expectedDelaysMillis.isEmpty()) { + return; + } + + for (int expectedRetryTaskNumber = 0; expectedRetryTaskNumber < expectedDelaysMillis.size(); expectedRetryTaskNumber++) { final MockRetryTask task = executedRetryTasks.get(expectedRetryTaskNumber); + assertThat(task.nextRetryTaskNumber()).isEqualTo(expectedRetryTaskNumber); - final long schedulingDifferenceMillis = TimeUnit.NANOSECONDS.toMillis( - task.executionTimeNanos() - - expectedExecutionTimesNanos.get(expectedRetryTaskNumber)); - assertThat(schedulingDifferenceMillis) - .as("Retry task %d executed at the expected time", expectedRetryTaskNumber) - .isCloseTo(0, within(RETRY_TASK_EXECUTION_TIME_TOLERANCE)); + + final long actualDelayMillis = TimeUnit.NANOSECONDS.toMillis( + task.executionTimeNanos() - task.scheduledTimeNanos() + ); + final long expectedDelayMillis = expectedDelaysMillis.get(expectedRetryTaskNumber); + + assertThat(actualDelayMillis) + .as("Retry task %d being delayed appropriately", expectedRetryTaskNumber) + .isCloseTo(expectedDelayMillis, within(RETRY_TASK_EXECUTION_TIME_TOLERANCE)); } } From a7fc28b90ce7d5ad9e729569a18d99c1f6e3c20b Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sat, 13 Sep 2025 15:30:39 +0200 Subject: [PATCH 40/42] docs: add comment in DefaultRetryScheduler --- .../linecorp/armeria/client/retry/DefaultRetryScheduler.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java index b9c68ce1895..616837c16e0 100644 --- a/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java +++ b/core/src/main/java/com/linecorp/armeria/client/retry/DefaultRetryScheduler.java @@ -56,6 +56,8 @@ public void run() { assert retryEventLoop.inEventLoop(); if (isClosed) { + // Very unexpected as we would cancel this future on the same event loop here in the scheduler + // upon close() or closeExceptionally(). logger.debug("Tried to run a retry task after the scheduler was closed. Skipping this task."); return; } From 006e74e25560f79e01c179f6f56f2a0c7428f545 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sat, 13 Sep 2025 15:31:18 +0200 Subject: [PATCH 41/42] test: stabilize and extend RetrySchedulerTest --- .../client/retry/RetrySchedulerTest.java | 849 +++++++++++------- 1 file changed, 511 insertions(+), 338 deletions(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java index 62a6303c3be..835fc66105e 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -18,16 +18,16 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.fail; -import static org.assertj.core.api.AssertionsForClassTypes.within; import static org.awaitility.Awaitility.await; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -35,160 +35,110 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.LongStream; +import java.util.stream.Stream; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.google.common.collect.ImmutableList; import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.ResponseTimeoutException; +import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.internal.testing.AnticipatedException; +import com.linecorp.armeria.internal.testing.BlockingUtils; import io.netty.channel.DefaultEventLoop; +import io.netty.util.concurrent.ScheduledFuture; class RetrySchedulerTest { - private static final class MockRetryTask implements Runnable { - private final Consumer runnable; - private final int retryTaskNumber; - private final AtomicBoolean executed; - private final AtomicLong scheduledTimeNanos; - private final AtomicLong executionTimeNanos; - - MockRetryTask(Consumer runnable, int retryTaskNumber) { - this.runnable = runnable; - this.retryTaskNumber = retryTaskNumber; - executed = new AtomicBoolean(false); - scheduledTimeNanos = new AtomicLong(System.nanoTime()); - executionTimeNanos = new AtomicLong(); - } - - @Override - public void run() { - assertThat(executed.get()).isFalse(); - executed.set(true); - executionTimeNanos.set(System.nanoTime()); - runnable.accept(this); - } - - void setScheduledTimeNanos() { - scheduledTimeNanos.set(System.nanoTime()); - } - - long scheduledTimeNanos() { - return scheduledTimeNanos.get(); - } - - long executionTimeNanos() { - assertThat(executed.get()).isTrue(); - return executionTimeNanos.get(); - } - - int nextRetryTaskNumber() { - return retryTaskNumber; - } - } - - private static final class ExceptionCatchingEventLoop extends DefaultEventLoop { - private final List exceptionsCaughtOnEventLoop = Collections.synchronizedList( - new ArrayList<>()); - - @Override - protected void run() { - try { - super.run(); - } catch (Throwable t) { - exceptionsCaughtOnEventLoop.add(t); - throw t; - } - } - - List exceptionsCaughtOnEventLoop() { - return exceptionsCaughtOnEventLoop; - } - } + private static final long RETRY_SCHEDULER_SCHEDULE_ADJUSTMENT_TOLERANCE_MILLIS = 5L; + // Number of millis we expect the retry event loop will take to actually execute a delayed task + // *after* its delay. + private static final long MAX_EXECUTION_DELAY_MILLIS = 100L; - private final AtomicInteger nextRetryTaskNumber = new AtomicInteger(0); - private final List executedRetryTasks = Collections.synchronizedList(new ArrayList<>()); - private static final long RETRY_TASK_EXECUTION_TIME_TOLERANCE = 50L; - private ExceptionCatchingEventLoop retryEventLoop; + private final List executedRetryTasks = Collections.synchronizedList(new ArrayList<>()); + private ManagedRetryEventLoop retryEventLoop; + private DefaultRetryScheduler scheduler; @BeforeEach void setUp() { - retryEventLoop = new ExceptionCatchingEventLoop(); + retryEventLoop = new ManagedRetryEventLoop(); + scheduler = new DefaultRetryScheduler( + retryEventLoop, + System.nanoTime() + TimeUnit.SECONDS.toNanos(10) + ); } @AfterEach void tearDown() throws ExecutionException, InterruptedException { - nextRetryTaskNumber.set(0); - executedRetryTasks.clear(); + // If the test terminates the event loop it needs to a local copy. + runOnRetryEventLoop(() -> { + if (!scheduler.whenClosed().isDone()) { + scheduler.close(); + assertThat(scheduler.whenClosed()).isCompleted(); + } + }); + retryEventLoop.shutdownGracefully(0, 3, TimeUnit.SECONDS).get(); - retryEventLoop = null; + assertNoExceptionsOnRetryEventLoop(retryEventLoop); + + ManagedRetryTask.nextRetryTaskNumber.set(0); + executedRetryTasks.clear(); } @Test void schedule_noDelay_executeImmediately() { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + TimeUnit.SECONDS.toNanos(5) - ); - runOnRetryEventLoop(() -> { assertThat(scheduler.trySchedule(nextRetryTask(), 0)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); - assertRetryTaskExecutionsAt(0); - - scheduler.close(); - assertThat(scheduler.whenClosed()).isCompleted(); + assertNumRetryTaskExecutions(1); + assertNoScheduleCalls(); }); - - assertNoExceptionsOnRetryEventLoop(); } @Test void schedule_withDelay_executeAfterDelay() { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler(retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) - ); - runOnRetryEventLoop(() -> { assertThat(scheduler.trySchedule(nextRetryTask(), 200L)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); }); await().untilAsserted( - () -> assertRetryTaskExecutionsAt(200L) + () -> { + assertNumRetryTaskExecutions(1); + assertEventLoopScheduleCalls(200L); + } ); - - runOnRetryEventLoop(() -> { - scheduler.close(); - assertThat(scheduler.whenClosed()).isCompleted(); - }); - - assertNoExceptionsOnRetryEventLoop(); } // We only support sequential scheduling at the moment. @Test void schedule_whileScheduling_throwsIllegalStateException() throws InterruptedException { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) - ); - final AtomicLong schedulingTimeNanos = new AtomicLong(); runOnRetryEventLoop(() -> { schedulingTimeNanos.set(System.nanoTime()); - assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); + assertThat(scheduler.trySchedule(nextRetryTask(), 1_000L)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); }); + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(1_000L); + Thread.sleep(200); + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(1_000L); + runOnRetryEventLoop(() -> { assertThatThrownBy(() -> scheduler.trySchedule(nextRetryTask(), 0)) .isInstanceOf(IllegalStateException.class); @@ -201,103 +151,118 @@ void schedule_whileScheduling_throwsIllegalStateException() throws InterruptedEx assertThat(scheduler.whenClosed()).isNotDone(); }); - Thread.sleep(1_000 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(1_000L); - assertRetryTaskExecutionsAt(1_000); - - assertNoExceptionsOnRetryEventLoop(); + await().untilAsserted(() -> { + assertNumRetryTaskExecutions(1); + assertEventLoopScheduleCalls(1_000L); + }); } - @RepeatedTest(5) - void schedule_multiple_executeMultipleAfterDelay() throws InterruptedException { - // then retry task each after 100ms - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) + private static Stream schedule_multiple_executeMultipleAfterDelay_args() { + return Stream.of( + Arguments.of( + new RetryPlan( + "All increasing delays", + Arrays.asList( + new ScheduleCall(100L), + new ScheduleCall(200L), + new ScheduleCall(300L) + ), + Arrays.asList(100L, 200L, 300L) + ) + ), + Arguments.of( + new RetryPlan( + "All decreasing delays", + Arrays.asList( + new ScheduleCall(300L), + new ScheduleCall(200L), + new ScheduleCall(100L) + ), + Arrays.asList(300L, 200L, 100L) + ) + ), + Arguments.of( + new RetryPlan( + "No order in delays, some direct", + Arrays.asList( + new ScheduleCall(200L), + new ScheduleCall(-10L), + new ScheduleCall(0L), + new ScheduleCall(100L) + ), + Arrays.asList(200L, /* direct, */ /* direct, */ 100L + ) + ) + ), + Arguments.of( + new RetryPlan( + "Many direct", + IntStream.range(0, 1024) + .mapToObj(i -> new ScheduleCall(0L)) + .collect(Collectors.toList()), + Collections.emptyList() // All direct + ) + ), + Arguments.of( + new RetryPlan( + "Mixed delays and minimum backoffs", + Arrays.asList( + new ScheduleCall(0L), + new ScheduleCall(100L, Collections.singletonList(50L)), + new ScheduleCall(-42L), + new ScheduleCall(50L, Collections.singletonList(100L)), + new ScheduleCall(0L) + ), + Arrays.asList(/* direct, */ 100L, /* direct, */ 100L/*, direct */) + ) + ), + Arguments.of( + new RetryPlan( + "Multiple minimum backoffs", + Arrays.asList( + new ScheduleCall(0L, + Arrays.asList(Long.MIN_VALUE, -50L, 100L) + ), + new ScheduleCall(100L, + Arrays.asList(50L, Long.MIN_VALUE) + ), + new ScheduleCall(-10L, Collections.singletonList(200L)) + ), + Arrays.asList( + 100L, + 100L, + 200L + ) + ) + ) ); + } - final CountDownLatch retryDone = new CountDownLatch(1); - final AtomicLong schedulingTimeNanos = new AtomicLong(); - - final int numberOfAttempts = 10; - final List expectedDelaysMillis = Collections.synchronizedList(new ArrayList<>()); - final List tasks = Collections.synchronizedList(new ArrayList<>()); - - final Random random = new Random(); - - for (int i = 0; i < numberOfAttempts; i++) { - if (random.nextBoolean()) { - expectedDelaysMillis.add(0L); - } else { - expectedDelaysMillis.add((long) random.nextInt(300)); - } - } - - for (int i = 0; i < numberOfAttempts - 1; i++) { - final int attemptIndex = i; - tasks.add(nextRetryTask(() -> { - final MockRetryTask nextRetryTask = tasks.get(attemptIndex + 1); - final long nextDelayMillis = expectedDelaysMillis.get(attemptIndex + 1); - final long nextDelayMillisForCall; - - if (random.nextInt(3) < 2) { - // 0, 1 - final long diffNextDelayAndMinimumBackoff = random.nextInt(50); - - if (random.nextBoolean()) { - scheduler.applyMinimumBackoffMillisForNextRetry( - nextDelayMillis - diffNextDelayAndMinimumBackoff); - nextDelayMillisForCall = nextDelayMillis; - } else { - scheduler.applyMinimumBackoffMillisForNextRetry( - nextDelayMillis); - nextDelayMillisForCall = nextDelayMillis - diffNextDelayAndMinimumBackoff; - } - } else { - // 2 - nextDelayMillisForCall = nextDelayMillis; - } - - nextRetryTask.setScheduledTimeNanos(); - // In the end we expect to execute that retry task after nextDelayMillis. - assertThat(scheduler.trySchedule(nextRetryTask, nextDelayMillisForCall)).isTrue(); - })); - } - tasks.add(nextRetryTask(retryDone::countDown)); - - runOnRetryEventLoop(() -> { - schedulingTimeNanos.set(System.nanoTime()); - tasks.get(0).setScheduledTimeNanos(); - assertThat(scheduler.trySchedule(tasks.get(0), expectedDelaysMillis.get(0))).isTrue(); - assertThat(scheduler.whenClosed()).isNotDone(); - }); + @ParameterizedTest + @MethodSource("schedule_multiple_executeMultipleAfterDelay_args") + void schedule_multiple_executeMultipleAfterDelay(RetryPlan retryPlan) + throws InterruptedException { + final int numberOfAttempts = retryPlan.retryTaskDelaysMillis.size(); + assert numberOfAttempts >= 1; - await().until(() -> retryDone.getCount() == 0); + final CountDownLatch retryDone = execRetryPlan(retryPlan); await().untilAsserted( () -> { - assertRetryTaskExecutionsAt(expectedDelaysMillis); + assertNumRetryTaskExecutions(numberOfAttempts); + assertEventLoopScheduleCalls(retryPlan.expectedScheduleDelayMillis); }); - Thread.sleep(RETRY_TASK_EXECUTION_TIME_TOLERANCE + 1_000); - assertRetryTaskExecutionsAt(expectedDelaysMillis); - assertNoExceptionsOnRetryEventLoop(); + assertThat(retryDone.getCount()).isZero(); } @Test - void schedule_withDelayAndMinimumBackoff_executeAfterDelay() { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) - ); - + void schedule_withDelayAndMinimumBackoff_executeAfterDelay() throws InterruptedException { final long minimumBackoffMillis = 200L; - final long delayMillis = minimumBackoffMillis + - RETRY_TASK_EXECUTION_TIME_TOLERANCE + - RETRY_TASK_EXECUTION_TIME_TOLERANCE + - 200L; + final long delayMillis = minimumBackoffMillis + 200L; runOnRetryEventLoop(() -> { scheduler.applyMinimumBackoffMillisForNextRetry(Long.MIN_VALUE); @@ -310,31 +275,21 @@ void schedule_withDelayAndMinimumBackoff_executeAfterDelay() { assertThat(scheduler.whenClosed()).isNotDone(); }); + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(delayMillis); + await().untilAsserted( - () -> assertRetryTaskExecutionsAt(delayMillis) + () -> { + assertNumRetryTaskExecutions(1); + assertEventLoopScheduleCalls(delayMillis); + } ); - - runOnRetryEventLoop(() -> { - scheduler.close(); - assertThat(scheduler.whenClosed()).isCompleted(); - }); - - assertNoExceptionsOnRetryEventLoop(); } @Test - void schedule_withDelayAndMinimumBackoff_executeAfterMinimumBackoff() { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) - ); - + void schedule_withDelayAndMinimumBackoff_executeAfterMinimumBackoff() throws InterruptedException { final long delayMillis = 200L; - final long minimumBackoffMillis = delayMillis + - RETRY_TASK_EXECUTION_TIME_TOLERANCE + - RETRY_TASK_EXECUTION_TIME_TOLERANCE + - 200L; + final long minimumBackoffMillis = delayMillis + 200L; runOnRetryEventLoop(() -> { scheduler.applyMinimumBackoffMillisForNextRetry(minimumBackoffMillis); @@ -343,50 +298,31 @@ void schedule_withDelayAndMinimumBackoff_executeAfterMinimumBackoff() { }); await().untilAsserted( - () -> assertRetryTaskExecutionsAt(minimumBackoffMillis) + () -> { + assertNumRetryTaskExecutions(1); + assertEventLoopScheduleCalls(minimumBackoffMillis); + } ); - - runOnRetryEventLoop(() -> { - scheduler.close(); - assertThat(scheduler.whenClosed()).isCompleted(); - }); - - assertNoExceptionsOnRetryEventLoop(); } @Test void schedule_beyondDeadline_returnFalse() throws InterruptedException { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.MILLISECONDS.toNanos(1_000) - ); - runOnRetryEventLoop(() -> { - assertThat(scheduler.trySchedule(nextRetryTask(), 1_001)).isFalse(); + assertThat(scheduler.trySchedule( + nextRetryTask(), + 10_000 + + RETRY_SCHEDULER_SCHEDULE_ADJUSTMENT_TOLERANCE_MILLIS + 1 + ) + ).isFalse(); assertThat(scheduler.whenClosed()).isNotDone(); }); - Thread.sleep(1_001 + 100); - - assertNoRetryTaskExecutions(); - - runOnRetryEventLoop(() -> { - scheduler.close(); - assertThat(scheduler.whenClosed()).isCompleted(); - }); - - assertNoExceptionsOnRetryEventLoop(); + assertNumRetryTaskExecutions(0); + assertNoScheduleCalls(); } @Test - void schedule_exceptionInRetryTask_closeExceptionally() { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(1) - ); - + void schedule_exceptionInRetryTask_closeExceptionally() throws InterruptedException { final AtomicReference> whenClosedRef = new AtomicReference<>(); runOnRetryEventLoop(() -> { @@ -410,22 +346,21 @@ void schedule_exceptionInRetryTask_closeExceptionally() { } }); - assertRetryTaskExecutionsAt(200L); - - assertNoExceptionsOnRetryEventLoop(); + assertNumRetryTaskExecutions(1); + assertEventLoopScheduleCalls(200L); } @Test void schedule_closeRetryEventLoop_closeExceptionally() throws Exception { + final ManagedRetryEventLoop retryEventLoop = new ManagedRetryEventLoop(); final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) + System.nanoTime() + TimeUnit.SECONDS.toNanos(10) ); final AtomicReference> whenClosedRef = new AtomicReference<>(); - runOnRetryEventLoop(() -> { + runOnRetryEventLoop(retryEventLoop, () -> { assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); whenClosedRef.set(scheduler.whenClosed()); @@ -444,128 +379,215 @@ void schedule_closeRetryEventLoop_closeExceptionally() throws Exception { .hasMessageContaining("has been closed"); } - Thread.sleep(1_000 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(retryEventLoop, Collections.singletonList(1_000L)); + + Thread.sleep(1_000 + RETRY_SCHEDULER_SCHEDULE_ADJUSTMENT_TOLERANCE_MILLIS + + MAX_EXECUTION_DELAY_MILLIS); + + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(retryEventLoop, Collections.singletonList(1_000L)); - assertNoRetryTaskExecutions(); - assertNoExceptionsOnRetryEventLoop(); + assertNoExceptionsOnRetryEventLoop(retryEventLoop); } @Test - void applyMinimumBackoff_whileScheduling_throwIllegalStateException() throws InterruptedException { + void schedule_clogRetryEventLoop_closeExceptionally() throws Exception { final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) + System.nanoTime() + TimeUnit.SECONDS.toNanos(1) ); runOnRetryEventLoop(() -> { - assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); + assertThat(scheduler.trySchedule(nextRetryTask(), 500L)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); }); - Thread.sleep(200); + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(500L); runOnRetryEventLoop(() -> { - assertThatThrownBy(() -> scheduler.applyMinimumBackoffMillisForNextRetry(42)) - .isInstanceOf(IllegalStateException.class); - assertThat(scheduler.whenClosed()).isNotDone(); + BlockingUtils.blockingRun(() -> { + Thread.sleep(1_500L); + }); + }); + + await().untilAsserted(() -> { + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(500L); + }); + + await().untilAsserted(() -> { + runOnRetryEventLoop(() -> { + try { + scheduler.whenClosed().getNow(null); + fail(); + } catch (Throwable e) { + assertThat(e.getCause()).isInstanceOf(ResponseTimeoutException.class); + } + }); }); - Thread.sleep(800 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + Thread.sleep(500 + MAX_EXECUTION_DELAY_MILLIS); + + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(500L); + } + + @Test + void schedule_retryEventLoopRejects_closeExceptionally() throws Exception { + retryEventLoop.setRejectSchedule(true); - assertRetryTaskExecutionsAt(1_000L); + final AtomicReference> whenClosedRef = new AtomicReference<>(); runOnRetryEventLoop(() -> { + assertThat(scheduler.trySchedule(nextRetryTask(), 500L)).isFalse(); + assertThat(scheduler.whenClosed()).isDone(); + whenClosedRef.set(scheduler.whenClosed()); + }); + + try { + whenClosedRef.get().get(); + fail(); + } catch (Throwable e) { + assertThat(e.getCause()).isInstanceOf(RejectedExecutionException.class); + } + } + + @Test + void schedule_scheduledFutureCompletesExceptionally_closeExceptionally() throws InterruptedException { + final AtomicReference> whenClosedRef = new AtomicReference<>(); + + runOnRetryEventLoop(() -> { + assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); - scheduler.close(); - assertThat(scheduler.whenClosed()).isCompleted(); + whenClosedRef.set(scheduler.whenClosed()); + }); + + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(1_000L); + + runOnRetryEventLoop(() -> { + retryEventLoop.setExceptionWhenTaskRun(new AnticipatedException()); + }); + + await().untilAsserted(() -> { + try { + assertThat(whenClosedRef.get()).isCompletedExceptionally(); + whenClosedRef.get().get(); + fail(); + } catch (Throwable e) { + assertThat(e.getCause()).isInstanceOf(AnticipatedException.class); + } }); - assertNoExceptionsOnRetryEventLoop(); + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(1_000L); } @Test - void applyMinimumBackoff_thatExceedsDeadline_rejectEverySchedule() throws InterruptedException { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(1) + void applyMinimumBackoff_whileScheduling_throwIllegalStateException() throws InterruptedException { + runOnRetryEventLoop(() -> { + assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(1_000L); + Thread.sleep(200); + + runOnRetryEventLoop(() -> { + assertThatThrownBy(() -> scheduler.applyMinimumBackoffMillisForNextRetry(42)) + .isInstanceOf(IllegalStateException.class); + assertThat(scheduler.whenClosed()).isNotDone(); + }); + + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(1_000L); + + await().untilAsserted( + () -> { + assertNumRetryTaskExecutions(1); + assertEventLoopScheduleCalls(1_000L); + } ); + runOnRetryEventLoop(() -> assertThat(scheduler.whenClosed()).isNotDone()); + } + + @Test + void applyMinimumBackoff_thatExceedsDeadline_rejectEverySchedule() throws InterruptedException { runOnRetryEventLoop(() -> { - scheduler.applyMinimumBackoffMillisForNextRetry(1_001); + scheduler.applyMinimumBackoffMillisForNextRetry( + 10_000 + + RETRY_SCHEDULER_SCHEDULE_ADJUSTMENT_TOLERANCE_MILLIS + 1); assertThat(scheduler.whenClosed()).isNotDone(); assertThat(scheduler.trySchedule(nextRetryTask(), 0)).isFalse(); assertThat(scheduler.trySchedule(nextRetryTask(), 100)).isFalse(); }); - Thread.sleep(100 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + assertNoScheduleCalls(); + assertNumRetryTaskExecutions(0); runOnRetryEventLoop(() -> { assertThat(scheduler.whenClosed()).isNotDone(); + }); + } + + @Test + void close_afterNothing_returnTrue() { + runOnRetryEventLoop(() -> { scheduler.close(); assertThat(scheduler.whenClosed()).isCompleted(); }); - assertNoRetryTaskExecutions(); - assertNoExceptionsOnRetryEventLoop(); + assertNoScheduleCalls(); + assertNumRetryTaskExecutions(0); } @Test void close_thenSchedule_returnFalse() throws InterruptedException { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) - ); - runOnRetryEventLoop(() -> { scheduler.close(); assertThat(scheduler.whenClosed()).isCompleted(); assertThat(scheduler.trySchedule(nextRetryTask(), 200)).isFalse(); }); - Thread.sleep(200 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); - - assertNoRetryTaskExecutions(); - assertNoExceptionsOnRetryEventLoop(); + assertNoScheduleCalls(); + assertNumRetryTaskExecutions(0); } @Test void close_whileSchedule_cancelRetryTask() throws InterruptedException { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) - ); - runOnRetryEventLoop(() -> { assertThat(scheduler.trySchedule(nextRetryTask(), 1000L)).isTrue(); assertThat(scheduler.whenClosed()).isNotDone(); }); - Thread.sleep(250); - assertNoRetryTaskExecutions(); + assertNumRetryTaskExecutions(0); + assertEventLoopScheduleCalls(1000L); + + Thread.sleep(200); + + assertEventLoopScheduleCalls(1000L); + assertNumRetryTaskExecutions(0); runOnRetryEventLoop(() -> { scheduler.close(); assertThat(scheduler.whenClosed()).isCompleted(); }); - Thread.sleep(750 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + assertEventLoopScheduleCalls(1000L); + assertNumRetryTaskExecutions(0); + + Thread.sleep(800 + MAX_EXECUTION_DELAY_MILLIS); - assertNoRetryTaskExecutions(); - assertNoExceptionsOnRetryEventLoop(); + assertEventLoopScheduleCalls(1000L); + assertNumRetryTaskExecutions(0); } @Test void close_thenDoSomething_doesNotScheduleAndThrow() throws InterruptedException { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) - ); - runOnRetryEventLoop(() -> { scheduler.close(); assertThat(scheduler.whenClosed()).isCompleted(); @@ -576,7 +598,7 @@ void close_thenDoSomething_doesNotScheduleAndThrow() throws InterruptedException assertThat(scheduler.trySchedule(nextRetryTask(), 200)).isFalse(); }); - Thread.sleep(200 + RETRY_TASK_EXECUTION_TIME_TOLERANCE + 100); + Thread.sleep(200 + MAX_EXECUTION_DELAY_MILLIS); runOnRetryEventLoop(() -> { scheduler.applyMinimumBackoffMillisForNextRetry(Long.MIN_VALUE); @@ -587,21 +609,17 @@ void close_thenDoSomething_doesNotScheduleAndThrow() throws InterruptedException scheduler.close(); }); - assertNoRetryTaskExecutions(); - assertNoExceptionsOnRetryEventLoop(); + assertNoScheduleCalls(); + assertNumRetryTaskExecutions(0); } @Test - void schedule_applyMinimumBackoff_close_outsideRetryEventLoop_throwsIllegalStateException() { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, - System.nanoTime() + - TimeUnit.SECONDS.toNanos(5) - ); - + void all_outsideRetryEventLoop_throwsIllegalStateException() { assertThatThrownBy(() -> scheduler.trySchedule(nextRetryTask(), 100)).isInstanceOf( IllegalStateException.class); + assertNoScheduleCalls(); + assertNumRetryTaskExecutions(0); runOnRetryEventLoop(() -> { assertThat(scheduler.whenClosed()).isNotDone(); }); @@ -609,38 +627,40 @@ void schedule_applyMinimumBackoff_close_outsideRetryEventLoop_throwsIllegalState assertThatThrownBy(() -> scheduler.applyMinimumBackoffMillisForNextRetry(200)) .isInstanceOf(IllegalStateException.class); + assertNoScheduleCalls(); + assertNumRetryTaskExecutions(0); runOnRetryEventLoop(() -> { assertThat(scheduler.whenClosed()).isNotDone(); }); assertThatThrownBy(scheduler::close).isInstanceOf(IllegalStateException.class); + assertNoScheduleCalls(); + assertNumRetryTaskExecutions(0); runOnRetryEventLoop(() -> { assertThat(scheduler.whenClosed()).isNotDone(); - scheduler.close(); - assertThat(scheduler.whenClosed()).isCompleted(); }); - - assertNoRetryTaskExecutions(); - assertNoExceptionsOnRetryEventLoop(); } - private MockRetryTask nextRetryTask() { + private ManagedRetryTask nextRetryTask() { return nextRetryTask(() -> { // Default does nothing. }); } - private MockRetryTask nextRetryTask(Runnable innerTask) { - final int taskNumber = nextRetryTaskNumber.getAndIncrement(); - return new MockRetryTask(task -> { + private ManagedRetryTask nextRetryTask(Runnable innerTask) { + return new ManagedRetryTask(task -> { assertThat(retryEventLoop.inEventLoop()).isTrue(); executedRetryTasks.add(task); innerTask.run(); - }, taskNumber); + }); } private void runOnRetryEventLoop(Runnable runnable) { + runOnRetryEventLoop(retryEventLoop, runnable); + } + + private void runOnRetryEventLoop(ManagedRetryEventLoop retryEventLoop, Runnable runnable) { final CountDownLatch latch = new CountDownLatch(1); retryEventLoop.execute(() -> { try { @@ -655,49 +675,202 @@ private void runOnRetryEventLoop(Runnable runnable) { fail(e); } - assertNoExceptionsOnRetryEventLoop(); + assertNoExceptionsOnRetryEventLoop(retryEventLoop); } - private void assertNoRetryTaskExecutions() { - assertRetryTaskExecutionsAt(); + private static final class RetryPlan { + final String name; + final List retryTaskDelaysMillis; + // Does not include direct invocations as they don't call the retry event loop. + final List expectedScheduleDelayMillis; + + RetryPlan(String name, List retryTaskDelaysMillis, + List expectedScheduleDelayMillis) { + this.name = name; + this.retryTaskDelaysMillis = ImmutableList.copyOf(retryTaskDelaysMillis); + this.expectedScheduleDelayMillis = ImmutableList.copyOf(expectedScheduleDelayMillis); + } + + @Override + public String toString() { + return name; + } } - private void assertRetryTaskExecutionsAt(long... expectedDelaysMillis) { - assertRetryTaskExecutionsAt( - expectedDelaysMillis == null ? - Collections.emptyList() - : LongStream.of(expectedDelaysMillis).boxed() - .collect(Collectors.toList()) - ); + private static final class ScheduleCall { + final long delayMillis; + final List minimumBackoffMillis; + + ScheduleCall(long delayMillis, List minimumBackoffMillis) { + this.delayMillis = delayMillis; + this.minimumBackoffMillis = ImmutableList.copyOf(minimumBackoffMillis); + } + + ScheduleCall(long delayMillis) { + this.delayMillis = delayMillis; + minimumBackoffMillis = ImmutableList.of(); + } } - private void assertRetryTaskExecutionsAt(List expectedDelaysMillis) { - assertThat(executedRetryTasks).hasSize(expectedDelaysMillis.size()); + @NotNull + private CountDownLatch execRetryPlan(RetryPlan retryPlan) { + final int numberOfAttempts = retryPlan.retryTaskDelaysMillis.size(); + final List tasks = Collections.synchronizedList(new ArrayList<>()); - if (expectedDelaysMillis.isEmpty()) { - return; + final Consumer scheduleTask = retryNumber -> { + assert 0 <= retryNumber && retryNumber < numberOfAttempts; + assertThat(scheduler.whenClosed()).isNotDone(); + + final ScheduleCall scheduleCall = retryPlan.retryTaskDelaysMillis.get(retryNumber); + + for (long minimumBackoffMillis : scheduleCall.minimumBackoffMillis) { + scheduler.applyMinimumBackoffMillisForNextRetry(minimumBackoffMillis); + } + + // In the end we expect to execute that retry task after scheduledDelayMillis. + assertThat(scheduler.trySchedule( + tasks.get(retryNumber), + scheduleCall.delayMillis) + ) + .as("scheduling retry %d", retryNumber) + .isTrue(); + }; + + final CountDownLatch retryDone = new CountDownLatch(1); + + for (int retryNumber = 0; retryNumber < numberOfAttempts - 1; retryNumber++) { + final int nextRetryNumber = retryNumber + 1; + tasks.add(nextRetryTask(() -> scheduleTask.accept(nextRetryNumber))); } + tasks.add(nextRetryTask(retryDone::countDown)); + assert tasks.size() == numberOfAttempts; + + // Execute the first task. + runOnRetryEventLoop(() -> scheduleTask.accept(0)); - for (int expectedRetryTaskNumber = 0; expectedRetryTaskNumber < expectedDelaysMillis.size(); - expectedRetryTaskNumber++) { - final MockRetryTask task = executedRetryTasks.get(expectedRetryTaskNumber); + return retryDone; + } + + private void assertNoScheduleCalls() { + assertEventLoopScheduleCalls(Collections.emptyList()); + } - assertThat(task.nextRetryTaskNumber()).isEqualTo(expectedRetryTaskNumber); + private void assertEventLoopScheduleCalls(long... expectedScheduleDelaysMillisOnEventLoop) { + assert expectedScheduleDelaysMillisOnEventLoop != null; + assertEventLoopScheduleCalls(LongStream.of(expectedScheduleDelaysMillisOnEventLoop) + .boxed() + .collect(Collectors.toList())); + } + + private void assertEventLoopScheduleCalls(List expectedScheduleDelaysMillisOnEventLoop) { + assertEventLoopScheduleCalls(retryEventLoop, expectedScheduleDelaysMillisOnEventLoop); + } + + private void assertEventLoopScheduleCalls(ManagedRetryEventLoop retryEventLoop, + List expectedScheduleDelaysMillisOnEventLoop) { + assertThat(retryEventLoop.scheduleDelaysMillis()) + .as("Expected number of schedule() calls") + .hasSize(expectedScheduleDelaysMillisOnEventLoop.size()); + + for (int i = 0; i < expectedScheduleDelaysMillisOnEventLoop.size(); i++) { + final long expectedDelayMillis = expectedScheduleDelaysMillisOnEventLoop.get(i); + + assertThat(retryEventLoop.scheduleDelaysMillis().get(i)) + .isBetween( + expectedDelayMillis - RETRY_SCHEDULER_SCHEDULE_ADJUSTMENT_TOLERANCE_MILLIS, + expectedDelayMillis); + } + } - final long actualDelayMillis = TimeUnit.NANOSECONDS.toMillis( - task.executionTimeNanos() - task.scheduledTimeNanos() - ); - final long expectedDelayMillis = expectedDelaysMillis.get(expectedRetryTaskNumber); + private void assertNumRetryTaskExecutions(int numRetryTasks) { + assertThat(executedRetryTasks).hasSize(numRetryTasks); - assertThat(actualDelayMillis) - .as("Retry task %d being delayed appropriately", expectedRetryTaskNumber) - .isCloseTo(expectedDelayMillis, within(RETRY_TASK_EXECUTION_TIME_TOLERANCE)); + for (int i = 0; i < numRetryTasks; i++) { + final ManagedRetryTask task = executedRetryTasks.get(i); + assertThat(task.nextRetryTaskNumber()).isEqualTo(i); } } - private void assertNoExceptionsOnRetryEventLoop() { + private void assertNoExceptionsOnRetryEventLoop(ManagedRetryEventLoop retryEventLoop) { assertThat(retryEventLoop.exceptionsCaughtOnEventLoop()) .as("No exceptions thrown on retry event loop") .isEmpty(); } + + private static final class ManagedRetryTask implements Runnable { + static final AtomicInteger nextRetryTaskNumber = new AtomicInteger(0); + + private final Consumer runnable; + private final int retryTaskNumber; + private final AtomicBoolean executed; + + ManagedRetryTask(Consumer runnable) { + this.runnable = runnable; + retryTaskNumber = nextRetryTaskNumber.getAndIncrement(); + executed = new AtomicBoolean(false); + } + + @Override + public void run() { + assertThat(executed.get()).isFalse(); + executed.set(true); + runnable.accept(this); + } + + int nextRetryTaskNumber() { + return retryTaskNumber; + } + } + + private static final class ManagedRetryEventLoop extends DefaultEventLoop { + private final List exceptionsCaughtOnEventLoop = Collections.synchronizedList( + new ArrayList<>()); + private final List scheduleDelaysMillis = Collections.synchronizedList( + new ArrayList<>()); + private final AtomicBoolean rejectSchedule = new AtomicBoolean(false); + private final AtomicReference<@Nullable RuntimeException> throwWhenTaskRun = new AtomicReference<>( + null); + + @Override + protected void run() { + try { + super.run(); + } catch (Throwable t) { + exceptionsCaughtOnEventLoop.add(t); + throw t; + } + } + + void setRejectSchedule(boolean rejectSchedule) { + this.rejectSchedule.set(rejectSchedule); + } + + void setExceptionWhenTaskRun(@Nullable RuntimeException t) { + throwWhenTaskRun.set(t); + } + + @Override + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { + scheduleDelaysMillis.add(TimeUnit.MILLISECONDS.convert(delay, unit)); + if (rejectSchedule.get()) { + reject(); + } + + return super.schedule(() -> { + final RuntimeException t = throwWhenTaskRun.get(); + if (t != null) { + throw t; + } + command.run(); + }, delay, unit); + } + + List exceptionsCaughtOnEventLoop() { + return exceptionsCaughtOnEventLoop; + } + + List scheduleDelaysMillis() { + return scheduleDelaysMillis; + } + } } From edfc33b31760fbb987de56a880604ef1aa5c4a09 Mon Sep 17 00:00:00 2001 From: "szymon.habrainski" Date: Sat, 13 Sep 2025 15:56:15 +0200 Subject: [PATCH 42/42] test: reduce number of direct invocations for CI pipeline --- .../client/retry/RetrySchedulerTest.java | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java index 835fc66105e..ddce2c3284f 100644 --- a/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java +++ b/core/src/test/java/com/linecorp/armeria/client/retry/RetrySchedulerTest.java @@ -200,7 +200,7 @@ private static Stream schedule_multiple_executeMultipleAfterDelay_arg Arguments.of( new RetryPlan( "Many direct", - IntStream.range(0, 1024) + IntStream.range(0, 128) .mapToObj(i -> new ScheduleCall(0L)) .collect(Collectors.toList()), Collections.emptyList() // All direct @@ -243,8 +243,7 @@ private static Stream schedule_multiple_executeMultipleAfterDelay_arg @ParameterizedTest @MethodSource("schedule_multiple_executeMultipleAfterDelay_args") - void schedule_multiple_executeMultipleAfterDelay(RetryPlan retryPlan) - throws InterruptedException { + void schedule_multiple_executeMultipleAfterDelay(RetryPlan retryPlan) { final int numberOfAttempts = retryPlan.retryTaskDelaysMillis.size(); assert numberOfAttempts >= 1; @@ -260,7 +259,7 @@ void schedule_multiple_executeMultipleAfterDelay(RetryPlan retryPlan) } @Test - void schedule_withDelayAndMinimumBackoff_executeAfterDelay() throws InterruptedException { + void schedule_withDelayAndMinimumBackoff_executeAfterDelay() { final long minimumBackoffMillis = 200L; final long delayMillis = minimumBackoffMillis + 200L; @@ -287,7 +286,7 @@ void schedule_withDelayAndMinimumBackoff_executeAfterDelay() throws InterruptedE } @Test - void schedule_withDelayAndMinimumBackoff_executeAfterMinimumBackoff() throws InterruptedException { + void schedule_withDelayAndMinimumBackoff_executeAfterMinimumBackoff() { final long delayMillis = 200L; final long minimumBackoffMillis = delayMillis + 200L; @@ -306,7 +305,7 @@ void schedule_withDelayAndMinimumBackoff_executeAfterMinimumBackoff() throws Int } @Test - void schedule_beyondDeadline_returnFalse() throws InterruptedException { + void schedule_beyondDeadline_returnFalse() { runOnRetryEventLoop(() -> { assertThat(scheduler.trySchedule( nextRetryTask(), @@ -322,7 +321,7 @@ void schedule_beyondDeadline_returnFalse() throws InterruptedException { } @Test - void schedule_exceptionInRetryTask_closeExceptionally() throws InterruptedException { + void schedule_exceptionInRetryTask_closeExceptionally() { final AtomicReference> whenClosedRef = new AtomicReference<>(); runOnRetryEventLoop(() -> { @@ -352,22 +351,22 @@ void schedule_exceptionInRetryTask_closeExceptionally() throws InterruptedExcept @Test void schedule_closeRetryEventLoop_closeExceptionally() throws Exception { - final ManagedRetryEventLoop retryEventLoop = new ManagedRetryEventLoop(); - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( - retryEventLoop, + final ManagedRetryEventLoop localRetryEventLoop = new ManagedRetryEventLoop(); + final DefaultRetryScheduler localScheduler = new DefaultRetryScheduler( + localRetryEventLoop, System.nanoTime() + TimeUnit.SECONDS.toNanos(10) ); final AtomicReference> whenClosedRef = new AtomicReference<>(); - runOnRetryEventLoop(retryEventLoop, () -> { - assertThat(scheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); - assertThat(scheduler.whenClosed()).isNotDone(); - whenClosedRef.set(scheduler.whenClosed()); + runOnRetryEventLoop(localRetryEventLoop, () -> { + assertThat(localScheduler.trySchedule(nextRetryTask(), 1_000)).isTrue(); + assertThat(localScheduler.whenClosed()).isNotDone(); + whenClosedRef.set(localScheduler.whenClosed()); }); // Close the event loop before the task is executed. - retryEventLoop.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).get(); + localRetryEventLoop.shutdownGracefully(0, 500, TimeUnit.MILLISECONDS).get(); assertThat(whenClosedRef.get()).isCompletedExceptionally(); try { @@ -380,27 +379,26 @@ void schedule_closeRetryEventLoop_closeExceptionally() throws Exception { } assertNumRetryTaskExecutions(0); - assertEventLoopScheduleCalls(retryEventLoop, Collections.singletonList(1_000L)); + assertEventLoopScheduleCalls(localRetryEventLoop, Collections.singletonList(1_000L)); - Thread.sleep(1_000 + RETRY_SCHEDULER_SCHEDULE_ADJUSTMENT_TOLERANCE_MILLIS + - MAX_EXECUTION_DELAY_MILLIS); + Thread.sleep(1_000 + MAX_EXECUTION_DELAY_MILLIS); assertNumRetryTaskExecutions(0); - assertEventLoopScheduleCalls(retryEventLoop, Collections.singletonList(1_000L)); + assertEventLoopScheduleCalls(localRetryEventLoop, Collections.singletonList(1_000L)); - assertNoExceptionsOnRetryEventLoop(retryEventLoop); + assertNoExceptionsOnRetryEventLoop(localRetryEventLoop); } @Test void schedule_clogRetryEventLoop_closeExceptionally() throws Exception { - final DefaultRetryScheduler scheduler = new DefaultRetryScheduler( + final DefaultRetryScheduler localScheduler = new DefaultRetryScheduler( retryEventLoop, System.nanoTime() + TimeUnit.SECONDS.toNanos(1) ); runOnRetryEventLoop(() -> { - assertThat(scheduler.trySchedule(nextRetryTask(), 500L)).isTrue(); - assertThat(scheduler.whenClosed()).isNotDone(); + assertThat(localScheduler.trySchedule(nextRetryTask(), 500L)).isTrue(); + assertThat(localScheduler.whenClosed()).isNotDone(); }); assertNumRetryTaskExecutions(0); @@ -420,7 +418,7 @@ void schedule_clogRetryEventLoop_closeExceptionally() throws Exception { await().untilAsserted(() -> { runOnRetryEventLoop(() -> { try { - scheduler.whenClosed().getNow(null); + localScheduler.whenClosed().getNow(null); fail(); } catch (Throwable e) { assertThat(e.getCause()).isInstanceOf(ResponseTimeoutException.class); @@ -455,7 +453,7 @@ void schedule_retryEventLoopRejects_closeExceptionally() throws Exception { } @Test - void schedule_scheduledFutureCompletesExceptionally_closeExceptionally() throws InterruptedException { + void schedule_scheduledFutureCompletesExceptionally_closeExceptionally() { final AtomicReference> whenClosedRef = new AtomicReference<>(); runOnRetryEventLoop(() -> { @@ -516,7 +514,7 @@ void applyMinimumBackoff_whileScheduling_throwIllegalStateException() throws Int } @Test - void applyMinimumBackoff_thatExceedsDeadline_rejectEverySchedule() throws InterruptedException { + void applyMinimumBackoff_thatExceedsDeadline_rejectEverySchedule() { runOnRetryEventLoop(() -> { scheduler.applyMinimumBackoffMillisForNextRetry( 10_000 + @@ -546,7 +544,7 @@ void close_afterNothing_returnTrue() { } @Test - void close_thenSchedule_returnFalse() throws InterruptedException { + void close_thenSchedule_returnFalse() { runOnRetryEventLoop(() -> { scheduler.close(); assertThat(scheduler.whenClosed()).isCompleted();