diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py index e84d3b148d32..659714202e3c 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py @@ -162,6 +162,15 @@ def get_non_retryable_errors(self) -> dict[str, str | None]: "Request had invalid authentication credentials": "Your Google Ads connection could not be authenticated. Please reconnect your Google Ads account.", } + def get_retryable_errors(self) -> set[str]: + # A quota/rate-limit RESOURCE_EXHAUSTED ("Resource has been exhausted (e.g. check + # quota).") is already ridden out in-process by `_call_with_transient_retry` (see + # `_is_transient_grpc_error` in google_ads.py). A search that still fails after that + # budget has hit a longer-lived quota window than a few seconds of backoff can clear, + # but Temporal's activity retry recovers once it does — self-recovering, not a bug, so + # keep it out of error tracking as noise. + return {"Resource has been exhausted (e.g. check quota)"} + # TODO: clean up google ads source to not have two auth config options def parse_config(self, job_inputs: dict) -> GoogleAdsSourceConfig | GoogleAdsServiceAccountSourceConfig: if "google_ads_integration_id" in job_inputs.keys(): diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py index 513fd3f732f6..344d6402423b 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py @@ -325,6 +325,33 @@ def test_access_not_configured_has_friendly_message(self): assert "admin" in friendly.lower() +class TestGoogleAdsRetryableErrors: + def setup_method(self): + self.source = GoogleAdsSource() + self.retryable = self.source.get_retryable_errors() + + @pytest.mark.parametrize( + "error_msg", + [ + # str(google.api_core.exceptions.ResourceExhausted) as it propagates once + # `_call_with_transient_retry`'s in-process retry budget (see google_ads.py) is + # exhausted on a quota/rate-limit RESOURCE_EXHAUSTED. + "Resource has been exhausted (e.g. check quota).", + ], + ) + def test_quota_exhausted_is_retryable(self, error_msg): + # If this pattern drops out of get_retryable_errors(), a quota window that outlasts the + # in-process retry budget starts polluting error tracking even though Temporal's activity + # retry still recovers once the quota clears. + assert any(pattern in error_msg for pattern in self.retryable) + + def test_receive_limit_exhausted_is_not_retryable(self): + # The client-side "Received message larger than max" abort is deterministic (see + # `_is_transient_grpc_error`) — it must not be swallowed as benign noise here. + error_msg = "Received message larger than max (90000000 vs. 67108864)" + assert not any(pattern in error_msg for pattern in self.retryable) + + class TestGoogleAdsLookbackDefault: _SCHEMAS_PATH = "products.warehouse_sources.backend.temporal.data_imports.sources.google_ads.google_ads.get_schemas"