fix: apply 429 backoff to organization connection writes to prevent rate-limit failures

src/tools/auth0/handlers/default.ts

@@ -35,7 +35,7 @@ const DEFAULT_MAX_RETRIES = 3;
 const DEFAULT_INITIAL_DELAY_MS = 1000; // 1 second
 const DEFAULT_MAX_DELAY_MS = 30000; // 30 seconds
 
-interface RetryOptions {
+export interface RetryOptions {
   maxRetries?: number;
   initialDelay?: number;
   maxDelay?: number;
@@ -50,7 +50,7 @@ interface RetryOptions {
  * @param options - Configuration options for retry behavior
  * @returns Promise that resolves with the function result or rejects after max retries
  */
-async function retryWithExponentialBackoff<T>(
+export async function retryWithExponentialBackoff<T>(
   fn: () => Promise<T>,
   options: RetryOptions = {}
 ): Promise<T> {
@@ -202,6 +202,30 @@ export default class APIHandler {
     return fn;
   }
 
+  /**
+   * Builds the exponential-backoff retry configuration for this handler from the
+   * shared `AUTH0_MAX_RETRIES` / `AUTH0_RETRY_INITIAL_DELAY_MS` /
+   * `AUTH0_RETRY_MAX_DELAY_MS` config keys. Exposed so that handlers which issue
+   * writes outside the default `processChanges` flow (e.g. organizations, which
+   * writes nested connections/grants directly) can wrap those calls in the same
+   * 429 backoff behaviour as the base handler.
+   */
+  getRetryConfig(): RetryOptions {
+    const retryConfig: RetryOptions = {
+      maxRetries: this.config('AUTH0_MAX_RETRIES') || DEFAULT_MAX_RETRIES,
+      initialDelay: this.config('AUTH0_RETRY_INITIAL_DELAY_MS') || DEFAULT_INITIAL_DELAY_MS,
+      maxDelay: this.config('AUTH0_RETRY_MAX_DELAY_MS') || DEFAULT_MAX_DELAY_MS,
+      onRetry: (error: any, attempt: number, delay: number) => {
+        log.warn(
+          `Rate limit hit for [${this.type}]. Retrying attempt ${attempt}/${
+            retryConfig.maxRetries
+          } after ${Math.round(delay / 1000)}s...`
+        );
+      },
+    };
+    return retryConfig;
+  }
+
   didDelete(item: Asset): void {
     log.info(`Deleted [${this.type}]: ${this.objString(item)}`);
   }
@@ -398,18 +422,7 @@ export default class APIHandler {
     );
 
     // Set retry configuration from config
-    const retryConfig: RetryOptions = {
-      maxRetries: this.config('AUTH0_MAX_RETRIES') || DEFAULT_MAX_RETRIES,
-      initialDelay: this.config('AUTH0_RETRY_INITIAL_DELAY_MS') || DEFAULT_INITIAL_DELAY_MS,
-      maxDelay: this.config('AUTH0_RETRY_MAX_DELAY_MS') || DEFAULT_MAX_DELAY_MS,
-      onRetry: (error: any, attempt: number, delay: number) => {
-        log.warn(
-          `Rate limit hit for [${this.type}]. Retrying attempt ${attempt}/${
-            retryConfig.maxRetries
-          } after ${Math.round(delay / 1000)}s...`
-        );
-      },
-    };
+    const retryConfig: RetryOptions = this.getRetryConfig();
 
     // Process Deleted
     if (del.length > 0) {

src/tools/auth0/handlers/organizations.ts

@@ -1,6 +1,6 @@
 import { omit, isEqual, pick } from 'lodash';
 import { Management } from 'auth0';
-import DefaultHandler, { order } from './default';
+import DefaultHandler, { order, retryWithExponentialBackoff } from './default';
 import { calculateChanges } from '../../calculateChanges';
 import log from '../../../logger';
 import { Asset, Assets, CalculatedChanges } from '../../../types';
@@ -187,12 +187,18 @@ export default class OrganizationsHandler extends DefaultHandler {
 
     const createdId = created.id;
 
+    const retryConfig = this.getRetryConfig();
+
     if (typeof org.connections !== 'undefined' && org.connections.length > 0) {
       await Promise.all(
         org.connections.map((conn) =>
-          this.client.organizations.connections.create(
-            createdId,
-            conn as Management.CreateOrganizationAllConnectionRequestParameters
+          retryWithExponentialBackoff(
+            () =>
+              this.client.organizations.connections.create(
+                createdId,
+                conn as Management.CreateOrganizationAllConnectionRequestParameters
+              ),
+            retryConfig
           )
         )
       );
@@ -322,53 +328,61 @@ export default class OrganizationsHandler extends DefaultHandler {
       changed = true;
     }
 
+    const retryConfig = this.getRetryConfig();
+
     // Handle updates first
     await Promise.all(
       connectionsToUpdate.map((conn: Management.CreateOrganizationAllConnectionRequestParameters) =>
-        this.client.organizations.connections
-          .update(params.id, conn.connection_id, {
-            organization_connection_name: conn.organization_connection_name,
-            assign_membership_on_login: conn.assign_membership_on_login,
-            show_as_button: conn.show_as_button,
-            is_signup_enabled: conn.is_signup_enabled,
-            is_enabled: conn.is_enabled,
-            organization_access_level: conn.organization_access_level,
-          })
-          .catch(() => {
-            throw new Error(
-              `Problem updating Enabled Connection ${conn.connection_id} for organizations ${params.id}`
-            );
-          })
+        retryWithExponentialBackoff(
+          () =>
+            this.client.organizations.connections.update(params.id, conn.connection_id, {
+              organization_connection_name: conn.organization_connection_name,
+              assign_membership_on_login: conn.assign_membership_on_login,
+              show_as_button: conn.show_as_button,
+              is_signup_enabled: conn.is_signup_enabled,
+              is_enabled: conn.is_enabled,
+              organization_access_level: conn.organization_access_level,
+            }),
+          retryConfig
+        ).catch(() => {
+          throw new Error(
+            `Problem updating Enabled Connection ${conn.connection_id} for organizations ${params.id}`
+          );
+        })
       )
     );
 
     await Promise.all(
       connectionsToAdd.map((conn: Management.CreateOrganizationAllConnectionRequestParameters) =>
-        this.client.organizations.connections
-          .create(
-            params.id,
-            omit<Management.OrganizationConnection>(
-              conn,
-              'connection'
-            ) as Management.AddOrganizationConnectionRequestContent
-          )
-          .catch(() => {
-            throw new Error(
-              `Problem adding Enabled Connection ${conn.connection_id} for organizations ${params.id}`
-            );
-          })
+        retryWithExponentialBackoff(
+          () =>
+            this.client.organizations.connections.create(
+              params.id,
+              omit<Management.OrganizationConnection>(
+                conn,
+                'connection'
+              ) as Management.AddOrganizationConnectionRequestContent
+            ),
+          retryConfig
+        ).catch(() => {
+          throw new Error(
+            `Problem adding Enabled Connection ${conn.connection_id} for organizations ${params.id}`
+          );
+        })
       )
     );
 
     await Promise.all(
       connectionsToRemove.map((conn: Management.OrganizationConnection) =>
-        this.client.organizations.connections
-          .delete(params.id, conn.connection_id as string)
-          .catch(() => {
-            throw new Error(
-              `Problem removing Enabled Connection ${conn.connection_id} for organizations ${params.id}`
-            );
-          })
+        retryWithExponentialBackoff(
+          () =>
+            this.client.organizations.connections.delete(params.id, conn.connection_id as string),
+          retryConfig
+        ).catch(() => {
+          throw new Error(
+            `Problem removing Enabled Connection ${conn.connection_id} for organizations ${params.id}`
+          );
+        })
       )
     );
 

test/tools/auth0/handlers/organizations.tests.js

@@ -967,6 +967,105 @@ describe('#organizations handler', () => {
       ]);
     });
 
+    it('should retry an enabled connection create when a 429 rate-limit error occurs', async () => {
+      // Config with a tiny retry delay so the exponential backoff resolves fast in tests.
+      const retryConfig = function (key) {
+        return retryConfig.data && retryConfig.data[key];
+      };
+      retryConfig.data = {
+        AUTH0_ALLOW_DELETE: true,
+        AUTH0_RETRY_INITIAL_DELAY_MS: 1,
+        AUTH0_RETRY_MAX_DELAY_MS: 5,
+      };
+
+      let createCallCount = 0;
+
+      const auth0 = {
+        organizations: {
+          create: () => Promise.resolve([]),
+          update: (id, data) => Promise.resolve(data),
+          delete: () => Promise.resolve([]),
+          list: (params) => Promise.resolve(mockPagedData(params, 'organizations', [sampleOrg])),
+          connections: {
+            list: () => ({
+              data: [],
+              hasNextPage: () => false,
+              getNextPage: () =>
+                Promise.resolve({
+                  data: [],
+                  hasNextPage: () => false,
+                  getNextPage: () => Promise.resolve({ data: [], hasNextPage: () => false }),
+                }),
+            }),
+            create: (orgId, data) => {
+              createCallCount += 1;
+              // Fail the first attempt with a 429, then succeed on the retry.
+              if (createCallCount === 1) {
+                const err = new Error('Too Many Requests');
+                err.statusCode = 429;
+                return Promise.reject(err);
+              }
+              expect(orgId).to.equal('123');
+              expect(data.connection_id).to.equal('con_123');
+              return Promise.resolve(data);
+            },
+          },
+          clientGrants: {
+            list: () => mockPagedData({}, 'client_grants', []),
+          },
+          discoveryDomains: {
+            list: () => mockPagedData({}, 'discovery_domains', []),
+          },
+          clients: {
+            list: () => ({ data: [], hasNextPage: () => false }),
+          },
+        },
+        connections: {
+          list: (params) =>
+            mockPagedData(params, 'connections', [
+              {
+                id: sampleEnabledConnection.connection_id,
+                name: sampleEnabledConnection.connection.name,
+                options: {},
+              },
+            ]),
+        },
+        clients: {
+          list: (params) => mockPagedData(params, 'clients', sampleClients),
+        },
+        clientGrants: {
+          list: (params) => mockPagedData(params, 'client_grants', [sampleClientGrant]),
+        },
+        pool,
+      };
+
+      const handler = new organizations.default({ client: pageClient(auth0), config: retryConfig });
+      const stageFn = Object.getPrototypeOf(handler).processChanges;
+
+      await stageFn.apply(handler, [
+        {
+          organizations: [
+            {
+              id: '123',
+              name: 'acme',
+              display_name: 'Acme 2',
+              connections: [
+                {
+                  name: 'Username-Password-Login',
+                  assign_membership_on_login: false,
+                  show_as_button: false,
+                  is_signup_enabled: false,
+                },
+              ],
+            },
+          ],
+        },
+      ]);
+
+      // The first call hit a 429 and the wrapper retried, so create is called twice.
+      expect(createCallCount).to.equal(2);
+    });
+
     it('should delete organizations', async () => {
       const auth0 = {
         organizations: {