feat: add support for YAML file includes and nested includes in context

src/context/yaml/index.ts

@@ -18,6 +18,52 @@ import { Assets, Config, Auth0APIClient, AssetTypes, KeywordMappings } from '../
 import { filterOnlyIncludedResourceTypes } from '..';
 import { preserveKeywords } from '../../keywordPreservation';
 
+// Custom YAML type for file includes
+const includeType = new yaml.Type('!include', {
+  kind: 'scalar',
+  resolve: (data) => typeof data === 'string',
+  construct: (data) => {
+    // This will be handled during the actual loading process
+    return { __include: data };
+  }
+});
+
+const schema = yaml.DEFAULT_SCHEMA.extend([includeType]);
+
+// Function to resolve includes
+function resolveIncludes(obj, basePath, mappings?: KeywordMappings, disableKeywordReplacement?: boolean) {
+  if (Array.isArray(obj)) {
+    return obj.map(item => resolveIncludes(item, basePath, mappings, disableKeywordReplacement));
+  }
+
+  if (obj && typeof obj === 'object') {
+    if (obj.__include) {
+      const filePath = path.resolve(basePath, obj.__include);
+      if (fs.existsSync(filePath)) {
+        let content = fs.readFileSync(filePath, 'utf8');
+
+        // Apply keyword replacement to included file content if mappings are provided
+        if (mappings && !disableKeywordReplacement) {
+          content = keywordReplace(content, mappings);
+        } else if (mappings && disableKeywordReplacement) {
+          content = wrapArrayReplaceMarkersInQuotes(content, mappings);
+        }
+
+        return resolveIncludes(yaml.load(content, { schema }), path.dirname(filePath), mappings, disableKeywordReplacement);
+      }
+      throw new Error(`Include file not found: ${filePath}`);
+    }
+
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = resolveIncludes(value, basePath, mappings, disableKeywordReplacement);
+    }
+    return result;
+  }
+
+  return obj;
+}
+
 export default class YAMLContext {
   basePath: string;
   configFile: string;
@@ -58,6 +104,10 @@ export default class YAMLContext {
     if (!isFile(toLoad)) {
       // try load not relative to yaml file
       toLoad = f;
+      if (!isFile(toLoad)) {
+        // try absolute path resolution
+        toLoad = path.resolve(f);
+      }
     }
     return loadFileAndReplaceKeywords(path.resolve(toLoad), {
       mappings: this.mappings,
@@ -74,13 +124,16 @@ export default class YAMLContext {
       try {
         const fPath = path.resolve(this.configFile);
         log.debug(`Loading YAML from ${fPath}`);
+        const loadedYaml = yaml.load(
+          opts.disableKeywordReplacement
+            ? wrapArrayReplaceMarkersInQuotes(fs.readFileSync(fPath, 'utf8'), this.mappings)
+            : keywordReplace(fs.readFileSync(fPath, 'utf8'), this.mappings),
+          { schema }
+        ) || {};
+
         Object.assign(
           this.assets,
-          yaml.load(
-            opts.disableKeywordReplacement
-              ? wrapArrayReplaceMarkersInQuotes(fs.readFileSync(fPath, 'utf8'), this.mappings)
-              : keywordReplace(fs.readFileSync(fPath, 'utf8'), this.mappings)
-          ) || {}
+          resolveIncludes(loadedYaml, path.dirname(fPath), this.mappings, opts.disableKeywordReplacement)
         );
       } catch (err) {
         log.debug(err.stack);

test/context/yaml/actions.test.js

@@ -66,6 +66,165 @@ describe('#YAML context actions', () => {
     expect(context.assets.actions).to.deep.equal(target);
   });
 
+  it('should process YAML with includes', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'includes');
+    cleanThenMkdir(dir);
+
+    const clientsYaml = `
+- name: "Test Client"
+  app_type: "spa"
+- name: "Test M2M"
+  app_type: "non_interactive"
+    `;
+    const clientsFile = path.join(dir, 'clients.yaml');
+    fs.writeFileSync(clientsFile, clientsYaml);
+
+    const mainYaml = `
+tenant:
+  friendly_name: 'Test Tenant'
+
+clients: !include clients.yaml
+    `;
+    const mainFile = path.join(dir, 'tenant.yaml');
+    fs.writeFileSync(mainFile, mainYaml);
+
+    const config = { AUTH0_INPUT_FILE: mainFile };
+    const context = new Context(config, mockMgmtClient());
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.tenant).to.deep.equal({
+      friendly_name: 'Test Tenant',
+    });
+    expect(context.assets.clients).to.deep.equal([
+      {
+        name: 'Test Client',
+        app_type: 'spa',
+      },
+      {
+        name: 'Test M2M',
+        app_type: 'non_interactive',
+      },
+    ]);
+  });
+
+  it('should handle nested includes', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'nested-includes');
+    cleanThenMkdir(dir);
+
+    const rolesYaml = `
+- name: Admin
+  description: Administrator
+- name: User
+  description: Regular User
+    `;
+    fs.writeFileSync(path.join(dir, 'roles.yaml'), rolesYaml);
+
+    const mainYaml = `
+tenant:
+  friendly_name: 'Main Tenant'
+
+roles: !include roles.yaml
+    `;
+    fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml);
+
+    const config = { AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml') };
+    const context = new Context(config, mockMgmtClient());
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.roles).to.deep.equal([
+      { name: 'Admin', description: 'Administrator' },
+      { name: 'User', description: 'Regular User' },
+    ]);
+  });
+
+  it('should process logStreams with includes', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'logstreams-includes');
+    cleanThenMkdir(dir);
+
+    const logStreamsYaml = `
+- name: LoggingSAAS
+  isPriority: false
+  filters:
+  - type: category
+    name: auth.login.fail
+  - type: category
+    name: auth.login.notification
+  - type: category
+    name: auth.login.success
+  - type: category
+    name: auth.logout.fail
+  sink:
+    httpContentFormat: JSONLINES
+    httpContentType: application/json
+    httpEndpoint: "##LOGGING_WEBHOOK_URL##"
+  type: http
+- name: SIEM
+  isPriority: false
+  filters:
+  - type: category
+    name: auth.login.fail
+  - type: category
+    name: auth.login.notification
+  - type: category
+    name: auth.login.success
+  sink:
+    httpContentFormat: JSONLINES
+    httpContentType: application/json
+    httpEndpoint: "##SIEM_WEBHOOK_URL##"
+  type: http
+    `;
+    fs.writeFileSync(path.join(dir, 'logStreams.yaml'), logStreamsYaml);
+
+    const mainYaml = `
+tenant:
+  friendly_name: 'Test Tenant'
+
+logStreams: !include logStreams.yaml
+    `;
+    fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml);
+
+    const config = {
+      AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml'),
+      AUTH0_KEYWORD_REPLACE_MAPPINGS: {
+        LOGGING_WEBHOOK_URL: 'https://logging.com/inputs/test',
+        SIEM_WEBHOOK_URL: 'https://siem.example.com/webhook'
+      }
+    };
+    const context = new Context(config, mockMgmtClient());
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.logStreams).to.have.length(2);
+    expect(context.assets.logStreams[0]).to.deep.include({
+      name: 'LoggingSAAS',
+      isPriority: false,
+      type: 'http'
+    });
+    expect(context.assets.logStreams[0].sink.httpEndpoint).to.equal('https://logging.com/inputs/test');
+    expect(context.assets.logStreams[1]).to.deep.include({
+      name: 'SIEM',
+      isPriority: false,
+      type: 'http'
+    });
+    expect(context.assets.logStreams[1].sink.httpEndpoint).to.equal('https://siem.example.com/webhook');
+  });
+
+  it('should error on missing include file', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'missing-include');
+    cleanThenMkdir(dir);
+
+    const mainYaml = `
+clients: !include missing.yaml
+    `;
+    fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml);
+
+    const config = { AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml') };
+    const context = new Context(config, mockMgmtClient());
+
+    await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith(
+      Error,
+      /Include file not found/
+    );
+  });
   it('should dump actions', async () => {
     const dir = path.join(testDataDir, 'yaml', 'actionsDump');
     cleanThenMkdir(dir);