From 11370bd5f723b5b660a982092996eb0bf3febd93 Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Mon, 29 Jun 2026 17:10:21 +0200 Subject: [PATCH 01/17] Add set-custom and delete-custom commands Ad-hoc manipulation of a live company file's custom data, reusing the update*Custom functions. set-custom sets one property (value JSON-parsed when possible); delete-custom soft-deletes via null. Both support --level (inferred from URL), --handle/--account targeting, --file batch, and a confirmation prompt (--yes to skip). Adds lib/customWriter.js + tests. --- CHANGELOG.md | 2 + bin/cli.js | 54 ++++++++++++++ lib/customWriter.js | 125 +++++++++++++++++++++++++++++++++ tests/lib/customWriter.test.js | 109 ++++++++++++++++++++++++++++ 4 files changed, 290 insertions(+) create mode 100644 lib/customWriter.js create mode 100644 tests/lib/customWriter.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index f8404048..0142a725 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] Added `update-text-properties` command. It uploads custom text properties from a Liquid Test YAML file to a company file at company, period, reconciliation and account levels for the entries referenced in the test scenario. Usage: `silverfin update-text-properties -u -t `. Supports `--handle` for faster YAML file lookup, `--dry-run` to preview the payload, and `--yes` to skip the confirmation prompt. +Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a live company file's custom data. `set-custom -u --namespace --key --value ` sets a single custom (value JSON-parsed when possible); `delete-custom` soft-deletes it (value null). Both support `--level company|period|reconciliation|account` (inferred from the URL by default), `--handle`/`--account` targeting, `--file` for batch, and `--yes` to skip the confirmation prompt. + ## [1.56.1] (08/06/2026) Increase the waiting time for the test runs to avoid timeout errors. diff --git a/bin/cli.js b/bin/cli.js index ec50a94d..d5d12782 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -21,6 +21,7 @@ const { AutoCompletions } = require("../lib/cli/autoCompletions"); const fsUtils = require("../lib/utils/fsUtils"); const textPropertyUtils = require("../lib/utils/textPropertyUtils"); const liquidTestUtils = require("../lib/utils/liquidTestUtils"); +const customWriter = require("../lib/customWriter"); const firmIdDefault = cliUtils.loadDefaultFirmId(); cliUtils.handleUncaughtErrors(); @@ -668,6 +669,59 @@ program } }); +// Shared orchestration for set-custom / delete-custom +const runCustomWrite = async (options, del) => { + const plan = await customWriter.prepareWrite(options.url, options, { del }); + if (!plan) { + process.exitCode = 1; + return; + } + const verb = del ? "delete" : "set"; + const count = plan.properties.length; + consola.warn(`About to ${verb} ${count} custom propert${count === 1 ? "y" : "ies"} at the ${plan.level} level (${plan.targetDesc}) on firm ${plan.firmId}, company ${plan.companyId}.`); + if (!options.yes) { + cliUtils.promptConfirmation(); + } + const response = await plan.apply(); + const responses = Array.isArray(response) ? response : [response]; + const failed = responses.filter((r) => !r || r.status < 200 || r.status >= 300); + if (failed.length > 0) { + consola.error(`${verb}: ${failed.length}/${responses.length} request(s) failed`); + process.exitCode = 1; + } else { + consola.success(`${del ? "Deleted" : "Set"} ${count} custom propert${count === 1 ? "y" : "ies"} at the ${plan.level} level`); + } +}; + +// SET a custom value on a live company file +program + .command("set-custom") + .description("Set a custom value on a live company file (company/period/reconciliation/account level)") + .requiredOption("-u, --url ", "Full Silverfin URL of the reconciliation/account in the company file (mandatory)") + .option("--level ", "company | period | reconciliation | account (default: inferred from the URL)") + .option("--namespace ", "Custom namespace") + .option("--key ", "Custom key") + .option("--value ", "Custom value (JSON-parsed when possible, otherwise treated as a string)") + .option("--handle ", "Reconciliation handle (for --level reconciliation, instead of the URL target)") + .option("--account ", "Account number (for --level account)") + .option("--file ", "JSON file with an array of {namespace, key, value} for a batch set") + .option("-y, --yes", "Skip the confirmation prompt (optional)", false) + .action((options) => runCustomWrite(options, false)); + +// DELETE (soft-delete via null) a custom value on a live company file +program + .command("delete-custom") + .description("Delete (soft-delete via null) a custom value on a live company file") + .requiredOption("-u, --url ", "Full Silverfin URL of the reconciliation/account in the company file (mandatory)") + .option("--level ", "company | period | reconciliation | account (default: inferred from the URL)") + .option("--namespace ", "Custom namespace") + .option("--key ", "Custom key") + .option("--handle ", "Reconciliation handle (for --level reconciliation)") + .option("--account ", "Account number (for --level account)") + .option("--file ", "JSON file with an array of {namespace, key} for a batch delete") + .option("-y, --yes", "Skip the confirmation prompt (optional)", false) + .action((options) => runCustomWrite(options, true)); + // Check Liquid Test dependencies for a reconciliation template program .command("check-dependencies") diff --git a/lib/customWriter.js b/lib/customWriter.js new file mode 100644 index 00000000..ffdb28ee --- /dev/null +++ b/lib/customWriter.js @@ -0,0 +1,125 @@ +const SF = require("./api/sfApi"); +const Utils = require("./utils/liquidTestUtils"); +const { consola } = require("consola"); +const fs = require("fs"); + +/** + * Coerce a CLI string value into its natural JS type when possible + * (numbers, booleans, JSON objects/arrays), otherwise keep it as a string. + * e.g. "10" -> 10, "true" -> true, '{"a":1}' -> {a:1}, "hello" -> "hello". + */ +function coerceValue(raw) { + if (raw === undefined) { + return undefined; + } + try { + return JSON.parse(raw); + } catch { + return raw; + } +} + +/** + * Build the array of {namespace, key, value} property objects to write. + * For deletes, value is forced to null (soft-delete). + */ +function buildProperties(options, del) { + if (options.file) { + const parsed = JSON.parse(fs.readFileSync(options.file, "utf-8")); + if (!Array.isArray(parsed)) { + throw new Error("--file must contain a JSON array of {namespace, key[, value]} objects"); + } + return parsed.map((property) => ({ + namespace: property.namespace, + key: property.key, + value: del ? null : property.value, + })); + } + + if (!options.namespace || !options.key) { + throw new Error("--namespace and --key are required (or use --file)"); + } + if (!del && options.value === undefined) { + throw new Error("--value is required for set-custom (or use --file)"); + } + + return [{ namespace: options.namespace, key: options.key, value: del ? null : coerceValue(options.value) }]; +} + +/** + * Resolve the write target from the URL + options and assemble an apply() + * closure that calls the correct level-specific update function. Returns null + * (after logging) when the target/property set cannot be resolved. + * + * @param {String} url + * @param {Object} options CLI options (level, namespace, key, value, handle, account, file) + * @param {Object} [meta] + * @param {Boolean} [meta.del=false] + * @returns {Promise} { level, firmId, companyId, targetDesc, properties, apply } + */ +async function prepareWrite(url, options, { del = false } = {}) { + const parameters = Utils.extractURL(url); + const level = options.level || (parameters.templateType === "accountTemplate" ? "account" : "reconciliation"); + + let properties; + try { + properties = buildProperties(options, del); + } catch (error) { + consola.error(error.message); + return null; + } + + let apply; + let targetDesc; + switch (level) { + case "company": + targetDesc = `company ${parameters.companyId}`; + apply = () => SF.updateCompanyCustom(parameters.firmId, parameters.companyId, properties); + break; + case "period": + targetDesc = `period ${parameters.ledgerId}`; + apply = () => SF.updatePeriodCustom(parameters.firmId, parameters.companyId, parameters.ledgerId, properties); + break; + case "reconciliation": { + let reconciliationId = parameters.reconciliationId; + if (options.handle) { + const reconciliation = await SF.findReconciliationInWorkflows(parameters.firmId, options.handle, parameters.companyId, parameters.ledgerId); + if (!reconciliation) { + consola.error(`Reconciliation "${options.handle}" not found in any workflow.`); + return null; + } + reconciliationId = reconciliation.id; + } + if (!reconciliationId) { + consola.error("No reconciliation id found in the URL; pass --handle to target a reconciliation."); + return null; + } + targetDesc = `reconciliation ${reconciliationId}`; + apply = () => SF.updateReconciliationCustom(parameters.firmId, parameters.companyId, parameters.ledgerId, reconciliationId, properties); + break; + } + case "account": { + const accountNumber = options.account || parameters.accountId; + if (!accountNumber) { + consola.error("No account number found; pass --account to target an account."); + return null; + } + const account = await SF.findAccountByNumber(parameters.firmId, parameters.companyId, parameters.ledgerId, accountNumber); + if (!account?.account?.id) { + consola.error(`Account "${accountNumber}" could not be resolved in this company file.`); + return null; + } + const accountId = account.account.id; + targetDesc = `account ${accountNumber}`; + apply = () => SF.updateAccountCustom(parameters.firmId, parameters.companyId, parameters.ledgerId, accountId, properties); + break; + } + default: + consola.error(`Unknown level "${level}". Use company | period | reconciliation | account.`); + return null; + } + + return { level, firmId: parameters.firmId, companyId: parameters.companyId, targetDesc, properties, apply }; +} + +module.exports = { prepareWrite, buildProperties, coerceValue }; diff --git a/tests/lib/customWriter.test.js b/tests/lib/customWriter.test.js new file mode 100644 index 00000000..a10c0fb8 --- /dev/null +++ b/tests/lib/customWriter.test.js @@ -0,0 +1,109 @@ +jest.mock("../../lib/api/sfApi"); +jest.mock("../../lib/utils/liquidTestUtils"); +jest.mock("consola"); +jest.mock("fs"); + +const SF = require("../../lib/api/sfApi"); +const Utils = require("../../lib/utils/liquidTestUtils"); +const { consola } = require("consola"); +const fs = require("fs"); +const { prepareWrite, buildProperties, coerceValue } = require("../../lib/customWriter"); + +const reconUrl = "https://live.getsilverfin.com/f/96/100/..."; + +describe("customWriter", () => { + beforeEach(() => { + jest.clearAllMocks(); + Utils.extractURL.mockReturnValue({ + templateType: "reconciliationText", + firmId: "96", + companyId: "100", + ledgerId: "200", + reconciliationId: "300", + }); + }); + + describe("coerceValue", () => { + it("parses numbers, booleans and JSON, and keeps plain strings", () => { + expect(coerceValue("10")).toBe(10); + expect(coerceValue("true")).toBe(true); + expect(coerceValue('{"a":1}')).toEqual({ a: 1 }); + expect(coerceValue("hello")).toBe("hello"); + }); + }); + + describe("buildProperties", () => { + it("builds a single set property with a coerced value", () => { + expect(buildProperties({ namespace: "ns", key: "k", value: "10" }, false)).toEqual([{ namespace: "ns", key: "k", value: 10 }]); + }); + + it("forces value null for deletes", () => { + expect(buildProperties({ namespace: "ns", key: "k" }, true)).toEqual([{ namespace: "ns", key: "k", value: null }]); + }); + + it("reads a batch from --file (and nulls values on delete)", () => { + fs.readFileSync.mockReturnValue('[{"namespace":"n","key":"k","value":1},{"namespace":"n2","key":"k2","value":2}]'); + expect(buildProperties({ file: "props.json" }, false)).toEqual([ + { namespace: "n", key: "k", value: 1 }, + { namespace: "n2", key: "k2", value: 2 }, + ]); + expect(buildProperties({ file: "props.json" }, true)).toEqual([ + { namespace: "n", key: "k", value: null }, + { namespace: "n2", key: "k2", value: null }, + ]); + }); + + it("throws when namespace/key are missing", () => { + expect(() => buildProperties({ value: "x" }, false)).toThrow(); + }); + }); + + describe("prepareWrite", () => { + it("defaults to reconciliation level using the URL reconciliationId", async () => { + const plan = await prepareWrite(reconUrl, { namespace: "ns", key: "k", value: "5" }, { del: false }); + expect(plan.level).toBe("reconciliation"); + await plan.apply(); + expect(SF.updateReconciliationCustom).toHaveBeenCalledWith("96", "100", "200", "300", [{ namespace: "ns", key: "k", value: 5 }]); + }); + + it("resolves the reconciliation by --handle", async () => { + SF.findReconciliationInWorkflows.mockResolvedValue({ id: 777 }); + const plan = await prepareWrite(reconUrl, { handle: "some_handle", namespace: "ns", key: "k", value: "1" }); + await plan.apply(); + expect(SF.findReconciliationInWorkflows).toHaveBeenCalledWith("96", "some_handle", "100", "200"); + expect(SF.updateReconciliationCustom).toHaveBeenCalledWith("96", "100", "200", 777, expect.any(Array)); + }); + + it("writes at company level", async () => { + const plan = await prepareWrite(reconUrl, { level: "company", namespace: "ns", key: "k", value: "1" }); + await plan.apply(); + expect(SF.updateCompanyCustom).toHaveBeenCalledWith("96", "100", [{ namespace: "ns", key: "k", value: 1 }]); + }); + + it("writes at period level", async () => { + const plan = await prepareWrite(reconUrl, { level: "period", namespace: "ns", key: "k", value: "1" }); + await plan.apply(); + expect(SF.updatePeriodCustom).toHaveBeenCalledWith("96", "100", "200", expect.any(Array)); + }); + + it("resolves the account and writes at account level", async () => { + SF.findAccountByNumber.mockResolvedValue({ account: { id: 555 } }); + const plan = await prepareWrite(reconUrl, { level: "account", account: "610000", namespace: "ns", key: "k", value: "1" }); + await plan.apply(); + expect(SF.findAccountByNumber).toHaveBeenCalledWith("96", "100", "200", "610000"); + expect(SF.updateAccountCustom).toHaveBeenCalledWith("96", "100", "200", 555, expect.any(Array)); + }); + + it("builds null-valued properties for a delete", async () => { + const plan = await prepareWrite(reconUrl, { namespace: "ns", key: "k" }, { del: true }); + expect(plan.properties).toEqual([{ namespace: "ns", key: "k", value: null }]); + }); + + it("returns null and logs when the account cannot be resolved", async () => { + SF.findAccountByNumber.mockResolvedValue(null); + const plan = await prepareWrite(reconUrl, { level: "account", account: "999", namespace: "ns", key: "k", value: "1" }); + expect(plan).toBeNull(); + expect(consola.error).toHaveBeenCalled(); + }); + }); +}); From 6340435fde67b959bc1871ceddf15c9dd9c7d655 Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Mon, 29 Jun 2026 17:29:37 +0200 Subject: [PATCH 02/17] Add get-results and capture commands Consolidates the live-bridge read commands onto this branch so the four new commands (get-results, capture, set-custom, delete-custom) ship together on top of update-text-properties (#249). get-results reads a live file's computed results+customs; capture snapshots a live file as JSON (scoped, or --full). Adds lib/resultsReader.js, lib/dataCapture.js (+ buildLiquidTest refactor of liquidTestGenerator), and their tests. --- CHANGELOG.md | 4 + bin/cli.js | 47 ++++++++++++ lib/dataCapture.js | 127 ++++++++++++++++++++++++++++++++ lib/liquidTestGenerator.js | 13 +++- lib/resultsReader.js | 58 +++++++++++++++ tests/lib/dataCapture.test.js | 79 ++++++++++++++++++++ tests/lib/resultsReader.test.js | 80 ++++++++++++++++++++ 7 files changed, 406 insertions(+), 2 deletions(-) create mode 100644 lib/dataCapture.js create mode 100644 lib/resultsReader.js create mode 100644 tests/lib/dataCapture.test.js create mode 100644 tests/lib/resultsReader.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 0142a725..557d43c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. ## [Unreleased] Added `update-text-properties` command. It uploads custom text properties from a Liquid Test YAML file to a company file at company, period, reconciliation and account levels for the entries referenced in the test scenario. Usage: `silverfin update-text-properties -u -t `. Supports `--handle` for faster YAML file lookup, `--dry-run` to preview the payload, and `--yes` to skip the confirmation prompt. +Added `get-results` command. It fetches the computed results and custom data of a reconciliation or account in a live company file (identified by its Silverfin URL) and prints them as JSON. Usage: `silverfin get-results -u `. Supports `-o, --output ` to write the JSON to a file instead of stdout. + +Added `capture` command. It captures a live company file's data as JSON. By default it captures the template at the URL and its dependencies (scoped); `--full` captures company/period/reconciliation customs and results across all periods. Usage: `silverfin capture -u [--full]`. Supports `-o, --output `. + Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a live company file's custom data. `set-custom -u --namespace --key --value ` sets a single custom (value JSON-parsed when possible); `delete-custom` soft-deletes it (value null). Both support `--level company|period|reconciliation|account` (inferred from the URL by default), `--handle`/`--account` targeting, `--file` for batch, and `--yes` to skip the confirmation prompt. ## [1.56.1] (08/06/2026) diff --git a/bin/cli.js b/bin/cli.js index d5d12782..c7a2d694 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -3,6 +3,8 @@ const toolkit = require("../index"); const liquidTestGenerator = require("../lib/liquidTestGenerator"); const liquidTestRunner = require("../lib/liquidTestRunner"); +const resultsReader = require("../lib/resultsReader"); +const dataCapture = require("../lib/dataCapture"); const { ExportFileInstanceGenerator } = require("../lib/exportFileInstanceGenerator"); const stats = require("../lib/cli/stats"); const { Command, Option } = require("commander"); @@ -530,6 +532,51 @@ program liquidTestGenerator.testGenerator(options.url, testName, reconciledStatus); }); +// GET RESULTS — read back a live company file's computed results + customs as JSON +program + .command("get-results") + .description("Fetch the computed results and custom data of a reconciliation or account in a live company file, printed as JSON") + .requiredOption("-u, --url ", "Specify the full Silverfin URL of the reconciliation/account in the company file (mandatory)") + .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") + .action(async (options) => { + const data = await resultsReader.fetchResults(options.url); + if (!data) { + process.exitCode = 1; + return; + } + const json = JSON.stringify(data, null, 2); + if (options.output) { + const fs = require("fs"); + fs.writeFileSync(options.output, json); + consola.success(`Wrote results to ${options.output}`); + } else { + console.log(json); + } + }); + +// CAPTURE — snapshot a live company file's data as JSON +program + .command("capture") + .description("Capture a live company file's data as JSON. Default: the template at the URL and its dependencies (scoped). Use --full to capture company/period/reconciliation customs and results across all periods") + .requiredOption("-u, --url ", "Specify the full Silverfin URL of the reconciliation/account in the company file (mandatory)") + .option("--full", "Capture the whole company file (all periods, workflows, reconciliations) instead of just the template's scope (optional)", false) + .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") + .action(async (options) => { + const data = await dataCapture.capture(options.url, { full: options.full }); + if (!data) { + process.exitCode = 1; + return; + } + const json = JSON.stringify(data, null, 2); + if (options.output) { + const fs = require("fs"); + fs.writeFileSync(options.output, json); + consola.success(`Wrote capture to ${options.output}`); + } else { + console.log(json); + } + }); + // Update Text Properties from Liquid Test data program .command("update-text-properties") diff --git a/lib/dataCapture.js b/lib/dataCapture.js new file mode 100644 index 00000000..9868a9d3 --- /dev/null +++ b/lib/dataCapture.js @@ -0,0 +1,127 @@ +const SF = require("./api/sfApi"); +const Utils = require("./utils/liquidTestUtils"); +const liquidTestGenerator = require("./liquidTestGenerator"); +const { consola } = require("consola"); + +const PER_PAGE = 200; +const MAX_PAGES = 50; + +/** + * Capture a live company file's data as a plain object (no I/O). + * + * Two modes: + * - scoped (default): reuses the create-test gathering (`buildLiquidTest`) to + * capture exactly what the template at the URL references (+ dependencies). + * - full: fans out across the whole company file (company + every period + + * every workflow reconciliation) collecting customs and results. + * + * The Silverfin API allows max 1 in-flight call per company, so the full + * capture is intentionally sequential. + * + * @param {String} url Full Silverfin URL of the reconciliation/account + * @param {Object} [opts] + * @param {Boolean} [opts.full=false] + * @returns {Promise} + */ +async function capture(url, opts = {}) { + return opts.full ? captureFull(url) : captureScoped(url); +} + +async function captureScoped(url) { + const built = await liquidTestGenerator.buildLiquidTest(url, "capture", true); + if (!built) { + return null; + } + const snapshot = (built.liquidTestObject && built.liquidTestObject.capture) || {}; + return { + mode: "scoped", + handle: built.templateHandle, + templateType: built.templateType, + context: snapshot.context ?? null, + data: snapshot.data ?? null, + expectation: snapshot.expectation ?? null, + }; +} + +async function captureFull(url) { + const parameters = Utils.extractURL(url); + const { firmId, companyId } = parameters; + + const out = { mode: "full", firmId, companyId, company: {}, periods: {} }; + + // Company-level drop + customs + const companyDrop = await SF.getCompanyDrop(firmId, companyId); + out.company.drop = companyDrop?.data ?? null; + const companyCustom = await SF.getCompanyCustom(firmId, companyId); + out.company.custom = companyCustom?.data ?? null; + + // Every period + const periods = await fetchAllPeriods(firmId, companyId); + for (const period of periods) { + const periodId = period.id; + const key = period.fiscal_year?.end_date ? String(period.fiscal_year.end_date) : String(periodId); + const periodEntry = { periodId, custom: [], workflows: {} }; + + // Period-level customs (paginated internally) + periodEntry.custom = (await SF.getAllPeriodCustom(firmId, companyId, periodId)) || []; + + // Workflows -> reconciliations (custom + results) + const workflowsResponse = await SF.getWorkflows(firmId, companyId, periodId); + const workflows = workflowsResponse?.data ?? []; + for (const workflow of workflows) { + const reconciliations = await fetchAllWorkflowReconciliations(firmId, companyId, periodId, workflow.id); + const reconciliationsOut = {}; + for (const reconciliation of reconciliations) { + const customResponse = await SF.getReconciliationCustom("firm", firmId, companyId, periodId, reconciliation.id); + const resultsResponse = await SF.getReconciliationResults("firm", firmId, companyId, periodId, reconciliation.id); + reconciliationsOut[reconciliation.handle || reconciliation.id] = { + id: reconciliation.id, + custom: customResponse?.data ?? null, + results: resultsResponse?.data ?? null, + }; + } + const workflowKey = workflow.name ? `${workflow.name} (${workflow.id})` : String(workflow.id); + periodEntry.workflows[workflowKey] = { id: workflow.id, reconciliations: reconciliationsOut }; + } + + out.periods[key] = periodEntry; + } + + // Account-level customs are not included in --full: there is no list-accounts + // endpoint wired into the CLI. Use scoped capture (or get-results) for accounts. + consola.info("Full capture covers company/period/reconciliation customs + results. Account-level customs are only captured in scoped mode."); + + return out; +} + +async function fetchAllPeriods(firmId, companyId) { + const items = []; + let page = 1; + while (page <= MAX_PAGES) { + const response = await SF.getPeriods(firmId, companyId, page); + const data = response?.data ?? []; + items.push(...data); + if (data.length < PER_PAGE) { + break; + } + page++; + } + return items; +} + +async function fetchAllWorkflowReconciliations(firmId, companyId, periodId, workflowId) { + const items = []; + let page = 1; + while (page <= MAX_PAGES) { + const response = await SF.getWorkflowInformation(firmId, companyId, periodId, workflowId, page); + const data = response?.data ?? []; + items.push(...data); + if (data.length < PER_PAGE) { + break; + } + page++; + } + return items; +} + +module.exports = { capture, captureScoped, captureFull }; diff --git a/lib/liquidTestGenerator.js b/lib/liquidTestGenerator.js index 1703969c..6de095a7 100644 --- a/lib/liquidTestGenerator.js +++ b/lib/liquidTestGenerator.js @@ -7,7 +7,7 @@ const { AccountTemplate } = require("./templates/accountTemplate"); const { SharedPart } = require("./templates/sharedPart"); // MainProcess -async function testGenerator(url, testName, reconciledStatus = true) { +async function buildLiquidTest(url, testName, reconciledStatus = true) { // Get parameters from URL provided and determine template type const parameters = Utils.extractURL(url); const templateType = parameters.templateType; @@ -313,10 +313,19 @@ async function testGenerator(url, testName, reconciledStatus = true) { } } + return { templateHandle, liquidTestObject, templateType }; +} + +async function testGenerator(url, testName, reconciledStatus = true) { + const built = await buildLiquidTest(url, testName, reconciledStatus); + if (!built) { + return; + } // Save YAML - Utils.exportYAML(templateHandle, liquidTestObject, templateType); + Utils.exportYAML(built.templateHandle, built.liquidTestObject, built.templateType); } module.exports = { testGenerator, + buildLiquidTest, }; diff --git a/lib/resultsReader.js b/lib/resultsReader.js new file mode 100644 index 00000000..6a92079f --- /dev/null +++ b/lib/resultsReader.js @@ -0,0 +1,58 @@ +const SF = require("./api/sfApi"); +const Utils = require("./utils/liquidTestUtils"); +const { consola } = require("consola"); + +/** + * Fetch the computed results and custom data of a reconciliation or account + * in a LIVE company file, identified by its Silverfin URL. + * + * Returns a plain object (no I/O) so it is easy to test and to serialise. + * Reuses the same getters and URL parsing as `create-test`. + * + * @param {String} url Full Silverfin URL of the reconciliation/account in the company file + * @returns {Promise} { templateType, firmId, companyId, periodId, ... , results, custom } or null on failure + */ +async function fetchResults(url) { + const parameters = Utils.extractURL(url); + + switch (parameters.templateType) { + case "reconciliationText": { + const customResponse = await SF.getReconciliationCustom("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); + const resultsResponse = await SF.getReconciliationResults("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); + return { + templateType: parameters.templateType, + firmId: parameters.firmId, + companyId: parameters.companyId, + periodId: parameters.ledgerId, + reconciliationId: parameters.reconciliationId, + results: resultsResponse?.data ?? null, + custom: customResponse?.data ?? null, + }; + } + case "accountTemplate": { + const account = await SF.findAccountByNumber(parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.accountId); + const accountId = account?.account?.id; + if (!accountId) { + consola.error(`Account "${parameters.accountId}" could not be resolved in this company file.`); + return null; + } + const customResponse = await SF.getAccountTemplateCustom("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, accountId); + const resultsResponse = await SF.getAccountTemplateResults("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, accountId); + return { + templateType: parameters.templateType, + firmId: parameters.firmId, + companyId: parameters.companyId, + periodId: parameters.ledgerId, + accountNumber: account.account.number, + accountId, + results: resultsResponse?.data ?? null, + custom: customResponse?.data ?? null, + }; + } + default: + consola.error(`Unsupported template type for URL: ${url}`); + return null; + } +} + +module.exports = { fetchResults }; diff --git a/tests/lib/dataCapture.test.js b/tests/lib/dataCapture.test.js new file mode 100644 index 00000000..68865ea6 --- /dev/null +++ b/tests/lib/dataCapture.test.js @@ -0,0 +1,79 @@ +jest.mock("../../lib/api/sfApi"); +jest.mock("../../lib/utils/liquidTestUtils"); +jest.mock("../../lib/liquidTestGenerator"); +jest.mock("consola"); + +const SF = require("../../lib/api/sfApi"); +const Utils = require("../../lib/utils/liquidTestUtils"); +const liquidTestGenerator = require("../../lib/liquidTestGenerator"); +const { capture } = require("../../lib/dataCapture"); + +describe("dataCapture.capture", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe("scoped (default)", () => { + it("reuses buildLiquidTest and returns the scoped snapshot", async () => { + liquidTestGenerator.buildLiquidTest.mockResolvedValue({ + templateHandle: "my_handle", + templateType: "reconciliationText", + liquidTestObject: { + capture: { + context: { period: "2024-12-31" }, + data: { periods: { "2024-12-31": {} } }, + expectation: { results: { r: "1" } }, + }, + }, + }); + + const result = await capture("https://live.getsilverfin.com/f/96/100/...", { full: false }); + + expect(liquidTestGenerator.buildLiquidTest).toHaveBeenCalledWith("https://live.getsilverfin.com/f/96/100/...", "capture", true); + expect(result).toEqual({ + mode: "scoped", + handle: "my_handle", + templateType: "reconciliationText", + context: { period: "2024-12-31" }, + data: { periods: { "2024-12-31": {} } }, + expectation: { results: { r: "1" } }, + }); + }); + + it("returns null when the template could not be built", async () => { + liquidTestGenerator.buildLiquidTest.mockResolvedValue(undefined); + const result = await capture("https://live.getsilverfin.com/f/96/100/..."); + expect(result).toBeNull(); + }); + }); + + describe("full", () => { + it("captures company + period + workflow reconciliation customs and results", async () => { + Utils.extractURL.mockReturnValue({ firmId: "96", companyId: "100" }); + SF.getCompanyDrop.mockResolvedValue({ data: { name: "Co" } }); + SF.getCompanyCustom.mockResolvedValue({ data: [{ namespace: "c", key: "k", value: 1 }] }); + SF.getPeriods.mockResolvedValue({ data: [{ id: 200, fiscal_year: { end_date: "2024-12-31" } }] }); + SF.getAllPeriodCustom.mockResolvedValue([{ namespace: "p", key: "k", value: 2 }]); + SF.getWorkflows.mockResolvedValue({ data: [{ id: 11, name: "WF" }] }); + SF.getWorkflowInformation.mockResolvedValue({ data: [{ id: 300, handle: "recon_a" }] }); + SF.getReconciliationCustom.mockResolvedValue({ data: [{ namespace: "r", key: "k", value: 3 }] }); + SF.getReconciliationResults.mockResolvedValue({ data: { res: "9" } }); + + const result = await capture("https://live.getsilverfin.com/f/96/100/...", { full: true }); + + expect(result.mode).toBe("full"); + expect(result.firmId).toBe("96"); + expect(result.company.drop).toEqual({ name: "Co" }); + expect(result.company.custom).toEqual([{ namespace: "c", key: "k", value: 1 }]); + + const period = result.periods["2024-12-31"]; + expect(period.periodId).toBe(200); + expect(period.custom).toEqual([{ namespace: "p", key: "k", value: 2 }]); + expect(period.workflows["WF (11)"].reconciliations.recon_a).toEqual({ + id: 300, + custom: [{ namespace: "r", key: "k", value: 3 }], + results: { res: "9" }, + }); + }); + }); +}); diff --git a/tests/lib/resultsReader.test.js b/tests/lib/resultsReader.test.js new file mode 100644 index 00000000..054b9475 --- /dev/null +++ b/tests/lib/resultsReader.test.js @@ -0,0 +1,80 @@ +jest.mock("../../lib/api/sfApi"); +jest.mock("../../lib/utils/liquidTestUtils"); +jest.mock("consola"); + +const SF = require("../../lib/api/sfApi"); +const Utils = require("../../lib/utils/liquidTestUtils"); +const { consola } = require("consola"); +const { fetchResults } = require("../../lib/resultsReader"); + +describe("resultsReader.fetchResults", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("fetches results + customs for a reconciliation URL", async () => { + Utils.extractURL.mockReturnValue({ + templateType: "reconciliationText", + firmId: "96", + companyId: "100", + ledgerId: "200", + reconciliationId: "300", + }); + SF.getReconciliationCustom.mockResolvedValue({ data: [{ namespace: "ns", key: "k", value: "v" }] }); + SF.getReconciliationResults.mockResolvedValue({ data: { vol_1022_man: "5000.0" } }); + + const result = await fetchResults("https://live.getsilverfin.com/f/96/100/..."); + + expect(SF.getReconciliationResults).toHaveBeenCalledWith("firm", "96", "100", "200", "300"); + expect(result).toEqual({ + templateType: "reconciliationText", + firmId: "96", + companyId: "100", + periodId: "200", + reconciliationId: "300", + results: { vol_1022_man: "5000.0" }, + custom: [{ namespace: "ns", key: "k", value: "v" }], + }); + }); + + it("resolves the account and fetches results + customs for an account URL", async () => { + Utils.extractURL.mockReturnValue({ + templateType: "accountTemplate", + firmId: "96", + companyId: "100", + ledgerId: "200", + accountId: "610000", + }); + SF.findAccountByNumber.mockResolvedValue({ account: { id: 555, number: "610000", name: "Costs" } }); + SF.getAccountTemplateCustom.mockResolvedValue({ data: [{ namespace: "a", key: "b", value: 1 }] }); + SF.getAccountTemplateResults.mockResolvedValue({ data: { unreconciled_amount: "0.0" } }); + + const result = await fetchResults("https://live.getsilverfin.com/f/96/100/..."); + + expect(SF.getAccountTemplateResults).toHaveBeenCalledWith("firm", "96", "100", "200", 555); + expect(result).toMatchObject({ + templateType: "accountTemplate", + accountNumber: "610000", + accountId: 555, + results: { unreconciled_amount: "0.0" }, + custom: [{ namespace: "a", key: "b", value: 1 }], + }); + }); + + it("returns null and logs an error when the account cannot be resolved", async () => { + Utils.extractURL.mockReturnValue({ + templateType: "accountTemplate", + firmId: "96", + companyId: "100", + ledgerId: "200", + accountId: "999999", + }); + SF.findAccountByNumber.mockResolvedValue(null); + + const result = await fetchResults("https://live.getsilverfin.com/f/96/100/..."); + + expect(result).toBeNull(); + expect(consola.error).toHaveBeenCalled(); + expect(SF.getAccountTemplateResults).not.toHaveBeenCalled(); + }); +}); From ca2c13469d54a13b553b7e23678c297c5517f0d1 Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Tue, 30 Jun 2026 14:28:59 +0200 Subject: [PATCH 03/17] Add describe-inputs command Lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results. Effective values come only from certain sources (stored override, a directly-echoed result, or a literal default); everything else is flagged unavailable rather than re-rendered (a re-render can't faithfully reproduce live company state). Adds lib/inputDescriber.js + tests. --- CHANGELOG.md | 2 + bin/cli.js | 23 +++++ lib/inputDescriber.js | 154 +++++++++++++++++++++++++++++++ tests/lib/inputDescriber.test.js | 96 +++++++++++++++++++ 4 files changed, 275 insertions(+) create mode 100644 lib/inputDescriber.js create mode 100644 tests/lib/inputDescriber.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 557d43c8..18680dc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Added `capture` command. It captures a live company file's data as JSON. By defa Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a live company file's custom data. `set-custom -u --namespace --key --value ` sets a single custom (value JSON-parsed when possible); `delete-custom` soft-deletes it (value null). Both support `--level company|period|reconciliation|account` (inferred from the URL by default), `--handle`/`--account` targeting, `--file` for batch, and `--yes` to skip the confirmation prompt. +Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). + ## [1.56.1] (08/06/2026) Increase the waiting time for the test runs to avoid timeout errors. diff --git a/bin/cli.js b/bin/cli.js index c7a2d694..796d1d0f 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -5,6 +5,7 @@ const liquidTestGenerator = require("../lib/liquidTestGenerator"); const liquidTestRunner = require("../lib/liquidTestRunner"); const resultsReader = require("../lib/resultsReader"); const dataCapture = require("../lib/dataCapture"); +const inputDescriber = require("../lib/inputDescriber"); const { ExportFileInstanceGenerator } = require("../lib/exportFileInstanceGenerator"); const stats = require("../lib/cli/stats"); const { Command, Option } = require("commander"); @@ -577,6 +578,28 @@ program } }); +// DESCRIBE INPUTS — list a reconciliation's custom inputs with declared defaults + live effective values +program + .command("describe-inputs") + .description("List a reconciliation's custom inputs with their declared defaults, stored values, and live effective values (certain sources only), plus the template's results. Run from your templates repo") + .requiredOption("-u, --url ", "Specify the full Silverfin URL of the reconciliation in the company file (mandatory)") + .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") + .action(async (options) => { + const data = await inputDescriber.describeInputs(options.url); + if (!data) { + process.exitCode = 1; + return; + } + const json = JSON.stringify(data, null, 2); + if (options.output) { + const fs = require("fs"); + fs.writeFileSync(options.output, json); + consola.success(`Wrote inputs to ${options.output}`); + } else { + console.log(json); + } + }); + // Update Text Properties from Liquid Test data program .command("update-text-properties") diff --git a/lib/inputDescriber.js b/lib/inputDescriber.js new file mode 100644 index 00000000..e6e9946f --- /dev/null +++ b/lib/inputDescriber.js @@ -0,0 +1,154 @@ +const SF = require("./api/sfApi"); +const Utils = require("./utils/liquidTestUtils"); +const { ReconciliationText } = require("./templates/reconciliationText"); +const { consola } = require("consola"); + +/** + * describe-inputs: list a reconciliation's custom inputs with their declared + * defaults, stored values, and live EFFECTIVE values — but only from sources we + * can be CERTAIN about (no re-render, which can't faithfully reproduce live + * company state). Effective value is filled from, in order: + * 1. the stored custom (an explicit override), else + * 2. the live result where the template directly exposes the input + * (`{% result 'tag' custom.ns.key %}`), else + * 3. a literal default (`default:0`, `default:"x"`), else + * 4. flagged unavailable (only resolvable in the rendered UI). + */ + +// Combine main liquid + every text part into one string for parsing. +function combineLiquid(template) { + const parts = [template.text || ""]; + const tp = template.text_parts; + if (Array.isArray(tp)) { + parts.push(...tp.map((p) => (p && p.content) || "")); + } else if (tp && typeof tp === "object") { + parts.push(...Object.values(tp).map((c) => (typeof c === "string" ? c : (c && c.content) || ""))); + } + return parts.join("\n"); +} + +// Parse `{% input custom.ns.key as:type default:expr ... %}` declarations. +function parseInputs(liquid) { + const inputRe = /\{%-?\s*input\s+(custom\.[a-zA-Z0-9_.]+)([^%]*?)-?%\}/g; + const byPath = new Map(); + let match; + while ((match = inputRe.exec(liquid)) !== null) { + const path = match[1]; + const opts = match[2] || ""; + const typeMatch = opts.match(/\bas:([a-zA-Z0-9_]+)/); + // default value runs until the next option keyword or the end of the tag + const defaultMatch = opts.match(/\bdefault:(.+?)(?:\s+\b(?:as|placeholder|precision|width|required|html|size|maximum|minimum|on|off):|\s*$)/); + const type = typeMatch ? typeMatch[1] : "text"; + const def = defaultMatch ? defaultMatch[1].trim() : null; + const segments = path.split("."); + const namespace = segments[1]; + const key = segments.slice(2).join("."); + const existing = byPath.get(path); + if (!existing || (def && !existing.default)) { + byPath.set(path, { path, namespace, key, type, default: def }); + } + } + return Array.from(byPath.values()); +} + +// Parse `{% result 'tag' custom.ns.key %}` echoes → Map(customPath -> resultTag). +function parseResultEchoes(liquid) { + const resultRe = /\{%-?\s*result\s+['"]([a-zA-Z0-9_]+)['"]\s+(custom\.[a-zA-Z0-9_.]+)\s*-?%\}/g; + const byCustomPath = new Map(); + let match; + while ((match = resultRe.exec(liquid)) !== null) { + if (!byCustomPath.has(match[2])) { + byCustomPath.set(match[2], match[1]); + } + } + return byCustomPath; +} + +// Build { "namespace.key": value } from the API custom array. +function storedMapFromCustom(customArray) { + const map = {}; + for (const custom of customArray || []) { + if (!custom || custom.namespace == null || custom.key == null) continue; + map[`${custom.namespace}.${custom.key}`] = custom.value; + } + return map; +} + +// If a default expression is a literal, return its value; otherwise undefined. +function literalValue(def) { + if (def == null) return undefined; + const trimmed = def.trim(); + if (/^-?\d+(\.\d+)?$/.test(trimmed)) return Number(trimmed); + const quoted = trimmed.match(/^["'](.*)["']$/); + if (quoted) return quoted[1]; + if (trimmed === "true" || trimmed === "false") return trimmed === "true"; + return undefined; +} + +async function describeInputs(url) { + const parameters = Utils.extractURL(url); + if (parameters.templateType !== "reconciliationText") { + consola.error("describe-inputs currently supports reconciliation templates only."); + return null; + } + + const detailsResponse = await SF.readReconciliationTextDetails("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); + const handle = detailsResponse?.data?.handle; + if (!handle) { + consola.error("Could not resolve the reconciliation handle from the URL."); + return null; + } + + const template = ReconciliationText.read(handle); + if (!template) { + consola.error(`Template "${handle}" was not found locally — run this command from your templates repo.`); + return null; + } + + const liquid = combineLiquid(template); + const inputs = parseInputs(liquid); + const echoes = parseResultEchoes(liquid); + + const customResponse = await SF.getReconciliationCustom("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); + const resultsResponse = await SF.getReconciliationResults("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); + const stored = storedMapFromCustom(customResponse?.data); + const results = resultsResponse?.data || {}; + + const rows = inputs.map((input) => { + const nsKey = `${input.namespace}.${input.key}`; + const hasStored = Object.hasOwn(stored, nsKey); + let effective = null; + let effectiveSource = null; + + if (hasStored && stored[nsKey] != null) { + effective = stored[nsKey]; + effectiveSource = "stored"; + } else { + const tag = echoes.get(input.path); + const literal = literalValue(input.default); + if (tag && Object.hasOwn(results, tag)) { + effective = results[tag]; + effectiveSource = `result:${tag}`; + } else if (literal !== undefined) { + effective = literal; + effectiveSource = "literal-default"; + } else { + effective = null; + effectiveSource = "unavailable: not exposed as a result (rendered-UI only)"; + } + } + + return { + input: input.path, + type: input.type, + stored: hasStored ? stored[nsKey] : null, + default: input.default, + effective, + effectiveSource, + }; + }); + + return { handle, reconciliationId: parameters.reconciliationId, inputs: rows, results }; +} + +module.exports = { describeInputs, parseInputs, parseResultEchoes, combineLiquid, storedMapFromCustom, literalValue }; diff --git a/tests/lib/inputDescriber.test.js b/tests/lib/inputDescriber.test.js new file mode 100644 index 00000000..e67efd1d --- /dev/null +++ b/tests/lib/inputDescriber.test.js @@ -0,0 +1,96 @@ +jest.mock("../../lib/api/sfApi"); +jest.mock("../../lib/utils/liquidTestUtils"); +jest.mock("../../lib/templates/reconciliationText"); +jest.mock("consola"); + +const SF = require("../../lib/api/sfApi"); +const Utils = require("../../lib/utils/liquidTestUtils"); +const { ReconciliationText } = require("../../lib/templates/reconciliationText"); +const { describeInputs, parseInputs, parseResultEchoes, storedMapFromCustom, literalValue } = require("../../lib/inputDescriber"); + +describe("inputDescriber pure helpers", () => { + describe("parseInputs", () => { + it("parses path, type and default expression", () => { + const liquid = "{% input custom.amount.withdrawal_1 as:currency default:some_var placeholder:0 %}\n{% input custom.note.text as:text %}"; + const inputs = parseInputs(liquid); + expect(inputs).toEqual([ + { path: "custom.amount.withdrawal_1", namespace: "amount", key: "withdrawal_1", type: "currency", default: "some_var" }, + { path: "custom.note.text", namespace: "note", key: "text", type: "text", default: null }, + ]); + }); + + it("dedupes by path, preferring the declaration that carries a default", () => { + const liquid = "{% input custom.a.b as:currency %}\n{% input custom.a.b as:currency default:zero %}"; + const inputs = parseInputs(liquid); + expect(inputs).toHaveLength(1); + expect(inputs[0].default).toBe("zero"); + }); + }); + + describe("parseResultEchoes", () => { + it("maps a custom path to the result tag that echoes it", () => { + const echoes = parseResultEchoes("{% result 'my_tag' custom.a.b %}"); + expect(echoes.get("custom.a.b")).toBe("my_tag"); + }); + }); + + describe("literalValue", () => { + it("resolves numeric, string and boolean literals, else undefined", () => { + expect(literalValue("0")).toBe(0); + expect(literalValue('"hi"')).toBe("hi"); + expect(literalValue("true")).toBe(true); + expect(literalValue("some_var")).toBeUndefined(); + expect(literalValue(null)).toBeUndefined(); + }); + }); + + describe("storedMapFromCustom", () => { + it("builds a namespace.key map", () => { + expect(storedMapFromCustom([{ namespace: "a", key: "b", value: 1 }])).toEqual({ "a.b": 1 }); + }); + }); +}); + +describe("describeInputs", () => { + beforeEach(() => { + jest.clearAllMocks(); + Utils.extractURL.mockReturnValue({ + templateType: "reconciliationText", + firmId: "96", + companyId: "100", + ledgerId: "200", + reconciliationId: "300", + }); + SF.readReconciliationTextDetails.mockResolvedValue({ data: { handle: "my_handle" } }); + }); + + it("fills effective values from stored, echoed result, and literal default; flags the rest", async () => { + ReconciliationText.read.mockReturnValue({ + text: + "{% input custom.over.ride as:currency %}\n" + + "{% input custom.echo.field as:currency default:some_var %}{% result 'echo_tag' custom.echo.field %}\n" + + "{% input custom.lit.field as:integer default:0 %}\n" + + "{% input custom.gap.field as:currency default:computed_var %}", + text_parts: [], + }); + SF.getReconciliationCustom.mockResolvedValue({ data: [{ namespace: "over", key: "ride", value: 999 }] }); + SF.getReconciliationResults.mockResolvedValue({ data: { echo_tag: "42.0" } }); + + const out = await describeInputs("https://live.getsilverfin.com/f/96/100/..."); + + expect(out.handle).toBe("my_handle"); + const byInput = Object.fromEntries(out.inputs.map((r) => [r.input, r])); + expect(byInput["custom.over.ride"]).toMatchObject({ effective: 999, effectiveSource: "stored" }); + expect(byInput["custom.echo.field"]).toMatchObject({ effective: "42.0", effectiveSource: "result:echo_tag" }); + expect(byInput["custom.lit.field"]).toMatchObject({ effective: 0, effectiveSource: "literal-default" }); + expect(byInput["custom.gap.field"].effective).toBeNull(); + expect(byInput["custom.gap.field"].effectiveSource).toMatch(/unavailable/); + expect(out.results).toEqual({ echo_tag: "42.0" }); + }); + + it("returns null for non-reconciliation templates", async () => { + Utils.extractURL.mockReturnValue({ templateType: "accountTemplate" }); + const out = await describeInputs("https://live.getsilverfin.com/f/96/100/..."); + expect(out).toBeNull(); + }); +}); From f25ea818276ddddb4b6d7c69b5dc2b3bf04f806a Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Tue, 30 Jun 2026 15:26:51 +0200 Subject: [PATCH 04/17] Add manifest command (static data scope of a template) New manifest -h / -u builds a static data manifest by scanning a reconciliation's Liquid (main + text_parts + shared parts, recursively): own customs, cross-template results/customs, period drop + prior-period depth, company drop, accounts and shared parts. First step of the deep-capture + self-validating render pipeline. Adds lib/templateManifest.js + tests. --- CHANGELOG.md | 2 + bin/cli.js | 35 ++++++++ lib/templateManifest.js | 136 +++++++++++++++++++++++++++++ tests/lib/templateManifest.test.js | 68 +++++++++++++++ 4 files changed, 241 insertions(+) create mode 100644 lib/templateManifest.js create mode 100644 tests/lib/templateManifest.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 18680dc1..a249b7f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a liv Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). +Added `manifest` command. It builds a static data manifest (scope) for a reconciliation template by scanning its Liquid (main, text_parts and shared parts, recursively): own customs, cross-template results/customs, period drop and prior-period depth, company drop, accounts and shared parts. Usage: `silverfin manifest -h ` or `-u ` (run from your templates repo). It is the first step of a deep-capture + self-validating render pipeline for resolving effective default values without a browser. + ## [1.56.1] (08/06/2026) Increase the waiting time for the test runs to avoid timeout errors. diff --git a/bin/cli.js b/bin/cli.js index 796d1d0f..348e5017 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -6,6 +6,7 @@ const liquidTestRunner = require("../lib/liquidTestRunner"); const resultsReader = require("../lib/resultsReader"); const dataCapture = require("../lib/dataCapture"); const inputDescriber = require("../lib/inputDescriber"); +const templateManifest = require("../lib/templateManifest"); const { ExportFileInstanceGenerator } = require("../lib/exportFileInstanceGenerator"); const stats = require("../lib/cli/stats"); const { Command, Option } = require("commander"); @@ -600,6 +601,40 @@ program } }); +// MANIFEST — static data scope of a template (drives deep-capture + self-validating render) +program + .command("manifest") + .description("Build a static data manifest (scope) for a reconciliation template by scanning its Liquid (main + text_parts + shared parts): own customs, cross-template results/customs, period drop + prior-period depth, company drop, accounts, shared parts. Run from your templates repo") + .option("-h, --handle ", "Reconciliation handle (local, no API call)") + .option("-u, --url ", "Silverfin URL (resolves the handle via the API instead of --handle)") + .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") + .action(async (options) => { + let handle = options.handle; + if (!handle && options.url) { + const parameters = liquidTestUtils.extractURL(options.url); + const details = await SF.readReconciliationTextDetails("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); + handle = details?.data?.handle; + } + if (!handle) { + consola.error("Provide --handle or --url ."); + process.exitCode = 1; + return; + } + const data = await templateManifest.buildManifest(handle); + if (!data) { + process.exitCode = 1; + return; + } + const json = JSON.stringify(data, null, 2); + if (options.output) { + const fs = require("fs"); + fs.writeFileSync(options.output, json); + consola.success(`Wrote manifest to ${options.output}`); + } else { + console.log(json); + } + }); + // Update Text Properties from Liquid Test data program .command("update-text-properties") diff --git a/lib/templateManifest.js b/lib/templateManifest.js new file mode 100644 index 00000000..459d97b3 --- /dev/null +++ b/lib/templateManifest.js @@ -0,0 +1,136 @@ +const Utils = require("./utils/liquidTestUtils"); +const { ReconciliationText } = require("./templates/reconciliationText"); +const { SharedPart } = require("./templates/sharedPart"); +const fsUtils = require("./utils/fsUtils"); +const { consola } = require("consola"); + +/** + * Build a static DATA MANIFEST (scope) for a reconciliation template by scanning + * its Liquid (main + text_parts + every shared part, recursively). + * + * The manifest describes the SCOPE of data the template reads — not a guaranteed + * exhaustive key list (runtime-built references can't be resolved statically). + * It is meant to drive a "deep capture": fetch everything within this scope, then + * self-validate a render against the live results. + */ + +function uniq(array) { + return [...new Set(array)]; +} + +// Concatenate main + text_parts + shared-part liquid into one string for scanning. +function combinedText(objects) { + const parts = []; + for (const object of objects) { + if (!object) continue; + if (object.text) parts.push(object.text); + if (Array.isArray(object.text_parts)) { + for (const part of object.text_parts) parts.push((part && part.content) || ""); + } + } + return parts.join("\n"); +} + +// Recursively resolve the shared parts a template includes (and their nested ones). +async function gatherSharedParts(template, handle) { + const objects = []; + const seen = new Set(); + const missing = []; + const queue = Utils.lookForSharedPartsInLiquid(template, handle) || []; + while (queue.length) { + const name = queue.shift(); + if (seen.has(name)) continue; + seen.add(name); + let sharedPart = null; + try { + sharedPart = await SharedPart.read(name); + } catch (error) { + sharedPart = null; + } + if (!sharedPart || !sharedPart.text) { + missing.push(name); + continue; + } + objects.push(sharedPart); + const nested = Utils.lookForSharedPartsInLiquid(sharedPart, name) || []; + for (const nestedName of nested) { + if (!seen.has(nestedName)) queue.push(nestedName); + } + } + return { objects, names: objects.map((object) => object.name), missing }; +} + +async function buildManifest(handle) { + const template = ReconciliationText.read(handle); + if (!template) { + consola.error(`Template "${handle}" was not found locally — run this from your templates repo.`); + return null; + } + + const { objects: sharedObjects, names: sharedParts, missing } = await gatherSharedParts(template, handle); + + // Cross-template results/customs via the existing (assign-aware) scanners, + // across the template AND every shared part. + let results = Utils.searchForResultsFromDependenciesInLiquid(template, handle); + let customs = Utils.searchForCustomsFromDependenciesInLiquid(template, handle); + for (const sharedPart of sharedObjects) { + results = Utils.searchForResultsFromDependenciesInLiquid(sharedPart, sharedPart.name, results); + customs = Utils.searchForCustomsFromDependenciesInLiquid(sharedPart, sharedPart.name, customs); + } + // A template's own results are not a cross-template dependency to fetch separately. + delete results[handle]; + delete customs[handle]; + const crossTemplate = {}; + for (const dependencyHandle of uniq([...Object.keys(results), ...Object.keys(customs)])) { + crossTemplate[dependencyHandle] = { + results: results[dependencyHandle] || [], + customs: customs[dependencyHandle] || [], + }; + } + + const text = combinedText([template, ...sharedObjects]); + + // Own custom drop reads (standalone `custom.ns.key`, not `.custom.ns.key`). + const ownCustoms = uniq([...text.matchAll(/(?:^|[^.\w])custom\.([a-z0-9_]+)\.([a-z0-9_]+)/g)].map((m) => `custom.${m[1]}.${m[2]}`)); + + // Period drop + prior-period depth (period.minus_Ny → how many prior years are read). + let priorPeriodDepth = 0; + for (const match of text.matchAll(/period\.minus_(\d+)y/g)) { + priorPeriodDepth = Math.max(priorPeriodDepth, Number(match[1])); + } + const periodDrop = uniq([...text.matchAll(/\bperiod\.([a-z_][a-z0-9_]*)/g)].map((m) => m[1])).filter( + (field) => !["reconciliations", "accounts", "custom"].includes(field) && !/^minus_\d+y$/.test(field) + ); + + // Company drop (standard + custom). + const companyCustom = uniq([...text.matchAll(/\bcompany\.custom\.([a-z0-9_]+)\.([a-z0-9_]+)/g)].map((m) => `${m[1]}.${m[2]}`)); + const companyStandard = uniq([...text.matchAll(/\bcompany\.([a-z_][a-z0-9_]*)/g)].map((m) => m[1])).filter((field) => field !== "custom"); + + // Accounts referenced directly (#number) + the configured account range. + const accounts = uniq([...text.matchAll(/#(\d{3,})/g)].map((m) => `#${m[1]}`)); + let accountRange = null; + try { + const config = fsUtils.readConfig("reconciliationText", handle); + accountRange = config && config.account_range ? config.account_range : null; + } catch (error) { + accountRange = null; + } + + const rollforward = /rollforward/i.test(text); + + return { + handle, + ownCustoms, + crossTemplate, + periodDrop, + priorPeriodDepth, + companyDrop: { standard: companyStandard, custom: companyCustom }, + accounts, + accountRange, + sharedParts, + missingSharedParts: missing, + rollforward, + }; +} + +module.exports = { buildManifest, combinedText }; diff --git a/tests/lib/templateManifest.test.js b/tests/lib/templateManifest.test.js new file mode 100644 index 00000000..f2217db3 --- /dev/null +++ b/tests/lib/templateManifest.test.js @@ -0,0 +1,68 @@ +jest.mock("../../lib/templates/reconciliationText"); +jest.mock("../../lib/templates/sharedPart"); +jest.mock("../../lib/utils/fsUtils"); +jest.mock("consola"); +// NOTE: liquidTestUtils is intentionally NOT mocked — we want the real scanners. + +const { ReconciliationText } = require("../../lib/templates/reconciliationText"); +const { SharedPart } = require("../../lib/templates/sharedPart"); +const fsUtils = require("../../lib/utils/fsUtils"); +const { buildManifest } = require("../../lib/templateManifest"); + +describe("templateManifest.buildManifest", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("scans the template + recurses shared parts to build the data scope", async () => { + ReconciliationText.read.mockReturnValue({ + text: + "{% input custom.a.b as:currency default:0 %} " + + "period.minus_3y period.fiscal_year " + + "period.reconciliations.dep.results.tag1 " + + "company.custom.ns.key company.street #280000 rollforward " + + "{% include 'shared/sp1' %}", + text_parts: [], + }); + SharedPart.read.mockImplementation((name) => { + if (name === "sp1") return { name: "sp1", text: "period.reconciliations.dep2.results.tag2 {% include 'shared/sp2' %}" }; + if (name === "sp2") return { name: "sp2", text: "custom.c.d" }; + return null; + }); + fsUtils.readConfig.mockReturnValue({ account_range: "280,282" }); + + const m = await buildManifest("my_handle"); + + expect(m.handle).toBe("my_handle"); + expect(m.priorPeriodDepth).toBe(3); + expect(m.rollforward).toBe(true); + expect(m.ownCustoms).toEqual(expect.arrayContaining(["custom.a.b", "custom.c.d"])); + // cross-template deps come from the template AND the recursed shared parts + expect(Object.keys(m.crossTemplate).sort()).toEqual(["dep", "dep2"]); + expect(m.crossTemplate.dep.results).toEqual(["tag1"]); + expect(m.crossTemplate.dep2.results).toEqual(["tag2"]); + expect(m.periodDrop).toEqual(expect.arrayContaining(["fiscal_year"])); + expect(m.periodDrop).not.toContain("minus_3y"); + expect(m.companyDrop.custom).toContain("ns.key"); + expect(m.companyDrop.standard).toContain("street"); + expect(m.accounts).toContain("#280000"); + expect(m.accountRange).toBe("280,282"); + expect(m.sharedParts.sort()).toEqual(["sp1", "sp2"]); + expect(m.missingSharedParts).toEqual([]); + }); + + it("returns null when the template is not found locally", async () => { + ReconciliationText.read.mockReturnValue(false); + const m = await buildManifest("missing"); + expect(m).toBeNull(); + }); + + it("records shared parts that cannot be read", async () => { + ReconciliationText.read.mockReturnValue({ text: "{% include 'shared/gone' %}", text_parts: [] }); + SharedPart.read.mockReturnValue(null); + fsUtils.readConfig.mockReturnValue({}); + const m = await buildManifest("h"); + expect(m.missingSharedParts).toEqual(["gone"]); + expect(m.sharedParts).toEqual([]); + }); +}); From 6bc0c406d0403cbecff41514e7c0a2cb301f382c Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Tue, 30 Jun 2026 16:34:52 +0200 Subject: [PATCH 05/17] Delegate the data-scope analysis to silverfin-ls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the regex-based lib/templateManifest.js with lib/dataScope.js, which shells out to 'silverfin-ls data-scope ' (the maintained tree-sitter language server) and returns its JSON scope. This stops duplicating STL parsing in the CLI and picks up silverfin-ls's AST-accurate, alias-aware analysis — including digit-prefixed handles (2018_*, 275_*) the regex couldn't follow. The 'manifest' command now delegates to it; set SILVERFIN_LS_CMD to point at a specific silverfin-ls binary/build. Removes the regex module + test. --- CHANGELOG.md | 2 +- bin/cli.js | 8 +- lib/dataScope.js | 51 +++++++++++ lib/templateManifest.js | 136 ----------------------------- tests/lib/templateManifest.test.js | 68 --------------- 5 files changed, 56 insertions(+), 209 deletions(-) create mode 100644 lib/dataScope.js delete mode 100644 lib/templateManifest.js delete mode 100644 tests/lib/templateManifest.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index a249b7f2..15c3745b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a liv Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). -Added `manifest` command. It builds a static data manifest (scope) for a reconciliation template by scanning its Liquid (main, text_parts and shared parts, recursively): own customs, cross-template results/customs, period drop and prior-period depth, company drop, accounts and shared parts. Usage: `silverfin manifest -h ` or `-u ` (run from your templates repo). It is the first step of a deep-capture + self-validating render pipeline for resolving effective default values without a browser. +Added `manifest` command. It prints a reconciliation template's static data scope: own customs, cross-template results/customs, period drop and prior-period depth, company drop, accounts, result echoes and involved files. Usage: `silverfin manifest -h ` or `-u ` (run from your templates repo). The STL analysis is delegated to silverfin-ls (the maintained tree-sitter language server) rather than duplicated here; set `SILVERFIN_LS_CMD` to point at a specific silverfin-ls binary/build. It is the first step of a deep-capture + default-resolution pipeline for resolving effective default values without a browser. ## [1.56.1] (08/06/2026) Increase the waiting time for the test runs to avoid timeout errors. diff --git a/bin/cli.js b/bin/cli.js index 348e5017..a984d532 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -6,7 +6,7 @@ const liquidTestRunner = require("../lib/liquidTestRunner"); const resultsReader = require("../lib/resultsReader"); const dataCapture = require("../lib/dataCapture"); const inputDescriber = require("../lib/inputDescriber"); -const templateManifest = require("../lib/templateManifest"); +const dataScope = require("../lib/dataScope"); const { ExportFileInstanceGenerator } = require("../lib/exportFileInstanceGenerator"); const stats = require("../lib/cli/stats"); const { Command, Option } = require("commander"); @@ -601,10 +601,10 @@ program } }); -// MANIFEST — static data scope of a template (drives deep-capture + self-validating render) +// MANIFEST — static data scope of a template (drives deep-capture + default resolution) program .command("manifest") - .description("Build a static data manifest (scope) for a reconciliation template by scanning its Liquid (main + text_parts + shared parts): own customs, cross-template results/customs, period drop + prior-period depth, company drop, accounts, shared parts. Run from your templates repo") + .description("Print a template's static data scope: own customs, cross-template results/customs, period drop + prior-period depth, company drop, accounts, result echoes, involved files. Delegates the analysis to silverfin-ls (set SILVERFIN_LS_CMD to override the binary). Run from your templates repo") .option("-h, --handle ", "Reconciliation handle (local, no API call)") .option("-u, --url ", "Silverfin URL (resolves the handle via the API instead of --handle)") .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") @@ -620,7 +620,7 @@ program process.exitCode = 1; return; } - const data = await templateManifest.buildManifest(handle); + const data = dataScope.getDataScope(handle); if (!data) { process.exitCode = 1; return; diff --git a/lib/dataScope.js b/lib/dataScope.js new file mode 100644 index 00000000..eafe82be --- /dev/null +++ b/lib/dataScope.js @@ -0,0 +1,51 @@ +const { execFileSync } = require("child_process"); +const path = require("path"); +const { consola } = require("consola"); + +/** + * Static data-scope analysis is delegated to silverfin-ls (the maintained + * tree-sitter language server), so we don't duplicate STL parsing here. + * + * `silverfin-ls data-scope ` prints the template's data scope as + * JSON: ownCustoms, crossTemplate {results, customs}, periodDrop, + * priorPeriodDepth, companyDrop, accounts, resultEchoes, involvedFiles. + * + * The binary is `silverfin-ls` by default; set SILVERFIN_LS_CMD to point at a + * local/dev build (e.g. "/path/to/node /path/to/silverfin-ls/out/index.js"). + */ + +function resolveMainPath(handle) { + return path.resolve(process.cwd(), "reconciliation_texts", handle, "main.liquid"); +} + +function lsCommand() { + // SILVERFIN_LS_CMD may be a single binary or "node /path/out/index.js". + const raw = (process.env.SILVERFIN_LS_CMD || "silverfin-ls").trim(); + const parts = raw.split(/\s+/); + return { bin: parts[0], prefixArgs: parts.slice(1) }; +} + +/** + * @param {String} handle reconciliation handle (its main.liquid is read from cwd) + * @returns {Object|null} the data scope, or null if silverfin-ls is unavailable + */ +function getDataScope(handle) { + const main = resolveMainPath(handle); + const { bin, prefixArgs } = lsCommand(); + try { + const stdout = execFileSync(bin, [...prefixArgs, "data-scope", main], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + stdio: ["ignore", "pipe", "ignore"], + }); + return JSON.parse(stdout); + } catch (error) { + consola.error( + `Could not get the data scope from silverfin-ls. Ensure silverfin-ls is installed and supports "data-scope" ` + + `(set SILVERFIN_LS_CMD to point at a local build). ${String(error.message).split("\n")[0]}` + ); + return null; + } +} + +module.exports = { getDataScope, resolveMainPath }; diff --git a/lib/templateManifest.js b/lib/templateManifest.js deleted file mode 100644 index 459d97b3..00000000 --- a/lib/templateManifest.js +++ /dev/null @@ -1,136 +0,0 @@ -const Utils = require("./utils/liquidTestUtils"); -const { ReconciliationText } = require("./templates/reconciliationText"); -const { SharedPart } = require("./templates/sharedPart"); -const fsUtils = require("./utils/fsUtils"); -const { consola } = require("consola"); - -/** - * Build a static DATA MANIFEST (scope) for a reconciliation template by scanning - * its Liquid (main + text_parts + every shared part, recursively). - * - * The manifest describes the SCOPE of data the template reads — not a guaranteed - * exhaustive key list (runtime-built references can't be resolved statically). - * It is meant to drive a "deep capture": fetch everything within this scope, then - * self-validate a render against the live results. - */ - -function uniq(array) { - return [...new Set(array)]; -} - -// Concatenate main + text_parts + shared-part liquid into one string for scanning. -function combinedText(objects) { - const parts = []; - for (const object of objects) { - if (!object) continue; - if (object.text) parts.push(object.text); - if (Array.isArray(object.text_parts)) { - for (const part of object.text_parts) parts.push((part && part.content) || ""); - } - } - return parts.join("\n"); -} - -// Recursively resolve the shared parts a template includes (and their nested ones). -async function gatherSharedParts(template, handle) { - const objects = []; - const seen = new Set(); - const missing = []; - const queue = Utils.lookForSharedPartsInLiquid(template, handle) || []; - while (queue.length) { - const name = queue.shift(); - if (seen.has(name)) continue; - seen.add(name); - let sharedPart = null; - try { - sharedPart = await SharedPart.read(name); - } catch (error) { - sharedPart = null; - } - if (!sharedPart || !sharedPart.text) { - missing.push(name); - continue; - } - objects.push(sharedPart); - const nested = Utils.lookForSharedPartsInLiquid(sharedPart, name) || []; - for (const nestedName of nested) { - if (!seen.has(nestedName)) queue.push(nestedName); - } - } - return { objects, names: objects.map((object) => object.name), missing }; -} - -async function buildManifest(handle) { - const template = ReconciliationText.read(handle); - if (!template) { - consola.error(`Template "${handle}" was not found locally — run this from your templates repo.`); - return null; - } - - const { objects: sharedObjects, names: sharedParts, missing } = await gatherSharedParts(template, handle); - - // Cross-template results/customs via the existing (assign-aware) scanners, - // across the template AND every shared part. - let results = Utils.searchForResultsFromDependenciesInLiquid(template, handle); - let customs = Utils.searchForCustomsFromDependenciesInLiquid(template, handle); - for (const sharedPart of sharedObjects) { - results = Utils.searchForResultsFromDependenciesInLiquid(sharedPart, sharedPart.name, results); - customs = Utils.searchForCustomsFromDependenciesInLiquid(sharedPart, sharedPart.name, customs); - } - // A template's own results are not a cross-template dependency to fetch separately. - delete results[handle]; - delete customs[handle]; - const crossTemplate = {}; - for (const dependencyHandle of uniq([...Object.keys(results), ...Object.keys(customs)])) { - crossTemplate[dependencyHandle] = { - results: results[dependencyHandle] || [], - customs: customs[dependencyHandle] || [], - }; - } - - const text = combinedText([template, ...sharedObjects]); - - // Own custom drop reads (standalone `custom.ns.key`, not `.custom.ns.key`). - const ownCustoms = uniq([...text.matchAll(/(?:^|[^.\w])custom\.([a-z0-9_]+)\.([a-z0-9_]+)/g)].map((m) => `custom.${m[1]}.${m[2]}`)); - - // Period drop + prior-period depth (period.minus_Ny → how many prior years are read). - let priorPeriodDepth = 0; - for (const match of text.matchAll(/period\.minus_(\d+)y/g)) { - priorPeriodDepth = Math.max(priorPeriodDepth, Number(match[1])); - } - const periodDrop = uniq([...text.matchAll(/\bperiod\.([a-z_][a-z0-9_]*)/g)].map((m) => m[1])).filter( - (field) => !["reconciliations", "accounts", "custom"].includes(field) && !/^minus_\d+y$/.test(field) - ); - - // Company drop (standard + custom). - const companyCustom = uniq([...text.matchAll(/\bcompany\.custom\.([a-z0-9_]+)\.([a-z0-9_]+)/g)].map((m) => `${m[1]}.${m[2]}`)); - const companyStandard = uniq([...text.matchAll(/\bcompany\.([a-z_][a-z0-9_]*)/g)].map((m) => m[1])).filter((field) => field !== "custom"); - - // Accounts referenced directly (#number) + the configured account range. - const accounts = uniq([...text.matchAll(/#(\d{3,})/g)].map((m) => `#${m[1]}`)); - let accountRange = null; - try { - const config = fsUtils.readConfig("reconciliationText", handle); - accountRange = config && config.account_range ? config.account_range : null; - } catch (error) { - accountRange = null; - } - - const rollforward = /rollforward/i.test(text); - - return { - handle, - ownCustoms, - crossTemplate, - periodDrop, - priorPeriodDepth, - companyDrop: { standard: companyStandard, custom: companyCustom }, - accounts, - accountRange, - sharedParts, - missingSharedParts: missing, - rollforward, - }; -} - -module.exports = { buildManifest, combinedText }; diff --git a/tests/lib/templateManifest.test.js b/tests/lib/templateManifest.test.js deleted file mode 100644 index f2217db3..00000000 --- a/tests/lib/templateManifest.test.js +++ /dev/null @@ -1,68 +0,0 @@ -jest.mock("../../lib/templates/reconciliationText"); -jest.mock("../../lib/templates/sharedPart"); -jest.mock("../../lib/utils/fsUtils"); -jest.mock("consola"); -// NOTE: liquidTestUtils is intentionally NOT mocked — we want the real scanners. - -const { ReconciliationText } = require("../../lib/templates/reconciliationText"); -const { SharedPart } = require("../../lib/templates/sharedPart"); -const fsUtils = require("../../lib/utils/fsUtils"); -const { buildManifest } = require("../../lib/templateManifest"); - -describe("templateManifest.buildManifest", () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it("scans the template + recurses shared parts to build the data scope", async () => { - ReconciliationText.read.mockReturnValue({ - text: - "{% input custom.a.b as:currency default:0 %} " + - "period.minus_3y period.fiscal_year " + - "period.reconciliations.dep.results.tag1 " + - "company.custom.ns.key company.street #280000 rollforward " + - "{% include 'shared/sp1' %}", - text_parts: [], - }); - SharedPart.read.mockImplementation((name) => { - if (name === "sp1") return { name: "sp1", text: "period.reconciliations.dep2.results.tag2 {% include 'shared/sp2' %}" }; - if (name === "sp2") return { name: "sp2", text: "custom.c.d" }; - return null; - }); - fsUtils.readConfig.mockReturnValue({ account_range: "280,282" }); - - const m = await buildManifest("my_handle"); - - expect(m.handle).toBe("my_handle"); - expect(m.priorPeriodDepth).toBe(3); - expect(m.rollforward).toBe(true); - expect(m.ownCustoms).toEqual(expect.arrayContaining(["custom.a.b", "custom.c.d"])); - // cross-template deps come from the template AND the recursed shared parts - expect(Object.keys(m.crossTemplate).sort()).toEqual(["dep", "dep2"]); - expect(m.crossTemplate.dep.results).toEqual(["tag1"]); - expect(m.crossTemplate.dep2.results).toEqual(["tag2"]); - expect(m.periodDrop).toEqual(expect.arrayContaining(["fiscal_year"])); - expect(m.periodDrop).not.toContain("minus_3y"); - expect(m.companyDrop.custom).toContain("ns.key"); - expect(m.companyDrop.standard).toContain("street"); - expect(m.accounts).toContain("#280000"); - expect(m.accountRange).toBe("280,282"); - expect(m.sharedParts.sort()).toEqual(["sp1", "sp2"]); - expect(m.missingSharedParts).toEqual([]); - }); - - it("returns null when the template is not found locally", async () => { - ReconciliationText.read.mockReturnValue(false); - const m = await buildManifest("missing"); - expect(m).toBeNull(); - }); - - it("records shared parts that cannot be read", async () => { - ReconciliationText.read.mockReturnValue({ text: "{% include 'shared/gone' %}", text_parts: [] }); - SharedPart.read.mockReturnValue(null); - fsUtils.readConfig.mockReturnValue({}); - const m = await buildManifest("h"); - expect(m.missingSharedParts).toEqual(["gone"]); - expect(m.sharedParts).toEqual([]); - }); -}); From 1367211e530d877792150e2445e21b2861e97353 Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Tue, 30 Jun 2026 16:42:01 +0200 Subject: [PATCH 06/17] Add describe-inputs --resolve via targeted deep capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the effective value of inputs whose default is a DIRECT reference to data created elsewhere — a cross-template result/custom, a period/company custom, optionally a prior period (period.minus_Ny...) — by reading it straight from the live API. lib/defaultResolver.js parses each still-unavailable input's default, captures only the referenced handles + deepest referenced period (via lib/deepCapture.js, driven by the silverfin-ls scope), and fills the value with an effectiveSource of 'captured:'. Computed/conditionally-assigned default variables stay flagged (they need a render). deepCapture now exposes periodOrder so period.minus_Ny maps to the right captured period. Adds resolver unit tests. --- CHANGELOG.md | 2 +- bin/cli.js | 5 +- lib/deepCapture.js | 124 +++++++++++++++++++++++ lib/defaultResolver.js | 157 ++++++++++++++++++++++++++++++ lib/inputDescriber.js | 23 ++++- tests/lib/defaultResolver.test.js | 74 ++++++++++++++ 6 files changed, 380 insertions(+), 5 deletions(-) create mode 100644 lib/deepCapture.js create mode 100644 lib/defaultResolver.js create mode 100644 tests/lib/defaultResolver.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 15c3745b..110fe09e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Added `capture` command. It captures a live company file's data as JSON. By defa Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a live company file's custom data. `set-custom -u --namespace --key --value ` sets a single custom (value JSON-parsed when possible); `delete-custom` soft-deletes it (value null). Both support `--level company|period|reconciliation|account` (inferred from the URL by default), `--handle`/`--account` targeting, `--file` for batch, and `--yes` to skip the confirmation prompt. -Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). +Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). Pass `--resolve` to additionally fill inputs whose default is a direct reference to data created elsewhere (a cross-template result/custom, a period/company custom, optionally a prior period via `period.minus_Ny...`); these are read straight from a targeted deep capture of the live company file (only the referenced handles/periods are fetched). Defaults that are computed or conditionally assigned variables remain flagged — they need an actual render. Requires silverfin-ls (for the scope). Added `manifest` command. It prints a reconciliation template's static data scope: own customs, cross-template results/customs, period drop and prior-period depth, company drop, accounts, result echoes and involved files. Usage: `silverfin manifest -h ` or `-u ` (run from your templates repo). The STL analysis is delegated to silverfin-ls (the maintained tree-sitter language server) rather than duplicated here; set `SILVERFIN_LS_CMD` to point at a specific silverfin-ls binary/build. It is the first step of a deep-capture + default-resolution pipeline for resolving effective default values without a browser. diff --git a/bin/cli.js b/bin/cli.js index a984d532..284b8726 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -582,11 +582,12 @@ program // DESCRIBE INPUTS — list a reconciliation's custom inputs with declared defaults + live effective values program .command("describe-inputs") - .description("List a reconciliation's custom inputs with their declared defaults, stored values, and live effective values (certain sources only), plus the template's results. Run from your templates repo") + .description("List a reconciliation's custom inputs with their declared defaults, stored values, and live effective values (certain sources only), plus the template's results. Use --resolve to additionally fill inputs whose default is a direct reference to data created elsewhere (cross-template results/customs, period/company customs, prior periods) by reading it from a targeted deep capture. Run from your templates repo") .requiredOption("-u, --url ", "Specify the full Silverfin URL of the reconciliation in the company file (mandatory)") + .option("--resolve", "Resolve `unavailable` defaults that reference data created elsewhere, via a targeted deep capture of the live company file (extra API calls). Requires silverfin-ls", false) .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") .action(async (options) => { - const data = await inputDescriber.describeInputs(options.url); + const data = await inputDescriber.describeInputs(options.url, { resolve: options.resolve }); if (!data) { process.exitCode = 1; return; diff --git a/lib/deepCapture.js b/lib/deepCapture.js new file mode 100644 index 00000000..177024b8 --- /dev/null +++ b/lib/deepCapture.js @@ -0,0 +1,124 @@ +const SF = require("./api/sfApi"); +const Utils = require("./utils/liquidTestUtils"); +const { consola } = require("consola"); + +const PER_PAGE = 200; +const MAX_PAGES = 50; + +/** + * Manifest-driven DEEP capture: fetch the complete data scope a template needs so + * a render can faithfully reproduce the live state — current + N prior periods, + * the template + its cross-template dependencies' customs/results per period, + * period custom, and company drop. Serialised per company (1 in-flight call max). + * + * Returns the gathered data; renderResolver shapes it into the test fixture. + */ + +async function fetchAllPeriods(firmId, companyId) { + const items = []; + let page = 1; + while (page <= MAX_PAGES) { + const response = await SF.getPeriods(firmId, companyId, page); + const data = response?.data ?? []; + items.push(...data); + if (data.length < PER_PAGE) break; + page++; + } + return items; +} + +async function fetchAllWorkflowReconciliations(firmId, companyId, periodId, workflowId) { + const items = []; + let page = 1; + while (page <= MAX_PAGES) { + const response = await SF.getWorkflowInformation(firmId, companyId, periodId, workflowId, page); + const data = response?.data ?? []; + items.push(...data); + if (data.length < PER_PAGE) break; + page++; + } + return items; +} + +// handle -> reconciliation id for a given period (one pass over the workflows). +async function buildHandleMap(firmId, companyId, periodId) { + const map = {}; + const workflowsResponse = await SF.getWorkflows(firmId, companyId, periodId); + const workflows = workflowsResponse?.data ?? []; + for (const workflow of workflows) { + const reconciliations = await fetchAllWorkflowReconciliations(firmId, companyId, periodId, workflow.id); + for (const reconciliation of reconciliations) { + if (reconciliation.handle && !(reconciliation.handle in map)) { + map[reconciliation.handle] = reconciliation.id; + } + } + } + return map; +} + +async function buildDeepFixture(url, manifest, opts = {}) { + const parameters = Utils.extractURL(url); + const { firmId, companyId, ledgerId } = parameters; + const maxPrior = opts.maxPriorPeriods ?? manifest.priorPeriodDepth ?? 0; + + const periods = await fetchAllPeriods(firmId, companyId); + const currentIndex = periods.findIndex((period) => String(period.id) === String(ledgerId)); + if (currentIndex === -1) { + consola.error("Current period not found in the company periods."); + return null; + } + // Periods come newest-first; the current period plus the next `maxPrior` older ones. + const selected = periods.slice(currentIndex, currentIndex + maxPrior + 1); + const periodKey = (period) => + period.fiscal_year?.end_date ? String(period.fiscal_year.end_date) : String(period.id); + // Ordered newest-first: index 0 is the current period, index N is period.minus_Ny. + const periodOrder = selected.map(periodKey); + + const neededHandles = [manifest.handle, ...Object.keys(manifest.crossTemplate || {})]; + + const companyDrop = await SF.getCompanyDrop(firmId, companyId); + const companyCustom = await SF.getCompanyCustom(firmId, companyId); + + const data = { + company: { + drop: companyDrop?.data ?? null, + custom: Utils.processCustom(companyCustom?.data || []), + }, + periods: {}, + }; + + for (const period of selected) { + const periodId = period.id; + const key = periodKey(period); + const entry = { periodId, custom: Utils.processCustom((await SF.getAllPeriodCustom(firmId, companyId, periodId)) || []), reconciliations: {} }; + + const handleMap = await buildHandleMap(firmId, companyId, periodId); + for (const handle of neededHandles) { + const reconciliationId = handleMap[handle]; + if (!reconciliationId) continue; + const customResponse = await SF.getReconciliationCustom("firm", firmId, companyId, periodId, reconciliationId); + const resultsResponse = await SF.getReconciliationResults("firm", firmId, companyId, periodId, reconciliationId); + entry.reconciliations[handle] = { + id: reconciliationId, + custom: Utils.processCustom(customResponse?.data || []), + results: resultsResponse?.data ?? null, + }; + } + data.periods[key] = entry; + } + + return { + handle: manifest.handle, + firmId, + companyId, + currentPeriodId: ledgerId, + currentPeriodKey: periodOrder[0] ?? null, + periodOrder, + periodsCaptured: selected.length, + priorPeriodsRequested: maxPrior, + accountsNote: manifest.accounts?.length || manifest.accountRange ? "manifest references accounts; account values are not captured in this deep fixture yet" : null, + data, + }; +} + +module.exports = { buildDeepFixture }; diff --git a/lib/defaultResolver.js b/lib/defaultResolver.js new file mode 100644 index 00000000..2ca80b05 --- /dev/null +++ b/lib/defaultResolver.js @@ -0,0 +1,157 @@ +const { buildDeepFixture } = require("./deepCapture"); + +/** + * Resolve the effective value of inputs whose default is a DIRECT reference to + * live data created elsewhere — a cross-template result/custom, a period/company + * custom, optionally a prior period (`period.minus_Ny...`). These values exist in + * the live API (no browser, no re-render), so we read them straight from a + * targeted deep capture of only the referenced handles/periods. + * + * Computed defaults (filters, arithmetic, conditionals) are NOT resolved here and + * stay flagged — they would need an actual render. + */ + +// A resolvable reference is a single dotted path (no spaces / filters / operators). +const REF_RE = /^[a-zA-Z0-9_.]+$/; + +// Parse a default expression into a typed reference, or null if not a direct ref. +function parseReference(def) { + if (typeof def !== "string") return null; + const ref = def.trim(); + if (!REF_RE.test(ref)) return null; + + let m = ref.match(/^company\.custom\.([a-z0-9_]+)\.([a-z0-9_]+)$/i); + if (m) return { kind: "companyCustom", ns: m[1], key: m[2] }; + + // period[.minus_Ny]. + let periodN = 0; + let rest = null; + m = ref.match(/^period\.minus_(\d+)y\.(.+)$/i); + if (m) { + periodN = Number(m[1]); + rest = m[2]; + } else { + m = ref.match(/^period\.(.+)$/i); + if (m) rest = m[1]; + } + if (rest == null) return null; + + m = rest.match(/^reconciliations\.([a-z0-9_]+)\.results\.([a-z0-9_]+)$/i); + if (m) return { kind: "reconResult", periodN, handle: m[1], tag: m[2] }; + + m = rest.match(/^reconciliations\.([a-z0-9_]+)\.custom\.([a-z0-9_]+)\.([a-z0-9_]+)$/i); + if (m) return { kind: "reconCustom", periodN, handle: m[1], ns: m[2], key: m[3] }; + + m = rest.match(/^custom\.([a-z0-9_]+)\.([a-z0-9_]+)$/i); + if (m) return { kind: "periodCustom", periodN, ns: m[1], key: m[2] }; + + return null; +} + +// Look up a parsed reference in a deep fixture; undefined if not present. +function lookup(parsed, deep) { + if (parsed.kind === "companyCustom") { + return deep?.data?.company?.custom?.[`${parsed.ns}.${parsed.key}`]; + } + const periodKey = deep?.periodOrder?.[parsed.periodN]; + if (!periodKey) return undefined; + const entry = deep?.data?.periods?.[periodKey]; + if (!entry) return undefined; + if (parsed.kind === "reconResult") { + return entry.reconciliations?.[parsed.handle]?.results?.[parsed.tag]; + } + if (parsed.kind === "reconCustom") { + return entry.reconciliations?.[parsed.handle]?.custom?.[`${parsed.ns}.${parsed.key}`]; + } + if (parsed.kind === "periodCustom") { + return entry.custom?.[`${parsed.ns}.${parsed.key}`]; + } + return undefined; +} + +function sourceLabel(parsed, deep) { + if (parsed.kind === "companyCustom") { + return `captured:company.custom.${parsed.ns}.${parsed.key}`; + } + const at = `@${deep?.periodOrder?.[parsed.periodN]}`; + switch (parsed.kind) { + case "reconResult": + return `captured:period${at}.reconciliations.${parsed.handle}.results.${parsed.tag}`; + case "reconCustom": + return `captured:period${at}.reconciliations.${parsed.handle}.custom.${parsed.ns}.${parsed.key}`; + case "periodCustom": + return `captured:period${at}.custom.${parsed.ns}.${parsed.key}`; + default: + return "captured"; + } +} + +/** + * Resolve still-unavailable rows in place. Captures only the referenced handles + * and the deepest referenced prior period. + * @returns {Object} a resolution summary + */ +async function resolveDefaults(url, scope, rows) { + const unresolved = rows.filter( + (r) => typeof r.effectiveSource === "string" && r.effectiveSource.startsWith("unavailable") && r.default + ); + + const targets = []; + const neededHandles = new Set(); + let maxPriorPeriods = 0; + for (const row of unresolved) { + const parsed = parseReference(row.default); + if (!parsed) continue; + targets.push({ row, parsed }); + // The own handle is always captured by buildDeepFixture; only add others. + if (parsed.handle && parsed.handle !== scope.handle) neededHandles.add(parsed.handle); + if (parsed.periodN) maxPriorPeriods = Math.max(maxPriorPeriods, parsed.periodN); + } + + if (targets.length === 0) { + return { attempted: 0, resolved: 0, stillUnavailable: unresolved.length, capture: null }; + } + + const crossTemplate = {}; + for (const h of neededHandles) crossTemplate[h] = { results: [], customs: [] }; + const trimmedManifest = { + handle: scope.handle, + crossTemplate, + priorPeriodDepth: maxPriorPeriods, + accounts: [], + }; + + const deep = await buildDeepFixture(url, trimmedManifest, { maxPriorPeriods }); + if (!deep) { + return { + attempted: targets.length, + resolved: 0, + stillUnavailable: unresolved.length, + capture: null, + warning: "Deep capture failed; direct-reference defaults left unresolved.", + }; + } + + let resolved = 0; + for (const { row, parsed } of targets) { + const value = lookup(parsed, deep); + if (value !== undefined) { + row.effective = value; + row.effectiveSource = sourceLabel(parsed, deep); + resolved++; + } + } + + return { + attempted: targets.length, + resolved, + stillUnavailable: unresolved.length - resolved, + capture: { + periodsCaptured: deep.periodsCaptured, + periodOrder: deep.periodOrder, + handles: [scope.handle, ...neededHandles], + }, + }; +} + +module.exports = { resolveDefaults, parseReference, lookup }; diff --git a/lib/inputDescriber.js b/lib/inputDescriber.js index e6e9946f..c11172f8 100644 --- a/lib/inputDescriber.js +++ b/lib/inputDescriber.js @@ -85,7 +85,7 @@ function literalValue(def) { return undefined; } -async function describeInputs(url) { +async function describeInputs(url, opts = {}) { const parameters = Utils.extractURL(url); if (parameters.templateType !== "reconciliationText") { consola.error("describe-inputs currently supports reconciliation templates only."); @@ -148,7 +148,26 @@ async function describeInputs(url) { }; }); - return { handle, reconciliationId: parameters.reconciliationId, inputs: rows, results }; + const output = { handle, reconciliationId: parameters.reconciliationId, inputs: rows, results }; + + // --resolve: fill the `unavailable` rows whose default is a direct reference to + // data created elsewhere (cross-template result/custom, period/company custom), + // by reading it straight from a targeted deep capture of the live company file. + if (opts.resolve) { + const { getDataScope } = require("./dataScope"); + const { resolveDefaults } = require("./defaultResolver"); + const scope = getDataScope(handle); + if (!scope) { + output.resolution = { + resolved: 0, + error: "Could not get the data scope from silverfin-ls; defaults left unresolved.", + }; + } else { + output.resolution = await resolveDefaults(url, scope, rows); + } + } + + return output; } module.exports = { describeInputs, parseInputs, parseResultEchoes, combineLiquid, storedMapFromCustom, literalValue }; diff --git a/tests/lib/defaultResolver.test.js b/tests/lib/defaultResolver.test.js new file mode 100644 index 00000000..676d7ab4 --- /dev/null +++ b/tests/lib/defaultResolver.test.js @@ -0,0 +1,74 @@ +const { parseReference, lookup } = require("../../lib/defaultResolver"); + +describe("defaultResolver.parseReference", () => { + it("parses a current-period cross-template result", () => { + expect(parseReference("period.reconciliations.2018_tax_module.results.taxable_base")).toEqual({ + kind: "reconResult", + periodN: 0, + handle: "2018_tax_module", + tag: "taxable_base", + }); + }); + + it("parses a prior-period cross-template result", () => { + expect(parseReference("period.minus_3y.reconciliations.foo.results.bar")).toEqual({ + kind: "reconResult", + periodN: 3, + handle: "foo", + tag: "bar", + }); + }); + + it("parses a cross-template custom", () => { + expect(parseReference("period.reconciliations.h.custom.ns.k")).toEqual({ + kind: "reconCustom", + periodN: 0, + handle: "h", + ns: "ns", + key: "k", + }); + }); + + it("parses a company custom and a period custom", () => { + expect(parseReference("company.custom.general.x")).toEqual({ kind: "companyCustom", ns: "general", key: "x" }); + expect(parseReference("period.custom.ns.k")).toEqual({ kind: "periodCustom", periodN: 0, ns: "ns", key: "k" }); + }); + + it("rejects computed defaults and non-references", () => { + expect(parseReference("some_var | default: 0")).toBeNull(); + expect(parseReference("period.reconciliations.h.results.tag + 5")).toBeNull(); + expect(parseReference("1000")).toBeNull(); + expect(parseReference(null)).toBeNull(); + expect(parseReference("custom.ns.k")).toBeNull(); // own custom — handled as stored, not a cross-reference + }); +}); + +describe("defaultResolver.lookup", () => { + const deep = { + periodOrder: ["2023-12-31", "2022-12-31"], + data: { + company: { custom: { "general.x": 42 } }, + periods: { + "2023-12-31": { + custom: { "ns.k": 7 }, + reconciliations: { foo: { results: { bar: 100 }, custom: { "n.k": 9 } } }, + }, + "2022-12-31": { reconciliations: { foo: { results: { bar: 88 } } } }, + }, + }, + }; + + it("reads company / period / cross-template values, current and prior period", () => { + expect(lookup({ kind: "companyCustom", ns: "general", key: "x" }, deep)).toBe(42); + expect(lookup({ kind: "reconResult", periodN: 0, handle: "foo", tag: "bar" }, deep)).toBe(100); + expect(lookup({ kind: "reconResult", periodN: 1, handle: "foo", tag: "bar" }, deep)).toBe(88); + expect(lookup({ kind: "periodCustom", periodN: 0, ns: "ns", key: "k" }, deep)).toBe(7); + expect(lookup({ kind: "reconCustom", periodN: 0, handle: "foo", ns: "n", key: "k" }, deep)).toBe(9); + }); + + it("returns undefined for absent values or missing prior periods", () => { + expect(lookup({ kind: "reconResult", periodN: 5, handle: "foo", tag: "bar" }, deep)).toBeUndefined(); + expect(lookup({ kind: "reconResult", periodN: 0, handle: "missing", tag: "bar" }, deep)).toBeUndefined(); + expect(lookup({ kind: "companyCustom", ns: "general", key: "nope" }, deep)).toBeUndefined(); + }); +}); From 510b210d6110e7e8448422c6a6e6297459c3b4ef Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Tue, 30 Jun 2026 17:04:21 +0200 Subject: [PATCH 07/17] Add --dry-run to set-custom and delete-custom Both commands now accept --dry-run, which prints the exact target and the {namespace, key, value} properties that would be written (value:null for delete) as JSON and sends no request. Lets the write path be exercised safely on any firm. Matches the --dry-run already on update-text-properties. --- CHANGELOG.md | 2 +- bin/cli.js | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 110fe09e..b3a505bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Added `get-results` command. It fetches the computed results and custom data of Added `capture` command. It captures a live company file's data as JSON. By default it captures the template at the URL and its dependencies (scoped); `--full` captures company/period/reconciliation customs and results across all periods. Usage: `silverfin capture -u [--full]`. Supports `-o, --output `. -Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a live company file's custom data. `set-custom -u --namespace --key --value ` sets a single custom (value JSON-parsed when possible); `delete-custom` soft-deletes it (value null). Both support `--level company|period|reconciliation|account` (inferred from the URL by default), `--handle`/`--account` targeting, `--file` for batch, and `--yes` to skip the confirmation prompt. +Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a live company file's custom data. `set-custom -u --namespace --key --value ` sets a single custom (value JSON-parsed when possible); `delete-custom` soft-deletes it (value null). Both support `--level company|period|reconciliation|account` (inferred from the URL by default), `--handle`/`--account` targeting, `--file` for batch, `--dry-run` to print the exact properties that would be written without sending the request, and `--yes` to skip the confirmation prompt. Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). Pass `--resolve` to additionally fill inputs whose default is a direct reference to data created elsewhere (a cross-template result/custom, a period/company custom, optionally a prior period via `period.minus_Ny...`); these are read straight from a targeted deep capture of the live company file (only the referenced handles/periods are fetched). Defaults that are computed or conditionally assigned variables remain flagged — they need an actual render. Requires silverfin-ls (for the scope). diff --git a/bin/cli.js b/bin/cli.js index 284b8726..a54cec32 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -784,6 +784,11 @@ const runCustomWrite = async (options, del) => { } const verb = del ? "delete" : "set"; const count = plan.properties.length; + if (options.dryRun) { + consola.info(`[dry-run] Would ${verb} ${count} custom propert${count === 1 ? "y" : "ies"} at the ${plan.level} level (${plan.targetDesc}) on firm ${plan.firmId}, company ${plan.companyId}. No request sent.`); + console.log(JSON.stringify({ dryRun: true, action: verb, level: plan.level, firmId: plan.firmId, companyId: plan.companyId, target: plan.targetDesc, properties: plan.properties }, null, 2)); + return; + } consola.warn(`About to ${verb} ${count} custom propert${count === 1 ? "y" : "ies"} at the ${plan.level} level (${plan.targetDesc}) on firm ${plan.firmId}, company ${plan.companyId}.`); if (!options.yes) { cliUtils.promptConfirmation(); @@ -811,6 +816,7 @@ program .option("--handle ", "Reconciliation handle (for --level reconciliation, instead of the URL target)") .option("--account ", "Account number (for --level account)") .option("--file ", "JSON file with an array of {namespace, key, value} for a batch set") + .option("--dry-run", "Show what would be written without sending the request (optional)", false) .option("-y, --yes", "Skip the confirmation prompt (optional)", false) .action((options) => runCustomWrite(options, false)); @@ -825,6 +831,7 @@ program .option("--handle ", "Reconciliation handle (for --level reconciliation)") .option("--account ", "Account number (for --level account)") .option("--file ", "JSON file with an array of {namespace, key} for a batch delete") + .option("--dry-run", "Show what would be deleted (value:null) without sending the request (optional)", false) .option("-y, --yes", "Skip the confirmation prompt (optional)", false) .action((options) => runCustomWrite(options, true)); From b6679fe55f4fb2c9316802011e9388a161bc9eeb Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Wed, 1 Jul 2026 09:43:16 +0200 Subject: [PATCH 08/17] Add --compute: bounded offline evaluator for variable-defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit describe-inputs --resolve --compute now computes input-default VARIABLES that reduce to a lookup into captured live data (a cross-template result/custom indexed by a date-derived dynamic key, with branch selection). lib/stlLite.js is a deliberately minimal, SAFE STL evaluator: it supports only assign/capture/ if-elsif-else plus a whitelist of filters (date, default, a few string/arith), tracks all block nesting so it never desyncs on the 6000-line shared parts, and NEVER fabricates — anything using currency/MAX()/infix/loops leaves the variable undefined. lib/offlineDefaultResolver.js builds the context from a deep capture and fills only resolved variable-defaults, labelled 'computed: (offline; validate vs live)'. Verified on 2018_275_A_liquidationreserve: all 6 taxable_yearN defaults computed offline match the live results (5000/0/4000/ 3000/2000/0). Values still require validation against a live render. --- CHANGELOG.md | 2 +- bin/cli.js | 3 +- lib/inputDescriber.js | 6 + lib/offlineDefaultResolver.js | 101 ++++++++++++ lib/stlLite.js | 302 ++++++++++++++++++++++++++++++++++ tests/lib/stlLite.test.js | 77 +++++++++ 6 files changed, 489 insertions(+), 2 deletions(-) create mode 100644 lib/offlineDefaultResolver.js create mode 100644 lib/stlLite.js create mode 100644 tests/lib/stlLite.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index b3a505bd..3fdbe151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Added `capture` command. It captures a live company file's data as JSON. By defa Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a live company file's custom data. `set-custom -u --namespace --key --value ` sets a single custom (value JSON-parsed when possible); `delete-custom` soft-deletes it (value null). Both support `--level company|period|reconciliation|account` (inferred from the URL by default), `--handle`/`--account` targeting, `--file` for batch, `--dry-run` to print the exact properties that would be written without sending the request, and `--yes` to skip the confirmation prompt. -Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). Pass `--resolve` to additionally fill inputs whose default is a direct reference to data created elsewhere (a cross-template result/custom, a period/company custom, optionally a prior period via `period.minus_Ny...`); these are read straight from a targeted deep capture of the live company file (only the referenced handles/periods are fetched). Defaults that are computed or conditionally assigned variables remain flagged — they need an actual render. Requires silverfin-ls (for the scope). +Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). Pass `--resolve` to additionally fill inputs whose default is a direct reference to data created elsewhere (a cross-template result/custom, a period/company custom, optionally a prior period via `period.minus_Ny...`); these are read straight from a targeted deep capture of the live company file (only the referenced handles/periods are fetched). Add `--compute` (used with `--resolve`) to additionally compute variable-defaults that reduce to a lookup into captured live data — a cross-template result/custom indexed by a date-derived dynamic key, with branch selection — using a bounded, safe offline STL evaluator (`lib/stlLite.js`) that runs over the template's involved liquid against a deep-capture context. It supports only a whitelist of constructs (assign, capture, if/elsif/else) and filters (date, default, a few string/arith helpers) and NEVER fabricates a value: anything using unsupported operations (`currency`, `MAX()`, infix arithmetic, loops) stays flagged. Computed values are labelled `computed: (offline; validate vs live)` and must be validated against a live render before being trusted. `--compute` is heavier than `--resolve` alone (it deep-captures the template scope). Requires silverfin-ls (for the scope). Added `manifest` command. It prints a reconciliation template's static data scope: own customs, cross-template results/customs, period drop and prior-period depth, company drop, accounts, result echoes and involved files. Usage: `silverfin manifest -h ` or `-u ` (run from your templates repo). The STL analysis is delegated to silverfin-ls (the maintained tree-sitter language server) rather than duplicated here; set `SILVERFIN_LS_CMD` to point at a specific silverfin-ls binary/build. It is the first step of a deep-capture + default-resolution pipeline for resolving effective default values without a browser. diff --git a/bin/cli.js b/bin/cli.js index a54cec32..7fe2039c 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -585,9 +585,10 @@ program .description("List a reconciliation's custom inputs with their declared defaults, stored values, and live effective values (certain sources only), plus the template's results. Use --resolve to additionally fill inputs whose default is a direct reference to data created elsewhere (cross-template results/customs, period/company customs, prior periods) by reading it from a targeted deep capture. Run from your templates repo") .requiredOption("-u, --url ", "Specify the full Silverfin URL of the reconciliation in the company file (mandatory)") .option("--resolve", "Resolve `unavailable` defaults that reference data created elsewhere, via a targeted deep capture of the live company file (extra API calls). Requires silverfin-ls", false) + .option("--compute", "With --resolve: additionally compute variable-defaults that reduce to a lookup into captured live data, using a bounded offline STL evaluator. Heavier (deep-captures the template scope); values are labelled `computed:… (offline; validate vs live)` and MUST be validated against a live render", false) .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") .action(async (options) => { - const data = await inputDescriber.describeInputs(options.url, { resolve: options.resolve }); + const data = await inputDescriber.describeInputs(options.url, { resolve: options.resolve, compute: options.compute }); if (!data) { process.exitCode = 1; return; diff --git a/lib/inputDescriber.js b/lib/inputDescriber.js index c11172f8..5ad355b6 100644 --- a/lib/inputDescriber.js +++ b/lib/inputDescriber.js @@ -164,6 +164,12 @@ async function describeInputs(url, opts = {}) { }; } else { output.resolution = await resolveDefaults(url, scope, rows); + // --compute: additionally compute variable-defaults that reduce to a lookup + // into captured live data (offline evaluator; values need live validation). + if (opts.compute) { + const { computeOfflineDefaults } = require("./offlineDefaultResolver"); + output.offlineResolution = await computeOfflineDefaults(url, scope, rows); + } } } diff --git a/lib/offlineDefaultResolver.js b/lib/offlineDefaultResolver.js new file mode 100644 index 00000000..afd0aa0d --- /dev/null +++ b/lib/offlineDefaultResolver.js @@ -0,0 +1,101 @@ +const fs = require("fs"); +const { buildDeepFixture } = require("./deepCapture"); +const stl = require("./stlLite"); + +/** + * Offline computation of input-default VARIABLES that reduce to a lookup into + * already-captured live data (a cross-template result/custom indexed by a + * date-derived dynamic key, with branch selection). It runs the bounded stlLite + * evaluator over the template's involved liquid against a context built from a + * deep capture, and fills only the defaults stlLite could resolve entirely from + * captured data. + * + * IMPORTANT: values from here are OFFLINE-COMPUTED. Because nearly every read + * ends in `| default:0`, a wrong key/branch silently yields 0 — so these must be + * validated against a live render/results before being trusted. They are labelled + * `computed: (offline; validate vs live)` to keep them distinct from the + * trustworthy `captured:` direct-reference resolutions. + */ + +// period drop (with minus_Ny), reconciliations, company, current_reconciliation. +function buildContext(deep) { + const byOffset = (deep.periodOrder || []).map((key) => { + const p = (deep.data && deep.data.periods && deep.data.periods[key]) || {}; + const recon = {}; + for (const [h, v] of Object.entries(p.reconciliations || {})) { + recon[h] = { results: v.results || {}, custom: v.custom || {} }; + } + return { year_end_date: key, reconciliations: recon, custom: p.custom || {} }; + }); + const period = Object.assign({}, byOffset[0]); + for (let k = 1; k < byOffset.length; k++) period["minus_" + k + "y"] = byOffset[k]; + return { + period, + company: { custom: (deep.data && deep.data.company && deep.data.company.custom) || {}, drop: (deep.data && deep.data.company && deep.data.company.drop) || {} }, + current_reconciliation: { handle: deep.handle }, + }; +} + +function loadLiquid(involvedFiles) { + let out = ""; + for (const file of involvedFiles || []) { + try { + out += "\n" + fs.readFileSync(file, "utf8"); + } catch { + // skip unreadable involved file + } + } + return out; +} + +// A default expression that is a bare variable name (not a literal or data path). +function isVariableDefault(def) { + return typeof def === "string" && /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(def.trim()); +} + +function numericish(v) { + return typeof v === "string" && /^-?\d+(\.\d+)?$/.test(v.trim()) ? Number(v) : v; +} + +/** + * @param {String} url + * @param {Object} scope silverfin-ls data scope (handle, crossTemplate, priorPeriodDepth, involvedFiles) + * @param {Array} rows describe-inputs rows (mutated in place for computed defaults) + * @returns {Object} summary + */ +async function computeOfflineDefaults(url, scope, rows) { + const targets = rows.filter( + (r) => typeof r.effectiveSource === "string" && r.effectiveSource.startsWith("unavailable") && isVariableDefault(r.default) + ); + if (targets.length === 0) { + return { attempted: 0, computed: 0, note: "no variable-defaults to compute" }; + } + + const deep = await buildDeepFixture(url, scope, { maxPriorPeriods: scope.priorPeriodDepth || 0 }); + if (!deep) { + return { attempted: targets.length, computed: 0, warning: "deep capture failed" }; + } + + const ctx = buildContext(deep); + const liquid = loadLiquid(scope.involvedFiles); + let env; + try { + env = stl.run(liquid, ctx); + } catch (error) { + return { attempted: targets.length, computed: 0, warning: `evaluator error: ${String(error.message)}` }; + } + + let computed = 0; + for (const row of targets) { + const value = env[row.default.trim()]; + if (value !== undefined && value !== null && value !== stl.UNRESOLVED) { + row.effective = numericish(value); + row.effectiveSource = `computed:${row.default.trim()} (offline; validate vs live)`; + computed++; + } + } + + return { attempted: targets.length, computed, periodsCaptured: deep.periodsCaptured, note: "offline-computed values require validation against a live render" }; +} + +module.exports = { computeOfflineDefaults, buildContext, isVariableDefault }; diff --git a/lib/stlLite.js b/lib/stlLite.js new file mode 100644 index 00000000..30797772 --- /dev/null +++ b/lib/stlLite.js @@ -0,0 +1,302 @@ +/** + * stlLite — a deliberately MINIMAL, SAFE evaluator for a bounded subset of STL, + * used only to resolve input-default variables that reduce to a *lookup into + * already-captured live data* (a cross-template result/custom indexed by a + * date-derived dynamic key, with branch selection). + * + * Design rules: + * - It supports only a whitelist of constructs (assign, capture, if/elsif/else, + * comment) and filters (date, default, a few string/arith helpers). Anything + * it does not understand (currency, MAX(), infix math, for-loops, includes) + * is SKIPPED — the affected variable stays undefined. It NEVER fabricates a + * value, so a resolved value is always derived entirely from captured data. + * - Block boundaries for unsupported constructs (for/case/tablerow/unless) are + * still tracked so nesting never desyncs; their bodies are just not executed. + * + * This is NOT a Silverfin engine. Values it produces MUST still be validated + * against a live render/results (silent `default:0` masking is real). + */ + +const UNRESOLVED = Symbol("unresolved"); + +// ---- tokenizer ------------------------------------------------------------- +function tokenize(src) { + const tokens = []; + const re = /\{\{-?\s*([\s\S]*?)\s*-?\}\}|\{%-?\s*([\s\S]*?)\s*-?%\}/g; + let last = 0; + let m; + while ((m = re.exec(src)) !== null) { + if (m.index > last) tokens.push({ type: "text", value: src.slice(last, m.index) }); + if (m[1] !== undefined) { + tokens.push({ type: "output", expr: m[1] }); + } else { + const body = m[2].trim(); + const name = body.split(/\s+/, 1)[0]; + tokens.push({ type: "tag", name, body, rest: body.slice(name.length).trim() }); + } + last = re.lastIndex; + } + if (last < src.length) tokens.push({ type: "text", value: src.slice(last) }); + return tokens; +} + +// ---- parser (block tree) --------------------------------------------------- +const BLOCK_OPENERS = { if: "endif", unless: "endunless", for: "endfor", case: "endcase", capture: "endcapture", comment: "endcomment", tablerow: "endtablerow", ifi: "endifi", fori: "endfori" }; + +function parse(tokens) { + let i = 0; + function parseUntil(closers) { + const nodes = []; + while (i < tokens.length) { + const t = tokens[i]; + if (t.type === "tag" && closers.includes(t.name)) return nodes; + if (t.type === "text" || t.type === "output") { nodes.push(t); i++; continue; } + const closer = BLOCK_OPENERS[t.name]; + if (closer) { + i++; + if (t.name === "if" || t.name === "unless" || t.name === "ifi") { + const branches = [{ cond: t.rest, negate: t.name === "unless", body: parseUntil(["elsif", "else", closer]) }]; + while (i < tokens.length && tokens[i].type === "tag" && (tokens[i].name === "elsif" || tokens[i].name === "else")) { + const b = tokens[i]; i++; + branches.push({ cond: b.name === "else" ? null : b.rest, body: parseUntil(["elsif", "else", closer]) }); + } + if (i < tokens.length) i++; // consume endif + nodes.push({ type: "if", branches }); + } else if (t.name === "capture") { + const body = parseUntil([closer]); + if (i < tokens.length) i++; + nodes.push({ type: "capture", name: t.rest, body }); + } else { + // unsupported block (for/case/comment/...): track nesting, don't execute + const body = parseUntil([closer]); + if (i < tokens.length) i++; + nodes.push({ type: "skip", name: t.name, body }); + } + } else { + nodes.push(t); i++; + } + } + return nodes; + } + return parseUntil([]); +} + +// ---- date (strftime subset) ------------------------------------------------ +function strftime(dateStr, fmt) { + const m = String(dateStr).match(/^(\d{4})-(\d{2})-(\d{2})/); + if (!m) return UNRESOLVED; + const [, Y, Mo, D] = m; + return fmt.replace(/%([YymdA-Za-z])/g, (_, c) => { + switch (c) { + case "Y": return Y; + case "y": return Y.slice(2); + case "m": return Mo; + case "d": return D; + default: return "%" + c; + } + }); +} + +// ---- value helpers --------------------------------------------------------- +function unquote(s) { + const q = s.match(/^["']([\s\S]*)["']$/); + return q ? q[1] : null; +} +function toNum(v) { + if (typeof v === "number") return v; + if (typeof v === "string" && /^-?\d+(\.\d+)?$/.test(v.trim())) return Number(v); + return null; +} + +// ---- expression evaluation ------------------------------------------------- +function resolvePath(path, env, ctx) { + // path like a.b.[c].d — supports .key, .[var-or-literal] dynamic access + const parts = []; + let buf = ""; + for (let k = 0; k < path.length; k++) { + const ch = path[k]; + if (ch === ".") { if (buf) { parts.push({ key: buf }); buf = ""; } } + else if (ch === "[") { + if (buf) { parts.push({ key: buf }); buf = ""; } + const end = path.indexOf("]", k); + if (end === -1) return UNRESOLVED; + parts.push({ dyn: path.slice(k + 1, end) }); + k = end; + } else buf += ch; + } + if (buf) parts.push({ key: buf }); + + let cur = undefined; + for (let p = 0; p < parts.length; p++) { + let key; + if (parts[p].dyn !== undefined) { + const dv = evalExpr(parts[p].dyn, env, ctx); + if (dv === UNRESOLVED || dv == null) return UNRESOLVED; + key = String(dv); + } else key = parts[p].key; + + if (p === 0) { + if (key in env) cur = env[key]; + else if (key in ctx) cur = ctx[key]; + else return UNRESOLVED; + } else { + if (cur == null || typeof cur !== "object") return UNRESOLVED; + if (!(key in cur)) return undefined; // known object, absent key -> undefined (feeds default) + cur = cur[key]; + } + } + return cur; +} + +function applyFilter(value, name, args, env, ctx) { + const a = args.map((x) => { + const u = unquote(x); + if (u !== null) return u; + const n = toNum(x); + if (n !== null) return n; + const v = evalExpr(x, env, ctx); + return v === UNRESOLVED ? undefined : v; + }); + switch (name) { + case "default": return value === undefined || value === null || value === "" ? a[0] : value; + case "date": return value == null ? value : strftime(value, a[0]); + case "upcase": return String(value).toUpperCase(); + case "downcase": return String(value).toLowerCase(); + case "strip": return String(value).trim(); + case "append": return String(value) + String(a[0]); + case "prepend": return String(a[0]) + String(value); + case "plus": return toNum(value) + toNum(a[0]); + case "minus": return toNum(value) - toNum(a[0]); + case "times": return toNum(value) * toNum(a[0]); + case "divided_by": return toNum(value) / toNum(a[0]); + case "round": { const n = toNum(value); return a[0] != null ? Number(n.toFixed(a[0])) : Math.round(n); } + case "integer": case "floor": return Math.floor(toNum(value)); + default: return UNRESOLVED; // unsupported filter -> unresolved (never fabricate) + } +} + +function splitTopLevel(str, sep) { + const out = []; + let buf = "", depth = 0, q = null; + for (const ch of str) { + if (q) { buf += ch; if (ch === q) q = null; continue; } + if (ch === '"' || ch === "'") { q = ch; buf += ch; continue; } + if (ch === "[" || ch === "(") depth++; + if (ch === "]" || ch === ")") depth--; + if (ch === sep && depth === 0) { out.push(buf); buf = ""; continue; } + buf += ch; + } + out.push(buf); + return out; +} + +function evalExpr(expr, env, ctx) { + expr = expr.trim(); + if (expr === "") return ""; + const segments = splitTopLevel(expr, "|").map((s) => s.trim()); + const head = segments[0]; + let value; + const u = unquote(head); + if (u !== null) value = u; + else if (/^-?\d+(\.\d+)?$/.test(head)) value = Number(head); + else if (head === "true" || head === "false") value = head === "true"; + else if (head === "blank" || head === "empty" || head === "nil" || head === "null") value = ""; + else if (/[+\-*/]/.test(head) && !/^[\w.[\]]+$/.test(head)) return UNRESOLVED; // infix math -> unsupported + else value = resolvePath(head, env, ctx); + if (value === UNRESOLVED) return UNRESOLVED; + + for (let s = 1; s < segments.length; s++) { + const fm = segments[s].match(/^([a-z_]+)\s*:?\s*([\s\S]*)$/i); + if (!fm) return UNRESOLVED; + const fname = fm[1]; + const fargs = fm[2].trim() === "" ? [] : splitTopLevel(fm[2], ",").map((x) => x.trim()); + value = applyFilter(value, fname, fargs, env, ctx); + if (value === UNRESOLVED) return UNRESOLVED; + } + return value; +} + +function evalCondition(cond, env, ctx) { + if (cond == null) return true; + // handle 'or' then 'and' (no parens support) + const ors = splitTopLevel(cond, "\n").length > 1 ? [cond] : cond.split(/\s+or\s+/i); + for (const orPart of ors) { + const ands = orPart.split(/\s+and\s+/i); + let all = true; + for (const clause of ands) { + if (!evalClause(clause.trim(), env, ctx)) { all = false; break; } + } + if (all) return true; + } + return false; +} + +function evalClause(clause, env, ctx) { + const m = clause.match(/^([\s\S]+?)\s*(==|!=|>=|<=|>|<|contains)\s*([\s\S]+)$/); + if (!m) { + const v = evalExpr(clause, env, ctx); + return v !== UNRESOLVED && v !== undefined && v !== null && v !== false && v !== ""; + } + const l = evalExpr(m[1].trim(), env, ctx); + const r = evalExpr(m[3].trim(), env, ctx); + if (l === UNRESOLVED || r === UNRESOLVED) return false; + const ln = toNum(l), rn = toNum(r); + const num = ln !== null && rn !== null; + switch (m[2]) { + case "==": return String(l) === String(r); + case "!=": return String(l) !== String(r); + case ">=": return num ? ln >= rn : String(l) >= String(r); + case "<=": return num ? ln <= rn : String(l) <= String(r); + case ">": return num ? ln > rn : String(l) > String(r); + case "<": return num ? ln < rn : String(l) < String(r); + case "contains": return String(l).includes(String(r)); + default: return false; + } +} + +// ---- evaluator ------------------------------------------------------------- +function renderText(nodes, env, ctx) { + let out = ""; + for (const n of nodes) { + if (n.type === "text") out += n.value; + else if (n.type === "output") { const v = evalExpr(n.expr, env, ctx); if (v === UNRESOLVED) return UNRESOLVED; out += v == null ? "" : String(v); } + else return UNRESOLVED; // capture body with logic -> unsupported + } + return out; +} + +function evaluate(nodes, env, ctx) { + for (const n of nodes) { + if (n.type === "tag" && n.name === "assign") { + const eq = n.rest.indexOf("="); + if (eq === -1) continue; + const name = n.rest.slice(0, eq).trim(); + const v = evalExpr(n.rest.slice(eq + 1).trim(), env, ctx); + if (v !== UNRESOLVED) env[name] = v; // else: leave undefined (never fabricate) + } else if (n.type === "capture") { + const v = renderText(n.body, env, ctx); + if (v !== UNRESOLVED) env[n.name] = v; + } else if (n.type === "if") { + for (const b of n.branches) { + let take; + if (b.cond === null) take = true; + else { take = evalCondition(b.cond, env, ctx); if (b.negate) take = !take; } + if (take) { evaluate(b.body, env, ctx); break; } + } + } + // text/output/skip: no effect on variable state + } + return env; +} + +/** + * Execute `liquid` against `ctx` and return the resulting variable environment. + * Only variables derivable from the supported subset + captured data are set. + */ +function run(liquid, ctx) { + const env = {}; + evaluate(parse(tokenize(liquid)), env, ctx); + return env; +} + +module.exports = { run, tokenize, parse, evaluate, evalExpr, evalCondition, strftime, UNRESOLVED }; diff --git a/tests/lib/stlLite.test.js b/tests/lib/stlLite.test.js new file mode 100644 index 00000000..8100875d --- /dev/null +++ b/tests/lib/stlLite.test.js @@ -0,0 +1,77 @@ +const stl = require("../../lib/stlLite"); + +const ctx = { + period: { + year_end_date: "2026-01-01", + minus_1y: { + year_end_date: "2024-12-31", + reconciliations: { lr: { results: { addition_12_2024: "5000.0" }, custom: {} } }, + }, + }, + current_reconciliation: { handle: "my_template" }, +}; + +describe("stlLite.run — dynamic-key lookup into captured data", () => { + it("builds a date-derived key and reads the captured value", () => { + const env = stl.run( + ` + {% assign cy = period.minus_1y.year_end_date | date:"%m_%Y" %} + {% capture k %}addition_{{ cy }}{% endcapture %} + {% assign v = period.minus_1y.reconciliations.lr.results.[k] | default:0 %} + `, + ctx + ); + expect(env.cy).toBe("12_2024"); + expect(env.k).toBe("addition_12_2024"); + expect(Number(env.v)).toBe(5000); + }); + + it("applies default:0 when the dynamic key is absent (short/missing period)", () => { + const env = stl.run( + ` + {% capture k %}addition_12_2020{% endcapture %} + {% assign v = period.minus_1y.reconciliations.lr.results.[k] | default:0 %} + `, + ctx + ); + expect(env.v).toBe(0); + }); + + it("selects the correct if/else branch by year comparison", () => { + const env = stl.run( + `{% assign yr = "2024" %}{% if yr >= "2026" %}{% assign b = "from2026" %}{% else %}{% assign b = "normal" %}{% endif %}`, + ctx + ); + expect(env.b).toBe("normal"); + }); + + it("evaluates a handle-gated OR condition using current_reconciliation.handle", () => { + const env = stl.run( + `{% if current_reconciliation.handle == "other" or current_reconciliation.handle == "my_template" %}{% assign inside = 1 %}{% endif %}`, + ctx + ); + expect(env.inside).toBe(1); + }); +}); + +describe("stlLite.run — never fabricates (safety)", () => { + it("leaves a variable undefined when it uses an unsupported filter", () => { + const env = stl.run(`{% assign c = 100 | currency %}`, ctx); + expect(env.c).toBeUndefined(); + }); + + it("leaves a variable undefined when it uses infix arithmetic", () => { + const env = stl.run(`{% assign a = 2 %}{% assign m = a + 3 %}`, ctx); + expect(env.a).toBe(2); + expect(env.m).toBeUndefined(); + }); + + it("does not desync on unsupported blocks (for/case are skipped)", () => { + const env = stl.run( + `{% for x in list %}{% assign ignored = 1 %}{% endfor %}{% assign after = "ok" %}`, + ctx + ); + expect(env.after).toBe("ok"); + expect(env.ignored).toBeUndefined(); + }); +}); From 467d98d9f7f601be66f50e3f075c8a6b3d6ca064 Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Wed, 1 Jul 2026 10:14:21 +0200 Subject: [PATCH 09/17] Widen the offline evaluator's period drop context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deepCapture now records per-period DROP scalars from getPeriods (year_end_date, year_start_date, end_date, fiscal_year) and buildContext exposes them, so the offline evaluator can read period[.minus_Ny].year_start_date and .fiscal_year.* — not just year_end_date. Additive; existing tests unaffected. --- lib/deepCapture.js | 14 +++++++++++++- lib/offlineDefaultResolver.js | 10 +++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/deepCapture.js b/lib/deepCapture.js index 177024b8..28747ae9 100644 --- a/lib/deepCapture.js +++ b/lib/deepCapture.js @@ -90,7 +90,19 @@ async function buildDeepFixture(url, manifest, opts = {}) { for (const period of selected) { const periodId = period.id; const key = periodKey(period); - const entry = { periodId, custom: Utils.processCustom((await SF.getAllPeriodCustom(firmId, companyId, periodId)) || []), reconciliations: {} }; + const entry = { + periodId, + // Period DROP scalars (from getPeriods) so the offline evaluator can read + // period.year_end_date / year_start_date / end_date / fiscal_year.*. + drop: { + year_end_date: key, + year_start_date: period.fiscal_year?.start_date ?? null, + end_date: period.end_date ?? null, + fiscal_year: period.fiscal_year ?? null, + }, + custom: Utils.processCustom((await SF.getAllPeriodCustom(firmId, companyId, periodId)) || []), + reconciliations: {}, + }; const handleMap = await buildHandleMap(firmId, companyId, periodId); for (const handle of neededHandles) { diff --git a/lib/offlineDefaultResolver.js b/lib/offlineDefaultResolver.js index afd0aa0d..a51fde85 100644 --- a/lib/offlineDefaultResolver.js +++ b/lib/offlineDefaultResolver.js @@ -25,7 +25,15 @@ function buildContext(deep) { for (const [h, v] of Object.entries(p.reconciliations || {})) { recon[h] = { results: v.results || {}, custom: v.custom || {} }; } - return { year_end_date: key, reconciliations: recon, custom: p.custom || {} }; + const drop = p.drop || {}; + return { + year_end_date: drop.year_end_date || key, + year_start_date: drop.year_start_date || null, + end_date: drop.end_date || null, + fiscal_year: drop.fiscal_year || null, + reconciliations: recon, + custom: p.custom || {}, + }; }); const period = Object.assign({}, byOffset[0]); for (let k = 1; k < byOffset.length; k++) period["minus_" + k + "y"] = byOffset[k]; From fa65458649545bb4ef44f720bdfe79eb6df5b8df Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Wed, 1 Jul 2026 10:31:42 +0200 Subject: [PATCH 10/17] Offline evaluator: account capture, loops, range aggregation, arithmetic Extends the bounded offline evaluator toward account-range and calculated defaults, keeping the never-fabricate guarantee: - sfApi.getPeriodAccounts (paginated GET .../periods/{id}/accounts). - deepCapture fetches period.accounts (number/name/type/value; no debit-credit arrays) when the template reads accounts (captureAccounts), across all captured periods. - buildContext exposes period.accounts (+ minus_Ny + year_end.accounts alias), period.fiscal_year (stringifies to the year), year_start_date, period.exists, and flattened company.* drop scalars. - stlLite executes for/fori loops, supports account range: filtering with .value/.count/.numbers aggregation, split/size/first/last, currency/percentage as numeric pass-throughs, at_least/at_most, and SAFE infix arithmetic (any unresolved operand -> UNRESOLVED, incl. no-space a-b). opening_value stays UNRESOLVED (not captured) rather than fabricating. Verified live: the 6 taxable_yearN defaults still compute correctly, and period.accounts range aggregation matches a manual sum. 613 tests pass. --- CHANGELOG.md | 2 +- lib/api/sfApi.js | 15 ++++ lib/deepCapture.js | 30 ++++++++ lib/offlineDefaultResolver.js | 51 +++++++++++--- lib/stlLite.js | 128 +++++++++++++++++++++++++++++++++- tests/lib/stlLite.test.js | 57 +++++++++++++-- 6 files changed, 264 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fdbe151..7f1c1fb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Added `capture` command. It captures a live company file's data as JSON. By defa Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a live company file's custom data. `set-custom -u --namespace --key --value ` sets a single custom (value JSON-parsed when possible); `delete-custom` soft-deletes it (value null). Both support `--level company|period|reconciliation|account` (inferred from the URL by default), `--handle`/`--account` targeting, `--file` for batch, `--dry-run` to print the exact properties that would be written without sending the request, and `--yes` to skip the confirmation prompt. -Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). Pass `--resolve` to additionally fill inputs whose default is a direct reference to data created elsewhere (a cross-template result/custom, a period/company custom, optionally a prior period via `period.minus_Ny...`); these are read straight from a targeted deep capture of the live company file (only the referenced handles/periods are fetched). Add `--compute` (used with `--resolve`) to additionally compute variable-defaults that reduce to a lookup into captured live data — a cross-template result/custom indexed by a date-derived dynamic key, with branch selection — using a bounded, safe offline STL evaluator (`lib/stlLite.js`) that runs over the template's involved liquid against a deep-capture context. It supports only a whitelist of constructs (assign, capture, if/elsif/else) and filters (date, default, a few string/arith helpers) and NEVER fabricates a value: anything using unsupported operations (`currency`, `MAX()`, infix arithmetic, loops) stays flagged. Computed values are labelled `computed: (offline; validate vs live)` and must be validated against a live render before being trusted. `--compute` is heavier than `--resolve` alone (it deep-captures the template scope). Requires silverfin-ls (for the scope). +Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). Pass `--resolve` to additionally fill inputs whose default is a direct reference to data created elsewhere (a cross-template result/custom, a period/company custom, optionally a prior period via `period.minus_Ny...`); these are read straight from a targeted deep capture of the live company file (only the referenced handles/periods are fetched). Add `--compute` (used with `--resolve`) to additionally compute variable-defaults that reduce to a lookup into captured live data — a cross-template result/custom indexed by a date-derived dynamic key, an account-range aggregation, or a simple calculation thereof — using a bounded, safe offline STL evaluator (`lib/stlLite.js`) that runs over the template's involved liquid against a deep-capture context. The evaluator supports assign / capture / if-elsif-else / for-loops, dynamic-key indexing, account `range:` filtering + `.value`/`.count` aggregation, safe infix arithmetic, and a whitelist of filters (date, default, split, size, currency/percentage as numeric pass-throughs, at_least/at_most, round, …). It NEVER fabricates: anything whose operands don't fully resolve from captured data (unsupported filter, `MAX()`, `opening_value`, an unresolved operand) stays flagged. The deep capture now also fetches `period.accounts` (value/number/type, when the template reads accounts) and per-period drop scalars, and the context exposes `period.fiscal_year`/`year_start_date`/`exists` and flattened `company.*`. Computed values are labelled `computed: (offline; validate vs live)` and must be validated against a live render before being trusted. `--compute` is heavier than `--resolve` alone (it deep-captures the template scope, incl. accounts). Requires silverfin-ls (for the scope). Added `manifest` command. It prints a reconciliation template's static data scope: own customs, cross-template results/customs, period drop and prior-period depth, company drop, accounts, result echoes and involved files. Usage: `silverfin manifest -h ` or `-u ` (run from your templates repo). The STL analysis is delegated to silverfin-ls (the maintained tree-sitter language server) rather than duplicated here; set `SILVERFIN_LS_CMD` to point at a specific silverfin-ls binary/build. It is the first step of a deep-capture + default-resolution pipeline for resolving effective default values without a browser. diff --git a/lib/api/sfApi.js b/lib/api/sfApi.js index 068a42a6..c32f5d1c 100644 --- a/lib/api/sfApi.js +++ b/lib/api/sfApi.js @@ -727,6 +727,20 @@ async function getAccountDetails(firmId, companyId, periodId, accountId) { } } +async function getPeriodAccounts(firmId, companyId, periodId, page = 1) { + const instance = AxiosFactory.createInstance("firm", firmId); + try { + const response = await instance.get(`companies/${companyId}/periods/${periodId}/accounts`, { + params: { page: page, per_page: PER_PAGE }, + }); + apiUtils.responseSuccessHandler(response); + return response; + } catch (error) { + const response = await apiUtils.responseErrorHandler(error); + return response; + } +} + async function findAccountByNumber(firmId, companyId, periodId, accountNumber, page = 1) { const instance = AxiosFactory.createInstance("firm", firmId); try { @@ -859,6 +873,7 @@ module.exports = { findReconciliationInWorkflow, findReconciliationInWorkflows, getAccountDetails, + getPeriodAccounts, findAccountByNumber, verifyLiquid, getFirmDetails, diff --git a/lib/deepCapture.js b/lib/deepCapture.js index 28747ae9..9e0fd637 100644 --- a/lib/deepCapture.js +++ b/lib/deepCapture.js @@ -56,10 +56,37 @@ async function buildHandleMap(firmId, companyId, periodId) { return map; } +// The period's account drop as { number: {number, name, account_type, value, starred} } +// (aggregate value only — no debit/credit arrays), for period.accounts / range: reads. +async function fetchPeriodAccounts(firmId, companyId, periodId) { + const byNumber = {}; + let page = 1; + while (page <= MAX_PAGES) { + const response = await SF.getPeriodAccounts(firmId, companyId, periodId, page); + const data = response?.data ?? []; + for (const row of data) { + const acc = row.account || {}; + const number = acc.number ?? acc.original_number; + if (number == null) continue; + byNumber[number] = { + number: String(number), + name: acc.name ?? null, + account_type: acc.account_type ?? null, + value: row.value != null ? Number(row.value) : null, + starred: !!row.starred, + }; + } + if (data.length < PER_PAGE) break; + page++; + } + return byNumber; +} + async function buildDeepFixture(url, manifest, opts = {}) { const parameters = Utils.extractURL(url); const { firmId, companyId, ledgerId } = parameters; const maxPrior = opts.maxPriorPeriods ?? manifest.priorPeriodDepth ?? 0; + const captureAccounts = !!opts.captureAccounts; const periods = await fetchAllPeriods(firmId, companyId); const currentIndex = periods.findIndex((period) => String(period.id) === String(ledgerId)); @@ -116,6 +143,9 @@ async function buildDeepFixture(url, manifest, opts = {}) { results: resultsResponse?.data ?? null, }; } + if (captureAccounts) { + entry.accounts = await fetchPeriodAccounts(firmId, companyId, periodId); + } data.periods[key] = entry; } diff --git a/lib/offlineDefaultResolver.js b/lib/offlineDefaultResolver.js index a51fde85..c4edf081 100644 --- a/lib/offlineDefaultResolver.js +++ b/lib/offlineDefaultResolver.js @@ -17,7 +17,21 @@ const stl = require("./stlLite"); * trustworthy `captured:` direct-reference resolutions. */ -// period drop (with minus_Ny), reconciliations, company, current_reconciliation. +// A fiscal_year value that stringifies to the year number (templates index +// custom maps with `[period.fiscal_year]`) while still exposing .start_date/.end_date. +function makeFiscalYear(fy, yearEndDate) { + const src = fy && typeof fy === "object" ? fy : {}; + const ym = String(src.end_date || yearEndDate || "").match(/^(\d{4})/); + const year = ym ? ym[1] : ""; + return { + start_date: src.start_date || null, + end_date: src.end_date || yearEndDate || null, + toString: () => year, + valueOf: () => (year ? Number(year) : NaN), + }; +} + +// period drop (with minus_Ny + accounts), reconciliations, company, current_reconciliation. function buildContext(deep) { const byOffset = (deep.periodOrder || []).map((key) => { const p = (deep.data && deep.data.periods && deep.data.periods[key]) || {}; @@ -26,22 +40,34 @@ function buildContext(deep) { recon[h] = { results: v.results || {}, custom: v.custom || {} }; } const drop = p.drop || {}; - return { + const period = { year_end_date: drop.year_end_date || key, year_start_date: drop.year_start_date || null, end_date: drop.end_date || null, - fiscal_year: drop.fiscal_year || null, + fiscal_year: makeFiscalYear(drop.fiscal_year, drop.year_end_date || key), + exists: true, reconciliations: recon, custom: p.custom || {}, }; + if (p.accounts) { + const arr = Object.values(p.accounts); + period.accounts = arr; + period.year_end = { accounts: arr }; // period.minus_Ny.year_end.accounts alias + } + return period; }); const period = Object.assign({}, byOffset[0]); for (let k = 1; k < byOffset.length; k++) period["minus_" + k + "y"] = byOffset[k]; - return { - period, - company: { custom: (deep.data && deep.data.company && deep.data.company.custom) || {}, drop: (deep.data && deep.data.company && deep.data.company.drop) || {} }, - current_reconciliation: { handle: deep.handle }, - }; + + const companyData = (deep.data && deep.data.company) || {}; + const companyDrop = companyData.drop && typeof companyData.drop === "object" ? companyData.drop : {}; + const company = { custom: companyData.custom || {}, drop: companyDrop }; + // Flatten the drop's top-level scalars onto company.* (name, vat_identifier, ...). + for (const [k, v] of Object.entries(companyDrop)) { + if (v === null || typeof v !== "object") company[k] = v; + } + + return { period, company, current_reconciliation: { handle: deep.handle } }; } function loadLiquid(involvedFiles) { @@ -79,13 +105,18 @@ async function computeOfflineDefaults(url, scope, rows) { return { attempted: 0, computed: 0, note: "no variable-defaults to compute" }; } - const deep = await buildDeepFixture(url, scope, { maxPriorPeriods: scope.priorPeriodDepth || 0 }); + const liquid = loadLiquid(scope.involvedFiles); + // Only pay for the (heavier) account capture when the template actually reads accounts. + const usesAccounts = /period(?:\.minus_\d+y)?\.accounts|\byear_end\.accounts|\brange:/.test(liquid); + const deep = await buildDeepFixture(url, scope, { + maxPriorPeriods: scope.priorPeriodDepth || 0, + captureAccounts: usesAccounts, + }); if (!deep) { return { attempted: targets.length, computed: 0, warning: "deep capture failed" }; } const ctx = buildContext(deep); - const liquid = loadLiquid(scope.involvedFiles); let env; try { env = stl.run(liquid, ctx); diff --git a/lib/stlLite.js b/lib/stlLite.js index 30797772..d6620aee 100644 --- a/lib/stlLite.js +++ b/lib/stlLite.js @@ -66,8 +66,14 @@ function parse(tokens) { const body = parseUntil([closer]); if (i < tokens.length) i++; nodes.push({ type: "capture", name: t.rest, body }); + } else if (t.name === "for" || t.name === "fori") { + const body = parseUntil([closer]); + if (i < tokens.length) i++; + const fm = t.rest.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\s+in\s+(.+?)(?:\s+(?:limit|offset|reversed|order|as)\b.*)?$/is); + if (fm) nodes.push({ type: "for", varName: fm[1], expr: fm[2].trim(), body }); + else nodes.push({ type: "skip", name: t.name, body }); } else { - // unsupported block (for/case/comment/...): track nesting, don't execute + // unsupported block (case/comment/...): track nesting, don't execute const body = parseUntil([closer]); if (i < tokens.length) i++; nodes.push({ type: "skip", name: t.name, body }); @@ -139,6 +145,14 @@ function resolvePath(path, env, ctx) { if (key in env) cur = env[key]; else if (key in ctx) cur = ctx[key]; else return UNRESOLVED; + } else if (Array.isArray(cur)) { + // Account-collection aggregation (period.accounts | range:… | value / count …). + if (key === "value") cur = cur.reduce((s, it) => s + (Number(it && it.value) || 0), 0); + else if (key === "size" || key === "count") cur = cur.length; + else if (key === "numbers") cur = cur.map((it) => it && it.number); + else if (key === "opening_value") return UNRESOLVED; // not captured -> never fabricate + else if (/^\d+$/.test(key)) cur = cur[Number(key)]; + else return UNRESOLVED; } else { if (cur == null || typeof cur !== "object") return UNRESOLVED; if (!(key in cur)) return undefined; // known object, absent key -> undefined (feeds default) @@ -148,6 +162,32 @@ function resolvePath(path, env, ctx) { return cur; } +// Parse a Silverfin account range ("700_709", "280,282,284", "7") into a predicate +// over an account's GL number. +function accountInRange(pattern) { + const digitsOf = (n) => String(n).replace(/[^0-9]/g, ""); + const preds = String(pattern) + .split(",") + .map((t) => t.trim()) + .filter(Boolean) + .map((tok) => { + const rng = tok.match(/^(\d+)_(\d+)$/); + if (rng) { + const width = Math.max(rng[1].length, rng[2].length); + const lo = Number(rng[1].padEnd(width, "0")); + const hi = Number(rng[2].padEnd(width, "9")); + return (num) => { + const d = digitsOf(num); + if (d.length < rng[1].length) return false; + const prefix = Number(d.slice(0, width).padEnd(width, "0")); + return prefix >= lo && prefix <= hi; + }; + } + return (num) => digitsOf(num).startsWith(tok); + }); + return (account) => account && account.number != null && preds.some((p) => p(account.number)); +} + function applyFilter(value, name, args, env, ctx) { const a = args.map((x) => { const u = unquote(x); @@ -171,6 +211,16 @@ function applyFilter(value, name, args, env, ctx) { case "divided_by": return toNum(value) / toNum(a[0]); case "round": { const n = toNum(value); return a[0] != null ? Number(n.toFixed(a[0])) : Math.round(n); } case "integer": case "floor": return Math.floor(toNum(value)); + case "range": return Array.isArray(value) ? value.filter(accountInRange(a[0])) : UNRESOLVED; + case "split": return String(value).split(a[0]); + case "size": return Array.isArray(value) ? value.length : String(value).length; + case "first": return Array.isArray(value) ? value[0] : UNRESOLVED; + case "last": return Array.isArray(value) ? value[value.length - 1] : UNRESOLVED; + // Display/format filters as numeric pass-throughs (they don't change the stored value). + case "currency": case "percentage": case "number": { const n = toNum(value); return n === null ? UNRESOLVED : n; } + case "abs": { const n = toNum(value); return n === null ? UNRESOLVED : Math.abs(n); } + case "at_least": { const n = toNum(value); return n === null ? UNRESOLVED : Math.max(n, toNum(a[0])); } + case "at_most": { const n = toNum(value); return n === null ? UNRESOLVED : Math.min(n, toNum(a[0])); } default: return UNRESOLVED; // unsupported filter -> unresolved (never fabricate) } } @@ -190,6 +240,70 @@ function splitTopLevel(str, sep) { return out; } +// Top-level binary +,-,*,/ (a '-' only counts when it sits between two operands). +function hasArithmetic(s) { + let depth = 0; + for (let i = 0; i < s.length; i++) { + const c = s[i]; + if (c === "[") depth++; + else if (c === "]") depth--; + else if (depth === 0 && "+*/".includes(c)) return true; + else if (depth === 0 && c === "-" && /[\w.\])]/.test(s[i - 1] || "")) return true; + } + return false; +} + +function tokenizeArith(s) { + const tokens = []; + let i = 0; + while (i < s.length) { + const c = s[i]; + if (c === " ") { i++; continue; } + const prev = tokens[tokens.length - 1]; + if ("+*/".includes(c) || (c === "-" && prev && prev.operand !== undefined)) { tokens.push({ op: c }); i++; continue; } + let j = i; + if (s[j] === "-") j++; + let depth = 0; + while (j < s.length) { + const ch = s[j]; + if (ch === "[") depth++; + else if (ch === "]") depth--; + else if (depth === 0 && (ch === " " || "+*/-".includes(ch))) break; + j++; + } + tokens.push({ operand: s.slice(i, j).trim() }); + i = j; + } + return tokens; +} + +// Evaluate top-level arithmetic; UNRESOLVED if any operand doesn't resolve to a +// number from captured data (never fabricates). +function evalArithmetic(s, env, ctx) { + const prec = { "+": 1, "-": 1, "*": 2, "/": 2 }; + const out = [], ops = []; + for (const t of tokenizeArith(s)) { + if (t.operand !== undefined) { + const v = evalExpr(t.operand, env, ctx); + const n = toNum(v); + if (v === UNRESOLVED || n === null) return UNRESOLVED; + out.push(n); + } else { + while (ops.length && prec[ops[ops.length - 1]] >= prec[t.op]) out.push({ op: ops.pop() }); + ops.push(t.op); + } + } + while (ops.length) out.push({ op: ops.pop() }); + const stack = []; + for (const o of out) { + if (typeof o === "number") { stack.push(o); continue; } + const b = stack.pop(), aa = stack.pop(); + if (aa === undefined || b === undefined) return UNRESOLVED; + stack.push(o.op === "+" ? aa + b : o.op === "-" ? aa - b : o.op === "*" ? aa * b : aa / b); + } + return stack.length === 1 ? stack[0] : UNRESOLVED; +} + function evalExpr(expr, env, ctx) { expr = expr.trim(); if (expr === "") return ""; @@ -201,7 +315,7 @@ function evalExpr(expr, env, ctx) { else if (/^-?\d+(\.\d+)?$/.test(head)) value = Number(head); else if (head === "true" || head === "false") value = head === "true"; else if (head === "blank" || head === "empty" || head === "nil" || head === "null") value = ""; - else if (/[+\-*/]/.test(head) && !/^[\w.[\]]+$/.test(head)) return UNRESOLVED; // infix math -> unsupported + else if (hasArithmetic(head)) value = evalArithmetic(head, env, ctx); else value = resolvePath(head, env, ctx); if (value === UNRESOLVED) return UNRESOLVED; @@ -283,6 +397,16 @@ function evaluate(nodes, env, ctx) { else { take = evalCondition(b.cond, env, ctx); if (b.negate) take = !take; } if (take) { evaluate(b.body, env, ctx); break; } } + } else if (n.type === "for") { + const coll = evalExpr(n.expr, env, ctx); + if (Array.isArray(coll)) { + const items = coll.slice(0, 10000); // safety cap + for (const item of items) { + env[n.varName] = item; + evaluate(n.body, env, ctx); + } + } + // non-array collection -> not resolvable from captured data; skip (never fabricate) } // text/output/skip: no effect on variable state } diff --git a/tests/lib/stlLite.test.js b/tests/lib/stlLite.test.js index 8100875d..3a87fff6 100644 --- a/tests/lib/stlLite.test.js +++ b/tests/lib/stlLite.test.js @@ -55,18 +55,17 @@ describe("stlLite.run — dynamic-key lookup into captured data", () => { }); describe("stlLite.run — never fabricates (safety)", () => { - it("leaves a variable undefined when it uses an unsupported filter", () => { - const env = stl.run(`{% assign c = 100 | currency %}`, ctx); + it("leaves a variable undefined when it uses a genuinely unsupported filter", () => { + const env = stl.run(`{% assign c = 100 | some_unknown_filter %}`, ctx); expect(env.c).toBeUndefined(); }); - it("leaves a variable undefined when it uses infix arithmetic", () => { - const env = stl.run(`{% assign a = 2 %}{% assign m = a + 3 %}`, ctx); - expect(env.a).toBe(2); + it("leaves arithmetic undefined when an operand does not resolve", () => { + const env = stl.run(`{% assign m = unknown_var - 3 %}`, ctx); expect(env.m).toBeUndefined(); }); - it("does not desync on unsupported blocks (for/case are skipped)", () => { + it("does not desync on unsupported blocks / unresolved for-collections", () => { const env = stl.run( `{% for x in list %}{% assign ignored = 1 %}{% endfor %}{% assign after = "ok" %}`, ctx @@ -75,3 +74,49 @@ describe("stlLite.run — never fabricates (safety)", () => { expect(env.ignored).toBeUndefined(); }); }); + +describe("stlLite.run — arithmetic, filters, loops, account aggregation", () => { + it("computes safe arithmetic (incl. no-space subtraction and precedence)", () => { + const env = stl.run(`{% assign a = 10 %}{% assign b = 3 %}{% assign r = a-b %}{% assign r2 = a + b * 2 %}`, ctx); + expect(env.r).toBe(7); + expect(env.r2).toBe(16); + }); + + it("treats currency/percentage as numeric pass-throughs", () => { + const env = stl.run(`{% assign c = 5000.5 | currency %}{% assign p = 21 | percentage %}`, ctx); + expect(env.c).toBe(5000.5); + expect(env.p).toBe(21); + }); + + it("clamps with at_least / at_most", () => { + const env = stl.run(`{% assign lo = 3 | at_least:10 %}{% assign hi = 30 | at_most:10 %}`, ctx); + expect(env.lo).toBe(10); + expect(env.hi).toBe(10); + }); + + it("splits a string into an array and iterates it in a for-loop", () => { + const env = stl.run(`{% assign parts = "a|b|c" | split:"|" %}{% assign n = 0 %}{% for p in parts %}{% assign n = n + 1 %}{% endfor %}`, ctx); + expect(env.n).toBe(3); + }); + + it("filters period.accounts by range and aggregates .value / .count", () => { + const actx = { + period: { + accounts: [ + { number: "700000", value: 100 }, + { number: "705000", value: 50 }, + { number: "710000", value: 999 }, + ], + }, + }; + const env = stl.run(`{% assign s = period.accounts | range:'700_709' %}{% assign total = s.value %}{% assign cnt = s.count %}`, actx); + expect(env.total).toBe(150); + expect(env.cnt).toBe(2); + }); + + it("does not fabricate opening_value (not captured)", () => { + const actx = { period: { accounts: [{ number: "700000", value: 100 }] } }; + const env = stl.run(`{% assign x = period.accounts | range:'7' %}{% assign ov = x.opening_value %}`, actx); + expect(env.ov).toBeUndefined(); + }); +}); From 8577a79868bad7ded91d9fcaaa1a7c9097e00c12 Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Wed, 1 Jul 2026 11:36:45 +0200 Subject: [PATCH 11/17] Add set-default: set a field's default via its upstream custom New 'set-default' command changes an input's DEFAULT (not an override) by writing the upstream custom the default derives from: - lib/provenanceTracer.js reverse-traces a default to a single settable custom, reusing the offline evaluator (an onAssign hook on stlLite) to resolve the taken branch + dynamic keys. Auto-invertible = a direct cross-template custom, or a result that statically echoes a custom; everything else (computed / branch-gated / dynamic result / arithmetic) is reported NOT invertible with the provenance chain, never guessed. - lib/defaultSetter.js orchestrates: light capture (period dates only, fast), trace, resolve the upstream reconciliation, write, verify, and emit a change table with old -> new + why + a blast-radius note when >1 template reads the source. Warns if the target field has a shadowing override. - lib/changeReport.js: shared change table + JSON. Live-verified on 275A (non-invertible dynamic-result path surfaces the chain in ~7s). 624 tests pass. --- CHANGELOG.md | 2 + bin/cli.js | 38 +++++++ lib/changeReport.js | 52 +++++++++ lib/defaultSetter.js | 172 +++++++++++++++++++++++++++++ lib/provenanceTracer.js | 130 ++++++++++++++++++++++ lib/stlLite.js | 16 ++- tests/lib/changeReport.test.js | 26 +++++ tests/lib/provenanceTracer.test.js | 57 ++++++++++ 8 files changed, 487 insertions(+), 6 deletions(-) create mode 100644 lib/changeReport.js create mode 100644 lib/defaultSetter.js create mode 100644 lib/provenanceTracer.js create mode 100644 tests/lib/changeReport.test.js create mode 100644 tests/lib/provenanceTracer.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f1c1fb6..121ebbf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] + +Added `set-default` command. Instead of overriding a field, it sets an input's **default** by writing the **upstream** custom the default derives from — so the field stays un-overridden and the value flows in as it does in the real case. Usage: `silverfin set-default -u --input custom.. --value X [--dry-run]` (run from your templates repo). It reverse-traces the default's provenance (via `lib/provenanceTracer.js` + the offline evaluator) to a single settable custom; it auto-proceeds when the default is auto-invertible (a direct cross-template custom, or a result that statically echoes a custom) and prints a change table (target · level · namespace.key · old → new · why) with a blast-radius note when >1 template reads that source. When the default is computed / branch-gated / a dynamic result (no single custom to set), it writes nothing and prints the provenance chain, pointing to the upstream template + period to set directly. Adds a shared `lib/changeReport.js` (change table + JSON) and requires silverfin-ls. Added `update-text-properties` command. It uploads custom text properties from a Liquid Test YAML file to a company file at company, period, reconciliation and account levels for the entries referenced in the test scenario. Usage: `silverfin update-text-properties -u -t `. Supports `--handle` for faster YAML file lookup, `--dry-run` to preview the payload, and `--yes` to skip the confirmation prompt. Added `get-results` command. It fetches the computed results and custom data of a reconciliation or account in a live company file (identified by its Silverfin URL) and prints them as JSON. Usage: `silverfin get-results -u `. Supports `-o, --output ` to write the JSON to a file instead of stdout. diff --git a/bin/cli.js b/bin/cli.js index 7fe2039c..fb4333ab 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -26,6 +26,7 @@ const fsUtils = require("../lib/utils/fsUtils"); const textPropertyUtils = require("../lib/utils/textPropertyUtils"); const liquidTestUtils = require("../lib/utils/liquidTestUtils"); const customWriter = require("../lib/customWriter"); +const defaultSetter = require("../lib/defaultSetter"); const firmIdDefault = cliUtils.loadDefaultFirmId(); cliUtils.handleUncaughtErrors(); @@ -836,6 +837,43 @@ program .option("-y, --yes", "Skip the confirmation prompt (optional)", false) .action((options) => runCustomWrite(options, true)); +// SET-DEFAULT — change an input's DEFAULT by writing the upstream custom it derives from +program + .command("set-default") + .description("Set a reconciliation input's DEFAULT to a value by writing the UPSTREAM custom it derives from — NOT an override on the field itself. Auto-proceeds when the default is auto-invertible (a direct cross-template custom, or a result that echoes a custom); otherwise it prints the provenance chain and writes nothing. Prints a change table (old → new + why + blast radius). Run from your templates repo. Requires silverfin-ls") + .requiredOption("-u, --url ", "Full Silverfin URL of the reconciliation in the company file (mandatory)") + .requiredOption("--input ", "The input whose default to set, e.g. custom.liquidation.taxable_year1 (mandatory)") + .requiredOption("--value ", "Desired default value (mandatory)") + .option("--dry-run", "Show the upstream change that would be made, without writing (optional)", false) + .option("-o, --output ", "Write the JSON result to a file (optional)") + .action(async (options) => { + const result = await defaultSetter.setDefault(options.url, options.input, options.value, { dryRun: options.dryRun }); + if (!result) { + process.exitCode = 1; + return; + } + if (!result.invertible) { + consola.warn(`"${options.input}" default is NOT auto-invertible — nothing written.`); + consola.info(result.trace.reason); + console.log("\nProvenance chain:\n " + result.trace.chain.join("\n → ")); + if (result.trace.upstreamResult) { + console.log(`\nTo change it, set ${result.trace.upstreamResult.handle}'s own inputs (period ${result.trace.upstreamResult.periodKey}) directly:`); + console.log(` silverfin set-custom -u "<${result.trace.upstreamResult.handle} url>" --namespace --key --value ${options.value}`); + } + return; + } + console.log(result.report.toTable()); + if (result.wrote) { + if (result.applied) consola.success(`Applied. Upstream value now: ${JSON.stringify(result.verified)}. Re-run describe-inputs to confirm the default flowed.`); + else consola.error("The upstream write failed."); + } else { + consola.info("[dry-run] no request sent."); + } + if (options.output) { + require("fs").writeFileSync(options.output, JSON.stringify({ ...result, report: result.report.toJSON() }, null, 2)); + } + }); + // Check Liquid Test dependencies for a reconciliation template program .command("check-dependencies") diff --git a/lib/changeReport.js b/lib/changeReport.js new file mode 100644 index 00000000..793f520f --- /dev/null +++ b/lib/changeReport.js @@ -0,0 +1,52 @@ +/** + * Collects the exact custom writes performed to satisfy a request and renders them + * as a human table + machine JSON, so the user always sees WHAT was changed, WHERE, + * from WHICH old value, and WHY (which target field the upstream write was for). + */ + +function fmt(v) { + if (v === null || v === undefined) return "∅"; + if (typeof v === "object") return JSON.stringify(v); + return String(v); +} + +class ChangeReport { + constructor() { + this.changes = []; + this.notes = []; + } + + // change: { target, level, namespace, key, oldValue, newValue, why, applied } + add(change) { + this.changes.push(change); + return this; + } + + note(message) { + this.notes.push(message); + return this; + } + + toJSON() { + return { changes: this.changes, notes: this.notes }; + } + + toTable() { + if (this.changes.length === 0) return "(no custom changes)"; + const headers = ["Target", "Level", "namespace.key", "old → new", "Why"]; + const rows = this.changes.map((c) => [ + String(c.target || ""), + String(c.level || ""), + `${c.namespace}.${c.key}`, + `${fmt(c.oldValue)} → ${fmt(c.newValue)}`, + String(c.why || ""), + ]); + const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length))); + const line = (cols) => cols.map((c, i) => c.padEnd(widths[i])).join(" "); + const out = [line(headers), widths.map((w) => "─".repeat(w)).join(" "), ...rows.map(line)]; + if (this.notes.length) out.push("", ...this.notes.map((n) => "• " + n)); + return out.join("\n"); + } +} + +module.exports = { ChangeReport, fmt }; diff --git a/lib/defaultSetter.js b/lib/defaultSetter.js new file mode 100644 index 00000000..88d27277 --- /dev/null +++ b/lib/defaultSetter.js @@ -0,0 +1,172 @@ +const fs = require("fs"); +const path = require("path"); +const { consola } = require("consola"); +const SF = require("./api/sfApi"); +const Utils = require("./utils/liquidTestUtils"); +const { ReconciliationText } = require("./templates/reconciliationText"); +const inputDescriber = require("./inputDescriber"); +const { getDataScope } = require("./dataScope"); +const { buildContext } = require("./offlineDefaultResolver"); +const { traceDefault } = require("./provenanceTracer"); +const { ChangeReport } = require("./changeReport"); +const { coerceValue } = require("./customWriter"); + +// Coarse blast radius: template dirs (reconciliation_texts + shared_parts) that +// reference the upstream handle's results/customs. Honest and cheap; a shared +// upstream is a broad change, so we surface who else reads it. +function downstreamReaders(handle, kind) { + const roots = ["reconciliation_texts", "shared_parts"]; + const readers = new Set(); + const needle = new RegExp(`reconciliations\\.${handle}\\.${kind}`); + for (const root of roots) { + let dirs; + try { dirs = fs.readdirSync(path.join(process.cwd(), root)); } catch { continue; } + for (const dir of dirs) { + const base = path.join(process.cwd(), root, dir); + const files = []; + try { + const walk = (p) => { + for (const e of fs.readdirSync(p, { withFileTypes: true })) { + const fp = path.join(p, e.name); + if (e.isDirectory()) walk(fp); + else if (e.name.endsWith(".liquid")) files.push(fp); + } + }; + walk(base); + } catch { continue; } + for (const f of files) { + try { + if (needle.test(fs.readFileSync(f, "utf8"))) { readers.add(`${root}/${dir}`); break; } + } catch { /* skip */ } + } + } + } + return [...readers]; +} + +// Lightweight capture for TRACING only: period dates (periodOrder + drop scalars). +// Tracing a default resolves a source PATH (handle + date-derived key), which needs +// only the period sequence — not the reconciliation data — so we skip the slow walk. +async function captureLight(url, handle, depth) { + const { firmId, companyId, ledgerId } = Utils.extractURL(url); + const periods = await SF.getAllPeriods(firmId, companyId); + const currentIndex = periods.findIndex((p) => String(p.id) === String(ledgerId)); + if (currentIndex === -1) return null; + const selected = periods.slice(currentIndex, currentIndex + (depth || 0) + 1); + const keyOf = (p) => (p.fiscal_year?.end_date ? String(p.fiscal_year.end_date) : String(p.id)); + const periodOrder = selected.map(keyOf); + const data = { company: { custom: {}, drop: null }, periods: {} }; + for (const p of selected) { + data.periods[keyOf(p)] = { + periodId: p.id, + drop: { + year_end_date: keyOf(p), + year_start_date: p.fiscal_year?.start_date ?? null, + end_date: p.end_date ?? null, + fiscal_year: p.fiscal_year ?? null, + }, + custom: {}, + reconciliations: {}, + }; + } + return { handle, periodOrder, currentPeriodKey: periodOrder[0] ?? null, data }; +} + +/** + * Set a target input's DEFAULT to `rawValue` by writing the upstream custom it + * derives from (never an override on the target field). Auto-proceeds when the + * default is auto-invertible; otherwise surfaces the provenance chain and does + * nothing. Returns a structured result (the command renders it). + */ +async function setDefault(url, inputPath, rawValue, opts = {}) { + const parameters = Utils.extractURL(url); + const { firmId, companyId } = parameters; + const details = await SF.readReconciliationTextDetails("firm", firmId, companyId, parameters.ledgerId, parameters.reconciliationId); + const handle = details?.data?.handle; + if (!handle) { consola.error("Could not resolve the reconciliation handle from the URL."); return null; } + + const template = ReconciliationText.read(handle); + if (!template) { consola.error(`Template "${handle}" not found locally — run from your templates repo.`); return null; } + const inputs = inputDescriber.parseInputs(inputDescriber.combineLiquid(template)); + const row = inputs.find((i) => i.path === inputPath); + if (!row) { consola.error(`Input "${inputPath}" not found in ${handle}.`); return null; } + + const scope = getDataScope(handle); + if (!scope) return null; + let liquid = ""; + for (const f of scope.involvedFiles || []) { try { liquid += "\n" + fs.readFileSync(f, "utf8"); } catch { /* skip */ } } + + const light = await captureLight(url, handle, scope.priorPeriodDepth || 0); + if (!light) { consola.error("Could not resolve the company periods."); return null; } + const ctx = buildContext(light); + const trace = traceDefault(row, liquid, ctx, light, (h) => getDataScope(h)); + + if (!trace.invertible) { + return { invertible: false, handle, input: inputPath, trace }; + } + + // Resolve the upstream write target (same company, traced period + handle). + const t = trace.target; + const periodEntry = light.data.periods[t.periodKey]; + if (!periodEntry) return { invertible: false, handle, input: inputPath, trace: { ...trace, invertible: false, reason: `period ${t.periodKey} not captured` } }; + const periodId = periodEntry.periodId; + const upstreamRecon = await SF.findReconciliationInWorkflows(firmId, t.handle, companyId, periodId); + if (!upstreamRecon || !upstreamRecon.id) { + return { invertible: false, handle, input: inputPath, trace: { ...trace, invertible: false, reason: `upstream template ${t.handle} not found in period ${t.periodKey}` } }; + } + const upstreamId = upstreamRecon.id; + const nsKey = `${t.namespace}.${t.key}`; + let oldValue = null; + try { + const cur = await SF.getReconciliationCustom("firm", firmId, companyId, periodId, upstreamId); + const map = Utils.processCustom(cur?.data || []); + if (Object.hasOwn(map, nsKey)) oldValue = map[nsKey]; + } catch { /* ignore */ } + const newValue = coerceValue(String(rawValue)); + + const report = new ChangeReport(); + report.add({ + target: `${t.handle} @ ${t.periodKey} (company ${companyId})`, + level: "reconciliation", + namespace: t.namespace, + key: t.key, + oldValue, + newValue, + why: `to set default of ${inputPath} (${trace.via})`, + applied: false, + }); + + const uniqueReaders = [...new Set(downstreamReaders(t.handle, "results").concat(downstreamReaders(t.handle, "custom")))]; + if (uniqueReaders.length > 1) { + report.note(`Blast radius: ${uniqueReaders.length} templates read ${t.handle} — this also affects: ${uniqueReaders.slice(0, 8).join(", ")}${uniqueReaders.length > 8 ? " …" : ""}`); + } + + // Warn if the target field has a stored override that would SHADOW the default. + try { + const tgt = await SF.getReconciliationCustom("firm", firmId, companyId, parameters.ledgerId, parameters.reconciliationId); + const tgtMap = Utils.processCustom(tgt?.data || []); + const tgtKey = `${row.namespace}.${row.key}`; + if (Object.hasOwn(tgtMap, tgtKey) && tgtMap[tgtKey] != null) { + report.note(`${inputPath} currently has a STORED override (${tgtMap[tgtKey]}); it keeps showing that until you clear it with delete-custom — set-default only changes the DEFAULT.`); + } + } catch { /* ignore */ } + + if (opts.dryRun) { + return { invertible: true, handle, input: inputPath, trace, report, applied: false, wrote: false }; + } + + const response = await SF.updateReconciliationCustom(firmId, companyId, periodId, upstreamId, [{ namespace: t.namespace, key: t.key, value: newValue }]); + const ok = response && response.status >= 200 && response.status < 300; + report.changes[0].applied = ok; + + let verified = null; + try { + const fresh = await SF.getReconciliationCustom("firm", firmId, companyId, periodId, upstreamId); + const map = Utils.processCustom(fresh?.data || []); + verified = Object.hasOwn(map, nsKey) ? map[nsKey] : null; + } catch { /* ignore */ } + + return { invertible: true, handle, input: inputPath, trace, report, applied: ok, wrote: true, verified }; +} + +module.exports = { setDefault, downstreamReaders, captureLight }; diff --git a/lib/provenanceTracer.js b/lib/provenanceTracer.js new file mode 100644 index 00000000..14454a4e --- /dev/null +++ b/lib/provenanceTracer.js @@ -0,0 +1,130 @@ +const stl = require("./stlLite"); + +/** + * Reverse provenance: given a target input whose value comes from a DEFAULT, find + * the single upstream CUSTOM that should be set so the default becomes the desired + * value — or explain why it isn't auto-invertible (computed / branch-gated / a + * dynamic result with no static echo / account-derived). Never guesses. + * + * A default is auto-invertible when it reduces to: + * - a direct cross-template custom: period[.minus_Ny].reconciliations..custom.. + * - a cross-template result that produces by directly echoing one of its own + * customs: {% result 'tag' custom.. %} (resolved via 's data scope) + */ + +// period[.minus_Ny].reconciliations..(results|custom) +function parseUpstreamRef(head) { + let periodOffset = 0; + let rest = head.trim(); + let m = rest.match(/^period\.minus_(\d+)y\.(.+)$/i); + if (m) { + periodOffset = Number(m[1]); + rest = m[2]; + } else { + m = rest.match(/^period\.(.+)$/i); + if (!m) return null; + rest = m[1]; + } + m = rest.match(/^reconciliations\.([a-z0-9_]+)\.(results|custom)(.*)$/i); + if (!m) return null; + return { periodOffset, handle: m[1], kind: m[2].toLowerCase(), tail: m[3] || "" }; +} + +// Resolve ".a.[b].c" against an env into ["a", , "c"] (dynamic keys first). +function resolveSegments(tail, env, ctx) { + const segs = []; + let i = 0; + while (i < tail.length) { + const c = tail[i]; + if (c === ".") { i++; continue; } + if (c === "[") { + const end = tail.indexOf("]", i); + if (end === -1) return null; + const v = stl.evalExpr(tail.slice(i + 1, end), env, ctx); + if (v === stl.UNRESOLVED || v == null) return null; + segs.push(String(v)); + i = end + 1; + } else { + let j = i; + while (j < tail.length && tail[j] !== "." && tail[j] !== "[") j++; + segs.push(tail.slice(i, j)); + i = j; + } + } + return segs; +} + +// Turn a resolved upstream ref into a settable-custom target, resolving a result +// tag to its producing custom via the upstream handle's data scope when needed. +function classify(ref, segs, deep, getScope) { + const periodKey = (deep.periodOrder || [])[ref.periodOffset] || null; + const base = { handle: ref.handle, periodOffset: ref.periodOffset, periodKey }; + if (ref.kind === "custom") { + if (!segs || segs.length < 2) return { invertible: false, reason: "custom reference key could not be resolved" }; + return { invertible: true, target: { ...base, namespace: segs[0], key: segs[1] }, via: "direct-custom" }; + } + // results + const tag = segs && segs[0]; + if (!tag) return { invertible: false, reason: "result tag could not be resolved" }; + const scope = getScope ? getScope(ref.handle) : null; + const echoed = scope && scope.resultEchoes && scope.resultEchoes[tag]; + if (!echoed) { + return { + invertible: false, + reason: `default comes from ${ref.handle}.results.${tag}, a computed result with no static custom echo — set ${ref.handle}'s inputs directly (mode 3) instead`, + upstreamResult: { handle: ref.handle, tag, periodKey }, + }; + } + const parts = String(echoed).split("."); // custom.ns.key + if (parts.length < 3) return { invertible: false, reason: `result ${tag} echoes ${echoed}, which is not a custom` }; + return { invertible: true, target: { ...base, namespace: parts[1], key: parts.slice(2).join(".") }, via: `result-echo:${tag}` }; +} + +/** + * @param {Object} input describe-inputs row (has .input, .default) + * @param {String} involvedLiquid concatenated involved-file liquid + * @param {Object} ctx offline-evaluator context (from buildContext) + * @param {Object} deep the deep fixture (for periodOrder) + * @param {Function} [getScope] handle -> data scope (for result-echo tracing) + * @returns {Object} { invertible, target?, via?, reason?, chain } + */ +function traceDefault(input, involvedLiquid, ctx, deep, getScope) { + const def = String(input.default || "").trim(); + const label = input.input || input.path || "input"; + const chain = [`${label} default = ${def || "(none)"}`]; + if (!def) return { invertible: false, reason: "input has no default", chain }; + + // Direct reference in the default expression itself. + const directHead = def.split("|")[0].trim(); + const directRef = parseUpstreamRef(directHead); + if (directRef) { + const segs = resolveSegments(directRef.tail, ctx.__env || {}, ctx); + chain.push(`${directRef.handle}.${directRef.kind}${directRef.tail}`); + return { ...classify(directRef, segs, deep, getScope), chain }; + } + + // Variable default: trace to the RHS of its taken assignment. + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(def)) { + return { invertible: false, reason: "default is a computed expression, not a single reference", chain }; + } + let captured = null; + stl.run(involvedLiquid, ctx, { + onAssign: (name, rhs, value, env) => { + if (name !== def) return; + const head = rhs.split("|")[0].trim(); + const ref = parseUpstreamRef(head); + captured = { ref, segs: ref ? resolveSegments(ref.tail, env, ctx) : null, rhs }; + }, + }); + if (!captured) return { invertible: false, reason: `could not find an assignment for ${def} in the template`, chain }; + if (!captured.ref) { + chain.push(`${def} = ${captured.rhs}`); + return { invertible: false, reason: `${def} is computed (${captured.rhs.trim()}), not a single cross-template reference`, chain }; + } + chain.push(`${def} = ${captured.ref.handle}.${captured.ref.kind}${captured.ref.tail}`); + const cls = classify(captured.ref, captured.segs, deep, getScope); + if (cls.target) chain.push(`set ${cls.target.handle}.custom.${cls.target.namespace}.${cls.target.key}`); + return { ...cls, chain }; +} + +module.exports = { traceDefault, parseUpstreamRef, resolveSegments, classify }; diff --git a/lib/stlLite.js b/lib/stlLite.js index d6620aee..27f1980c 100644 --- a/lib/stlLite.js +++ b/lib/stlLite.js @@ -379,14 +379,18 @@ function renderText(nodes, env, ctx) { return out; } -function evaluate(nodes, env, ctx) { +function evaluate(nodes, env, ctx, hooks) { for (const n of nodes) { if (n.type === "tag" && n.name === "assign") { const eq = n.rest.indexOf("="); if (eq === -1) continue; const name = n.rest.slice(0, eq).trim(); - const v = evalExpr(n.rest.slice(eq + 1).trim(), env, ctx); + const rhs = n.rest.slice(eq + 1).trim(); + const v = evalExpr(rhs, env, ctx); if (v !== UNRESOLVED) env[name] = v; // else: leave undefined (never fabricate) + // Trace hook: record the RHS of each executed (taken-branch) assignment so a + // reverse tracer can see what a variable's value was derived from. + if (hooks && hooks.onAssign) hooks.onAssign(name, rhs, v, env); } else if (n.type === "capture") { const v = renderText(n.body, env, ctx); if (v !== UNRESOLVED) env[n.name] = v; @@ -395,7 +399,7 @@ function evaluate(nodes, env, ctx) { let take; if (b.cond === null) take = true; else { take = evalCondition(b.cond, env, ctx); if (b.negate) take = !take; } - if (take) { evaluate(b.body, env, ctx); break; } + if (take) { evaluate(b.body, env, ctx, hooks); break; } } } else if (n.type === "for") { const coll = evalExpr(n.expr, env, ctx); @@ -403,7 +407,7 @@ function evaluate(nodes, env, ctx) { const items = coll.slice(0, 10000); // safety cap for (const item of items) { env[n.varName] = item; - evaluate(n.body, env, ctx); + evaluate(n.body, env, ctx, hooks); } } // non-array collection -> not resolvable from captured data; skip (never fabricate) @@ -417,9 +421,9 @@ function evaluate(nodes, env, ctx) { * Execute `liquid` against `ctx` and return the resulting variable environment. * Only variables derivable from the supported subset + captured data are set. */ -function run(liquid, ctx) { +function run(liquid, ctx, hooks) { const env = {}; - evaluate(parse(tokenize(liquid)), env, ctx); + evaluate(parse(tokenize(liquid)), env, ctx, hooks); return env; } diff --git a/tests/lib/changeReport.test.js b/tests/lib/changeReport.test.js new file mode 100644 index 00000000..5b4e0a83 --- /dev/null +++ b/tests/lib/changeReport.test.js @@ -0,0 +1,26 @@ +const { ChangeReport } = require("../../lib/changeReport"); + +describe("ChangeReport", () => { + it("renders a table with old → new and why, plus notes", () => { + const r = new ChangeReport(); + r.add({ target: "liquidation_reserve @ 2024-12-31", level: "reconciliation", namespace: "reserve", key: "addition_2024", oldValue: 3000, newValue: 5000, why: "to set default of 275A.taxable_year1" }); + r.note("Blast radius: 3 templates read liquidation_reserve"); + const table = r.toTable(); + expect(table).toMatch(/liquidation_reserve @ 2024-12-31/); + expect(table).toMatch(/reserve\.addition_2024/); + expect(table).toMatch(/3000 → 5000/); + expect(table).toMatch(/to set default of 275A\.taxable_year1/); + expect(table).toMatch(/Blast radius: 3 templates/); + }); + + it("shows ∅ for null old values and serializes to JSON", () => { + const r = new ChangeReport(); + r.add({ target: "t", level: "reconciliation", namespace: "ns", key: "k", oldValue: null, newValue: 1, why: "w" }); + expect(r.toTable()).toMatch(/∅ → 1/); + expect(r.toJSON().changes).toHaveLength(1); + }); + + it("handles the empty case", () => { + expect(new ChangeReport().toTable()).toBe("(no custom changes)"); + }); +}); diff --git a/tests/lib/provenanceTracer.test.js b/tests/lib/provenanceTracer.test.js new file mode 100644 index 00000000..af4c5f54 --- /dev/null +++ b/tests/lib/provenanceTracer.test.js @@ -0,0 +1,57 @@ +const { traceDefault, parseUpstreamRef } = require("../../lib/provenanceTracer"); + +const deep = { periodOrder: ["2024-12-31", "2023-12-31"] }; +const ctx = { period: {} }; + +describe("provenanceTracer.parseUpstreamRef", () => { + it("parses current and prior-period reconciliation refs", () => { + expect(parseUpstreamRef("period.reconciliations.foo.custom.ns.key")).toEqual({ periodOffset: 0, handle: "foo", kind: "custom", tail: ".ns.key" }); + expect(parseUpstreamRef("period.minus_2y.reconciliations.foo.results.tag")).toEqual({ periodOffset: 2, handle: "foo", kind: "results", tail: ".tag" }); + }); + it("rejects non-reconciliation refs", () => { + expect(parseUpstreamRef("period.year_end_date")).toBeNull(); + expect(parseUpstreamRef("company.custom.a.b")).toBeNull(); + }); +}); + +describe("provenanceTracer.traceDefault", () => { + it("auto-inverts a default that is a direct cross-template custom", () => { + const r = traceDefault({ input: "custom.x.y", default: "period.reconciliations.other.custom.ns.key" }, "", ctx, deep, () => null); + expect(r.invertible).toBe(true); + expect(r.target).toMatchObject({ handle: "other", namespace: "ns", key: "key", periodKey: "2024-12-31" }); + }); + + it("auto-inverts a variable default that assigns from an upstream custom", () => { + const liquid = `{% assign myvar = period.minus_1y.reconciliations.other.custom.ns.key %}`; + const r = traceDefault({ input: "custom.x.y", default: "myvar" }, liquid, ctx, deep, () => null); + expect(r.invertible).toBe(true); + expect(r.target).toMatchObject({ handle: "other", namespace: "ns", key: "key", periodKey: "2023-12-31" }); + }); + + it("inverts a result default via a static custom echo (upstream scope)", () => { + const getScope = (h) => (h === "other" ? { resultEchoes: { tag: "custom.ns2.key2" } } : null); + const r = traceDefault({ input: "custom.x.y", default: "period.reconciliations.other.results.tag" }, "", ctx, deep, getScope); + expect(r.invertible).toBe(true); + expect(r.via).toBe("result-echo:tag"); + expect(r.target).toMatchObject({ handle: "other", namespace: "ns2", key: "key2" }); + }); + + it("does NOT invert a result with no static echo (points to mode 3)", () => { + const r = traceDefault({ input: "custom.x.y", default: "period.reconciliations.other.results.tag" }, "", ctx, deep, () => null); + expect(r.invertible).toBe(false); + expect(r.upstreamResult).toMatchObject({ handle: "other", tag: "tag" }); + expect(r.reason).toMatch(/computed result/); + }); + + it("does NOT invert a computed (arithmetic) default", () => { + const liquid = `{% assign v = a - b %}`; + const r = traceDefault({ input: "custom.x.y", default: "v" }, liquid, ctx, deep, () => null); + expect(r.invertible).toBe(false); + expect(r.reason).toMatch(/computed/); + }); + + it("does NOT invert when there is no default", () => { + const r = traceDefault({ input: "custom.x.y", default: null }, "", ctx, deep, () => null); + expect(r.invertible).toBe(false); + }); +}); From e434d3573dbf48bef3c36f99658052becf1bd714 Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Wed, 1 Jul 2026 11:55:18 +0200 Subject: [PATCH 12/17] set-default: trace upstream result production + harden against wrong writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extension: when a default resolves to .results. with no static echo, traceUpstreamResultProduction statically inspects 's own liquid to see how it produces that (possibly dynamic) tag — matching its {% result %} whose tag literal/capture-pattern covers the tag — and auto-inverts when the value is a direct custom; otherwise it reports the actual production (e.g. 'produces addition_12_2024 from a computed value [addition]') for an actionable chain. Fixes from the adversarial review (all false-invertible -> wrong-write risks): - Reject value-changing filters: a default/RHS with | times/round/currency/… is NOT invertible (only | default: is safe) — was silently stripped, writing the raw value so the default recomputed wrong. (2x critical) - Custom target must be exactly namespace.key; a field access (custom.ns.key.value) is refused, not truncated to the wrong custom. (critical/high) - Resolve the upstream period by OFFSET, not by end-date key (two periods can share an end_date), so we never write to the wrong period. (medium) Adds safety unit tests. 628 tests pass. --- lib/defaultSetter.js | 9 ++- lib/provenanceTracer.js | 121 +++++++++++++++++++++++++---- tests/lib/provenanceTracer.test.js | 27 +++++++ 3 files changed, 140 insertions(+), 17 deletions(-) diff --git a/lib/defaultSetter.js b/lib/defaultSetter.js index 88d27277..f53f7525 100644 --- a/lib/defaultSetter.js +++ b/lib/defaultSetter.js @@ -69,7 +69,7 @@ async function captureLight(url, handle, depth) { reconciliations: {}, }; } - return { handle, periodOrder, currentPeriodKey: periodOrder[0] ?? null, data }; + return { handle, periodOrder, periodIds: selected.map((p) => p.id), currentPeriodKey: periodOrder[0] ?? null, data }; } /** @@ -106,10 +106,11 @@ async function setDefault(url, inputPath, rawValue, opts = {}) { } // Resolve the upstream write target (same company, traced period + handle). + // Resolve the period by OFFSET, not by end-date key (two periods can share an + // end_date), so we never write to the wrong period. const t = trace.target; - const periodEntry = light.data.periods[t.periodKey]; - if (!periodEntry) return { invertible: false, handle, input: inputPath, trace: { ...trace, invertible: false, reason: `period ${t.periodKey} not captured` } }; - const periodId = periodEntry.periodId; + const periodId = light.periodIds[t.periodOffset]; + if (periodId == null) return { invertible: false, handle, input: inputPath, trace: { ...trace, invertible: false, reason: `period offset ${t.periodOffset} (${t.periodKey}) not captured` } }; const upstreamRecon = await SF.findReconciliationInWorkflows(firmId, t.handle, companyId, periodId); if (!upstreamRecon || !upstreamRecon.id) { return { invertible: false, handle, input: inputPath, trace: { ...trace, invertible: false, reason: `upstream template ${t.handle} not found in period ${t.periodKey}` } }; diff --git a/lib/provenanceTracer.js b/lib/provenanceTracer.js index 14454a4e..2926536f 100644 --- a/lib/provenanceTracer.js +++ b/lib/provenanceTracer.js @@ -1,5 +1,54 @@ +const fs = require("fs"); const stl = require("./stlLite"); +// Trace ONE hop into an upstream handle's own liquid to see how it produces a +// (possibly dynamically-named) result `tag`: statically match its {% result %} +// whose tag (a literal, or a variable captured as PREFIX_{{…}}) covers `tag`, then +// analyse the result's VALUE. Auto-invertible only when the value is a direct +// custom; otherwise report what it actually is (a computed value) so the caller can +// surface a deeper, actionable chain. Never guesses. No execution, no extra API. +function traceUpstreamResultProduction(handle, tag, getScope) { + const scope = getScope ? getScope(handle) : null; + if (!scope || !scope.involvedFiles) return null; + let liquid = ""; + for (const f of scope.involvedFiles) { try { liquid += "\n" + fs.readFileSync(f, "utf8"); } catch { /* skip */ } } + + // capture VAR -> a regex covering its literal text with {{…}} as wildcards. + const captures = {}; + const capRe = /\{%-?\s*capture\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*-?%\}([\s\S]*?)\{%-?\s*endcapture\s*-?%\}/g; + let cm; + while ((cm = capRe.exec(liquid)) !== null) { + const esc = cm[2].trim().replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\{\\\{[\s\S]*?\\\}\\\}/g, ".+"); + captures[cm[1]] = "^" + esc + "$"; + } + + const resRe = /\{%-?\s*result\s+(\S+)\s+([\s\S]+?)\s*-?%\}/g; + let rm; + while ((rm = resRe.exec(liquid)) !== null) { + const tagRef = rm[1]; + const valueRef = rm[2].trim(); + let pattern = null; + const q = tagRef.match(/^['"](.*)['"]$/); + if (q) pattern = "^" + q[1].replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + "$"; + else { + const varName = tagRef.replace(/^\[|\]$/g, ""); + if (captures[varName]) pattern = captures[varName]; + } + if (!pattern) continue; + let matches = false; + try { matches = new RegExp(pattern).test(tag); } catch { matches = false; } + if (!matches) continue; + + const vHead = valueRef.split("|")[0].trim(); + const mCustom = vHead.match(/^custom\.([a-z0-9_]+)\.([a-z0-9_]+)$/i); + if (mCustom) return { invertible: true, target: { handle, namespace: mCustom[1], key: mCustom[2] }, via: `result-production:${tag}`, valueExpr: valueRef }; + const mDyn = vHead.match(/^custom\.([a-z0-9_]+)\.\[/i); + if (mDyn) return { invertible: false, reason: `${handle} produces ${tag} from custom.${mDyn[1]}. — set that custom directly for the matching period/key`, valueExpr: valueRef }; + return { invertible: false, reason: `${handle} produces ${tag} from a computed value (${valueRef.trim()}) — not a single settable custom; set ${handle}'s underlying inputs`, valueExpr: valueRef }; + } + return null; +} + /** * Reverse provenance: given a target input whose value comes from a DEFAULT, find * the single upstream CUSTOM that should be set so the default becomes the desired @@ -30,6 +79,35 @@ function parseUpstreamRef(head) { return { periodOffset, handle: m[1], kind: m[2].toLowerCase(), tail: m[3] || "" }; } +// Split on top-level | (respecting [] and quotes). +function splitPipes(s) { + const out = []; + let buf = "", depth = 0, q = null; + for (const ch of s) { + if (q) { buf += ch; if (ch === q) q = null; continue; } + if (ch === '"' || ch === "'") { q = ch; buf += ch; continue; } + if (ch === "[") depth++; + else if (ch === "]") depth--; + if (ch === "|" && depth === 0) { out.push(buf); buf = ""; continue; } + buf += ch; + } + out.push(buf); + return out; +} + +// The head of an expression, ONLY if it carries no value-changing filter (a +// `| times:2`, `| currency`, `| round`, … makes the default non-invertible — the +// raw upstream custom is not what the field ends up showing). `default` is the one +// safe filter (it no-ops once the upstream custom is set). Returns null if unsafe. +function safeHead(expr) { + const segs = splitPipes(String(expr)); + for (let i = 1; i < segs.length; i++) { + const fname = segs[i].trim().split(/[:\s]/)[0].toLowerCase(); + if (fname !== "default") return null; + } + return segs[0].trim(); +} + // Resolve ".a.[b].c" against an env into ["a", , "c"] (dynamic keys first). function resolveSegments(tail, env, ctx) { const segs = []; @@ -60,7 +138,12 @@ function classify(ref, segs, deep, getScope) { const periodKey = (deep.periodOrder || [])[ref.periodOffset] || null; const base = { handle: ref.handle, periodOffset: ref.periodOffset, periodKey }; if (ref.kind === "custom") { - if (!segs || segs.length < 2) return { invertible: false, reason: "custom reference key could not be resolved" }; + // A settable custom is exactly namespace.key. More segments (e.g. + // custom.ns.key.value) is a FIELD access on the custom, not the custom itself + // — writing custom.ns.key would not change what the field reads, so refuse. + if (!segs || segs.length !== 2) { + return { invertible: false, reason: "reference is not a plain custom.namespace.key (dynamic/field access or unresolved) — not auto-invertible" }; + } return { invertible: true, target: { ...base, namespace: segs[0], key: segs[1] }, via: "direct-custom" }; } // results @@ -68,16 +151,20 @@ function classify(ref, segs, deep, getScope) { if (!tag) return { invertible: false, reason: "result tag could not be resolved" }; const scope = getScope ? getScope(ref.handle) : null; const echoed = scope && scope.resultEchoes && scope.resultEchoes[tag]; - if (!echoed) { - return { - invertible: false, - reason: `default comes from ${ref.handle}.results.${tag}, a computed result with no static custom echo — set ${ref.handle}'s inputs directly (mode 3) instead`, - upstreamResult: { handle: ref.handle, tag, periodKey }, - }; + if (echoed) { + const parts = String(echoed).split("."); // custom.ns.key + if (parts.length >= 3) return { invertible: true, target: { ...base, namespace: parts[1], key: parts.slice(2).join(".") }, via: `result-echo:${tag}` }; } - const parts = String(echoed).split("."); // custom.ns.key - if (parts.length < 3) return { invertible: false, reason: `result ${tag} echoes ${echoed}, which is not a custom` }; - return { invertible: true, target: { ...base, namespace: parts[1], key: parts.slice(2).join(".") }, via: `result-echo:${tag}` }; + // One hop deeper: how does actually produce this (possibly dynamic) result? + const prod = traceUpstreamResultProduction(ref.handle, tag, getScope); + if (prod && prod.invertible) { + return { invertible: true, target: { ...base, namespace: prod.target.namespace, key: prod.target.key }, via: prod.via }; + } + return { + invertible: false, + reason: (prod && prod.reason) || `default comes from ${ref.handle}.results.${tag}, a computed result with no static custom echo — set ${ref.handle}'s inputs directly (mode 3) instead`, + upstreamResult: { handle: ref.handle, tag, periodKey, valueExpr: prod && prod.valueExpr }, + }; } /** @@ -95,10 +182,13 @@ function traceDefault(input, involvedLiquid, ctx, deep, getScope) { if (!def) return { invertible: false, reason: "input has no default", chain }; // Direct reference in the default expression itself. - const directHead = def.split("|")[0].trim(); + const directHead = safeHead(def); + if (directHead === null) { + return { invertible: false, reason: `default applies a value-changing filter (${def}) — not auto-invertible`, chain }; + } const directRef = parseUpstreamRef(directHead); if (directRef) { - const segs = resolveSegments(directRef.tail, ctx.__env || {}, ctx); + const segs = resolveSegments(directRef.tail, {}, ctx); chain.push(`${directRef.handle}.${directRef.kind}${directRef.tail}`); return { ...classify(directRef, segs, deep, getScope), chain }; } @@ -111,12 +201,17 @@ function traceDefault(input, involvedLiquid, ctx, deep, getScope) { stl.run(involvedLiquid, ctx, { onAssign: (name, rhs, value, env) => { if (name !== def) return; - const head = rhs.split("|")[0].trim(); + const head = safeHead(rhs); + if (head === null) { captured = { ref: null, rhs, filtered: true }; return; } const ref = parseUpstreamRef(head); captured = { ref, segs: ref ? resolveSegments(ref.tail, env, ctx) : null, rhs }; }, }); if (!captured) return { invertible: false, reason: `could not find an assignment for ${def} in the template`, chain }; + if (captured.filtered) { + chain.push(`${def} = ${captured.rhs}`); + return { invertible: false, reason: `${def} applies a value-changing filter (${captured.rhs.trim()}) — not auto-invertible`, chain }; + } if (!captured.ref) { chain.push(`${def} = ${captured.rhs}`); return { invertible: false, reason: `${def} is computed (${captured.rhs.trim()}), not a single cross-template reference`, chain }; diff --git a/tests/lib/provenanceTracer.test.js b/tests/lib/provenanceTracer.test.js index af4c5f54..9158e30d 100644 --- a/tests/lib/provenanceTracer.test.js +++ b/tests/lib/provenanceTracer.test.js @@ -55,3 +55,30 @@ describe("provenanceTracer.traceDefault", () => { expect(r.invertible).toBe(false); }); }); + +describe("provenanceTracer.traceDefault — safety (never a wrong write)", () => { + it("refuses a direct default with a value-changing filter (times)", () => { + const r = traceDefault({ input: "custom.x.y", default: "period.reconciliations.other.custom.ns.key | times: 2" }, "", ctx, deep, () => null); + expect(r.invertible).toBe(false); + expect(r.reason).toMatch(/value-changing filter/); + }); + + it("refuses a variable default whose assignment has a value-changing filter", () => { + const liquid = `{% assign foo = period.reconciliations.other.custom.ns.key | times: 2 %}`; + const r = traceDefault({ input: "custom.x.y", default: "foo" }, liquid, ctx, deep, () => null); + expect(r.invertible).toBe(false); + expect(r.reason).toMatch(/value-changing filter/); + }); + + it("still inverts through a harmless default: filter", () => { + const r = traceDefault({ input: "custom.x.y", default: "period.reconciliations.other.custom.ns.key | default:0" }, "", ctx, deep, () => null); + expect(r.invertible).toBe(true); + expect(r.target).toMatchObject({ namespace: "ns", key: "key" }); + }); + + it("refuses a custom field-access ref (custom.ns.key.value), not a plain custom", () => { + const r = traceDefault({ input: "custom.x.y", default: "period.reconciliations.other.custom.ns.key.value" }, "", ctx, deep, () => null); + expect(r.invertible).toBe(false); + expect(r.reason).toMatch(/not a plain custom/); + }); +}); From c99f01fbb4a358f91c73f19bb848efcbd2c3afad Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Wed, 1 Jul 2026 12:42:42 +0200 Subject: [PATCH 13/17] describe-inputs: resolve effective values from indirect (adjacent) result echoes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many templates echo an input as a result through a variable (`{% assign v = custom.ns.key | default:… | currency %}{% result 'tag' v %}`), not a direct `{% result 'tag' custom.ns.key %}`. Previously those inputs were flagged `unavailable` even though their effective value sits in the live results. parseAdjacentEchoes now attributes such a result to the nearest preceding input when the tag relates to the key (and stays on the same side of the 2026 split), so the value is read straight from the results table — no silverfin-ls, no re-render. On 2018_275_A_liquidationreserve this fills 19 of 38 inputs from the API (taxable_yearN, withdrawal_N, tax_N …) that were all `unavailable` before. --- lib/inputDescriber.js | 50 ++++++++++++++++++++++++++++---- tests/lib/inputDescriber.test.js | 38 +++++++++++++++++++++++- 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/lib/inputDescriber.js b/lib/inputDescriber.js index 5ad355b6..3741743e 100644 --- a/lib/inputDescriber.js +++ b/lib/inputDescriber.js @@ -64,6 +64,41 @@ function parseResultEchoes(liquid) { return byCustomPath; } +// Indirect / adjacent echoes: a `{% result 'tag' %}` whose value is NOT a +// direct `custom.ns.key` (it echoes a variable derived from the input, e.g. +// `{% assign v = custom.ns.key | default:… | currency %}{% result 'tag' v %}`). +// Attribute such a result to the nearest preceding `{% input custom.ns.key %}` when +// the tag textually relates to the key (and stays on the same side of the 2026 +// split), so the effective value can be read straight from the live results — no +// silverfin-ls and no re-render. Returns Map(inputPath -> resultTag). +function parseAdjacentEchoes(liquid) { + const inputRe = /\{%-?\s*input\s+(custom\.[a-zA-Z0-9_.]+)/g; + const resultRe = /\{%-?\s*result\s+['"]([a-zA-Z0-9_]+)['"]\s+([^%]*?)\s*-?%\}/g; + const inputs = []; + let m; + while ((m = inputRe.exec(liquid)) !== null) inputs.push({ path: m[1], pos: m.index }); + const byPath = new Map(); + while ((m = resultRe.exec(liquid)) !== null) { + const tag = m[1]; + const valueExpr = m[2].trim(); + if (/^custom\.[a-zA-Z0-9_.]+$/.test(valueExpr)) continue; // direct echo — handled elsewhere + let owner = null; + for (const inp of inputs) { + if (inp.pos < m.index) owner = inp; + else break; + } + if (!owner) continue; + const segs = owner.path.split("."); + const ns = segs[1] || ""; + const key = segs.slice(2).join("."); + if (!key) continue; + if (!(tag === key || tag.includes(key) || key.includes(tag))) continue; + if (/from_2026/.test(tag) !== /from_2026/.test(ns)) continue; // don't cross the 2026 split + if (!byPath.has(owner.path)) byPath.set(owner.path, tag); + } + return byPath; +} + // Build { "namespace.key": value } from the API custom array. function storedMapFromCustom(customArray) { const map = {}; @@ -108,6 +143,7 @@ async function describeInputs(url, opts = {}) { const liquid = combineLiquid(template); const inputs = parseInputs(liquid); const echoes = parseResultEchoes(liquid); + const adjacentEchoes = parseAdjacentEchoes(liquid); const customResponse = await SF.getReconciliationCustom("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); const resultsResponse = await SF.getReconciliationResults("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); @@ -124,11 +160,15 @@ async function describeInputs(url, opts = {}) { effective = stored[nsKey]; effectiveSource = "stored"; } else { - const tag = echoes.get(input.path); + const directTag = echoes.get(input.path); + const adjacentTag = adjacentEchoes.get(input.path); const literal = literalValue(input.default); - if (tag && Object.hasOwn(results, tag)) { - effective = results[tag]; - effectiveSource = `result:${tag}`; + if (directTag && Object.hasOwn(results, directTag)) { + effective = results[directTag]; + effectiveSource = `result:${directTag}`; + } else if (adjacentTag && Object.hasOwn(results, adjacentTag)) { + effective = results[adjacentTag]; + effectiveSource = `result:${adjacentTag} (adjacent echo)`; } else if (literal !== undefined) { effective = literal; effectiveSource = "literal-default"; @@ -176,4 +216,4 @@ async function describeInputs(url, opts = {}) { return output; } -module.exports = { describeInputs, parseInputs, parseResultEchoes, combineLiquid, storedMapFromCustom, literalValue }; +module.exports = { describeInputs, parseInputs, parseResultEchoes, parseAdjacentEchoes, combineLiquid, storedMapFromCustom, literalValue }; diff --git a/tests/lib/inputDescriber.test.js b/tests/lib/inputDescriber.test.js index e67efd1d..652944b9 100644 --- a/tests/lib/inputDescriber.test.js +++ b/tests/lib/inputDescriber.test.js @@ -6,7 +6,43 @@ jest.mock("consola"); const SF = require("../../lib/api/sfApi"); const Utils = require("../../lib/utils/liquidTestUtils"); const { ReconciliationText } = require("../../lib/templates/reconciliationText"); -const { describeInputs, parseInputs, parseResultEchoes, storedMapFromCustom, literalValue } = require("../../lib/inputDescriber"); +const { describeInputs, parseInputs, parseResultEchoes, parseAdjacentEchoes, storedMapFromCustom, literalValue } = require("../../lib/inputDescriber"); + +describe("parseAdjacentEchoes (indirect result echoes)", () => { + it("attributes a result echoing a variable to the nearest preceding input", () => { + const liquid = ` + {% input custom.liquidation.taxable_year1 as:currency default:some_var %} + {% assign v = custom.liquidation.taxable_year1 | default:some_var | currency %} + {% result 'taxable_year1_begin' v %}`; + const map = parseAdjacentEchoes(liquid); + expect(map.get("custom.liquidation.taxable_year1")).toBe("taxable_year1_begin"); + }); + + it("keeps the 2026 split: from_2026 input -> from_2026 tag only", () => { + const liquid = ` + {% input custom.liquidation_from_2026.taxable_year1 as:currency default:v %} + {% result 'taxable_year1_begin_from_2026' vv %}`; + const map = parseAdjacentEchoes(liquid); + expect(map.get("custom.liquidation_from_2026.taxable_year1")).toBe("taxable_year1_begin_from_2026"); + }); + + it("does NOT cross the 2026 split (before input, from_2026 tag)", () => { + const liquid = ` + {% input custom.liquidation.taxable_year1 as:currency default:v %} + {% result 'taxable_year1_begin_from_2026' vv %}`; + expect(parseAdjacentEchoes(liquid).has("custom.liquidation.taxable_year1")).toBe(false); + }); + + it("ignores DIRECT echoes (those are parseResultEchoes' job)", () => { + const liquid = `{% input custom.a.b %}{% result 'b' custom.a.b %}`; + expect(parseAdjacentEchoes(liquid).has("custom.a.b")).toBe(false); + }); + + it("requires the tag to relate to the input key", () => { + const liquid = `{% input custom.a.foo %}{% result 'totally_unrelated' some_var %}`; + expect(parseAdjacentEchoes(liquid).has("custom.a.foo")).toBe(false); + }); +}); describe("inputDescriber pure helpers", () => { describe("parseInputs", () => { From 6dee9872cbb90adece4e2ea0d1b40f2e23dd9bdd Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Wed, 1 Jul 2026 12:45:21 +0200 Subject: [PATCH 14/17] silverfin-ls: auto-detect, timeout, precise errors, and a doctor command - getDataScope now auto-detects the analyzer (SILVERFIN_LS_CMD -> silverfin-ls on PATH -> npx --no-install silverfin-ls) instead of only defaulting to a bare binary, so it works without manual SILVERFIN_LS_CMD when installed. - Every invocation has a 30s timeout, so an outdated silverfin-ls (which would ignore `data-scope` and start its LSP server) can no longer hang the CLI. - Errors are precise: what was tried + how to install/verify + a `silverfin doctor` pointer. - New `silverfin doctor` command probes silverfin-ls end-to-end against a throwaway template and reports OK or the exact failure + install steps. --- bin/cli.js | 15 +++++++ lib/dataScope.js | 107 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 97 insertions(+), 25 deletions(-) diff --git a/bin/cli.js b/bin/cli.js index fb4333ab..8a9493f6 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -604,6 +604,21 @@ program } }); +// DOCTOR — verify the silverfin-ls integration +program + .command("doctor") + .description("Check that silverfin-ls is installed and supports `data-scope` (required by manifest, describe-inputs --resolve/--compute, and set-default)") + .action(() => { + const result = dataScope.checkSilverfinLs(); + if (result.ok) { + consola.success(`silverfin-ls is working via "${result.command}" — data-scope OK (probe resolved ${result.sample.crossTemplate} cross-template refs).`); + } else { + consola.error(`silverfin-ls is NOT working${result.command ? ` (via "${result.command}")` : ""}: ${result.error}`); + console.log("\n" + dataScope.installHelp()); + process.exitCode = 1; + } + }); + // MANIFEST — static data scope of a template (drives deep-capture + default resolution) program .command("manifest") diff --git a/lib/dataScope.js b/lib/dataScope.js index eafe82be..090dffcb 100644 --- a/lib/dataScope.js +++ b/lib/dataScope.js @@ -1,4 +1,6 @@ const { execFileSync } = require("child_process"); +const fs = require("fs"); +const os = require("os"); const path = require("path"); const { consola } = require("consola"); @@ -6,46 +8,101 @@ const { consola } = require("consola"); * Static data-scope analysis is delegated to silverfin-ls (the maintained * tree-sitter language server), so we don't duplicate STL parsing here. * - * `silverfin-ls data-scope ` prints the template's data scope as - * JSON: ownCustoms, crossTemplate {results, customs}, periodDrop, - * priorPeriodDepth, companyDrop, accounts, resultEchoes, involvedFiles. + * `silverfin-ls data-scope ` prints the template's data scope as JSON. * - * The binary is `silverfin-ls` by default; set SILVERFIN_LS_CMD to point at a - * local/dev build (e.g. "/path/to/node /path/to/silverfin-ls/out/index.js"). + * Resolution order (auto-detect): SILVERFIN_LS_CMD if set, else `silverfin-ls` on + * PATH, else `npx --no-install silverfin-ls`. Every call uses a timeout so an old + * silverfin-ls (which would ignore `data-scope` and start its LSP server) can't hang. */ +const TIMEOUT_MS = 30000; +const MAX_BUFFER = 64 * 1024 * 1024; + function resolveMainPath(handle) { return path.resolve(process.cwd(), "reconciliation_texts", handle, "main.liquid"); } -function lsCommand() { - // SILVERFIN_LS_CMD may be a single binary or "node /path/out/index.js". - const raw = (process.env.SILVERFIN_LS_CMD || "silverfin-ls").trim(); - const parts = raw.split(/\s+/); - return { bin: parts[0], prefixArgs: parts.slice(1) }; +// Ordered list of candidate invocations, each as [bin, ...prefixArgs]. +function candidateCommands() { + const env = (process.env.SILVERFIN_LS_CMD || "").trim(); + if (env) return [env.split(/\s+/)]; + return [["silverfin-ls"], ["npx", "--no-install", "silverfin-ls"]]; +} + +function installHelp() { + return [ + "silverfin-ls provides the static data-scope analysis for `manifest`, `describe-inputs --resolve/--compute` and `set-default`.", + "Install it and make sure it supports `data-scope`:", + " npm install -g silverfin-ls # or: npx silverfin-ls", + "Then verify with: silverfin doctor", + "Or point at a specific build: SILVERFIN_LS_CMD=\"node /path/to/silverfin-ls/out/index.js\" silverfin manifest -h ", + ].join("\n"); +} + +// Run `data-scope ` against a candidate; returns { ok, stdout } or throws. +function runDataScope(cmd, arg) { + const [bin, ...prefix] = cmd; + return execFileSync(bin, [...prefix, "data-scope", arg], { + encoding: "utf8", + maxBuffer: MAX_BUFFER, + timeout: TIMEOUT_MS, + stdio: ["ignore", "pipe", "ignore"], + }); } /** * @param {String} handle reconciliation handle (its main.liquid is read from cwd) - * @returns {Object|null} the data scope, or null if silverfin-ls is unavailable + * @returns {Object|null} the data scope, or null (with a precise error) if unavailable */ function getDataScope(handle) { const main = resolveMainPath(handle); - const { bin, prefixArgs } = lsCommand(); + const tried = []; + for (const cmd of candidateCommands()) { + try { + return JSON.parse(runDataScope(cmd, main)); + } catch (error) { + tried.push(cmd.join(" ")); + if (error.code === "ENOENT") continue; // binary not found — try the next candidate + // It ran but failed (non-zero, timeout, or bad JSON): don't keep guessing. + const why = error.signal === "SIGTERM" || error.killed ? "timed out (an outdated silverfin-ls without `data-scope` will hang)" : String(error.message).split("\n")[0]; + consola.error(`silverfin-ls failed via "${cmd.join(" ")}": ${why}\n\n${installHelp()}`); + return null; + } + } + consola.error(`Could not find a working silverfin-ls (tried: ${tried.join(", ")}).\n\n${installHelp()}`); + return null; +} + +/** + * Probe silverfin-ls end-to-end against a throwaway template, for `silverfin doctor`. + * @returns {Object} { ok, command?, sample?, error? } + */ +function checkSilverfinLs() { + let tmp; try { - const stdout = execFileSync(bin, [...prefixArgs, "data-scope", main], { - encoding: "utf8", - maxBuffer: 64 * 1024 * 1024, - stdio: ["ignore", "pipe", "ignore"], - }); - return JSON.parse(stdout); - } catch (error) { - consola.error( - `Could not get the data scope from silverfin-ls. Ensure silverfin-ls is installed and supports "data-scope" ` + - `(set SILVERFIN_LS_CMD to point at a local build). ${String(error.message).split("\n")[0]}` - ); - return null; + tmp = fs.mkdtempSync(path.join(os.tmpdir(), "sfls-")); + const dir = path.join(tmp, "reconciliation_texts", "probe"); + fs.mkdirSync(dir, { recursive: true }); + const main = path.join(dir, "main.liquid"); + fs.writeFileSync(main, "{{ period.reconciliations.foo.results.bar }}\n"); + const tried = []; + for (const cmd of candidateCommands()) { + try { + const scope = JSON.parse(runDataScope(cmd, main)); + return { ok: true, command: cmd.join(" "), sample: { handle: scope.handle, crossTemplate: Object.keys(scope.crossTemplate || {}).length } }; + } catch (error) { + tried.push(cmd.join(" ")); + if (error.code === "ENOENT") continue; + const why = error.signal === "SIGTERM" || error.killed ? "timed out (outdated silverfin-ls without `data-scope`)" : String(error.message).split("\n")[0]; + return { ok: false, command: cmd.join(" "), error: why }; + } + } + return { ok: false, error: `silverfin-ls not found (tried: ${tried.join(", ")})` }; + } finally { + if (tmp) { + try { fs.rmSync(tmp, { recursive: true, force: true }); } catch { /* ignore */ } + } } } -module.exports = { getDataScope, resolveMainPath }; +module.exports = { getDataScope, resolveMainPath, checkSilverfinLs, installHelp }; From ed6530d41a7db7c994ccd34c6fd27e90c528d9a2 Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Wed, 1 Jul 2026 13:14:54 +0200 Subject: [PATCH 15/17] changelog: doctor command + indirect/adjacent echo resolution --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 121ebbf1..41c20014 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +Added `doctor` command. It checks that silverfin-ls is installed and supports `data-scope` (required by `manifest`, `describe-inputs --resolve/--compute` and `set-default`) by probing it end-to-end against a throwaway template, and reports the working invocation or the exact failure with install steps. silverfin-ls is now auto-detected (`SILVERFIN_LS_CMD` → `silverfin-ls` on PATH → `npx --no-install silverfin-ls`), and every invocation has a timeout so an outdated silverfin-ls (which would ignore `data-scope` and start its LSP server) can no longer hang the CLI. + Added `set-default` command. Instead of overriding a field, it sets an input's **default** by writing the **upstream** custom the default derives from — so the field stays un-overridden and the value flows in as it does in the real case. Usage: `silverfin set-default -u --input custom.. --value X [--dry-run]` (run from your templates repo). It reverse-traces the default's provenance (via `lib/provenanceTracer.js` + the offline evaluator) to a single settable custom; it auto-proceeds when the default is auto-invertible (a direct cross-template custom, or a result that statically echoes a custom) and prints a change table (target · level · namespace.key · old → new · why) with a blast-radius note when >1 template reads that source. When the default is computed / branch-gated / a dynamic result (no single custom to set), it writes nothing and prints the provenance chain, pointing to the upstream template + period to set directly. Adds a shared `lib/changeReport.js` (change table + JSON) and requires silverfin-ls. Added `update-text-properties` command. It uploads custom text properties from a Liquid Test YAML file to a company file at company, period, reconciliation and account levels for the entries referenced in the test scenario. Usage: `silverfin update-text-properties -u -t `. Supports `--handle` for faster YAML file lookup, `--dry-run` to preview the payload, and `--yes` to skip the confirmation prompt. @@ -13,7 +15,7 @@ Added `capture` command. It captures a live company file's data as JSON. By defa Added `set-custom` and `delete-custom` commands for ad-hoc manipulation of a live company file's custom data. `set-custom -u --namespace --key --value ` sets a single custom (value JSON-parsed when possible); `delete-custom` soft-deletes it (value null). Both support `--level company|period|reconciliation|account` (inferred from the URL by default), `--handle`/`--account` targeting, `--file` for batch, `--dry-run` to print the exact properties that would be written without sending the request, and `--yes` to skip the confirmation prompt. -Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled only from certain sources — the stored value, the live result where the template directly exposes the input (`{% result 'tag' custom.ns.key %}`), or a literal default — and any input whose effective value is not derivable from the API is flagged (it is only resolvable in the rendered UI). Pass `--resolve` to additionally fill inputs whose default is a direct reference to data created elsewhere (a cross-template result/custom, a period/company custom, optionally a prior period via `period.minus_Ny...`); these are read straight from a targeted deep capture of the live company file (only the referenced handles/periods are fetched). Add `--compute` (used with `--resolve`) to additionally compute variable-defaults that reduce to a lookup into captured live data — a cross-template result/custom indexed by a date-derived dynamic key, an account-range aggregation, or a simple calculation thereof — using a bounded, safe offline STL evaluator (`lib/stlLite.js`) that runs over the template's involved liquid against a deep-capture context. The evaluator supports assign / capture / if-elsif-else / for-loops, dynamic-key indexing, account `range:` filtering + `.value`/`.count` aggregation, safe infix arithmetic, and a whitelist of filters (date, default, split, size, currency/percentage as numeric pass-throughs, at_least/at_most, round, …). It NEVER fabricates: anything whose operands don't fully resolve from captured data (unsupported filter, `MAX()`, `opening_value`, an unresolved operand) stays flagged. The deep capture now also fetches `period.accounts` (value/number/type, when the template reads accounts) and per-period drop scalars, and the context exposes `period.fiscal_year`/`year_start_date`/`exists` and flattened `company.*`. Computed values are labelled `computed: (offline; validate vs live)` and must be validated against a live render before being trusted. `--compute` is heavier than `--resolve` alone (it deep-captures the template scope, incl. accounts). Requires silverfin-ls (for the scope). +Added `describe-inputs` command. It lists a reconciliation's custom inputs with their declared defaults, stored values and live effective values, plus the template's results, as JSON. Usage: `silverfin describe-inputs -u ` (run from your templates repo). Effective values are filled from certain sources — the stored value, the live result where the template exposes the input (**directly** via `{% result 'tag' custom.ns.key %}`, or **indirectly** where a result echoes a variable derived from the input, e.g. `{% assign v = custom.ns.key | default:… | currency %}{% result 'tag' v %}`, attributed to the nearest preceding input on the same side of any 2026 split), or a literal default — and any input whose effective value is not derivable is flagged (only resolvable in the rendered UI). The indirect/adjacent echo means most inputs resolve straight from the live results **without silverfin-ls**. Pass `--resolve` to additionally fill inputs whose default is a direct reference to data created elsewhere (a cross-template result/custom, a period/company custom, optionally a prior period via `period.minus_Ny...`); these are read straight from a targeted deep capture of the live company file (only the referenced handles/periods are fetched). Add `--compute` (used with `--resolve`) to additionally compute variable-defaults that reduce to a lookup into captured live data — a cross-template result/custom indexed by a date-derived dynamic key, an account-range aggregation, or a simple calculation thereof — using a bounded, safe offline STL evaluator (`lib/stlLite.js`) that runs over the template's involved liquid against a deep-capture context. The evaluator supports assign / capture / if-elsif-else / for-loops, dynamic-key indexing, account `range:` filtering + `.value`/`.count` aggregation, safe infix arithmetic, and a whitelist of filters (date, default, split, size, currency/percentage as numeric pass-throughs, at_least/at_most, round, …). It NEVER fabricates: anything whose operands don't fully resolve from captured data (unsupported filter, `MAX()`, `opening_value`, an unresolved operand) stays flagged. The deep capture now also fetches `period.accounts` (value/number/type, when the template reads accounts) and per-period drop scalars, and the context exposes `period.fiscal_year`/`year_start_date`/`exists` and flattened `company.*`. Computed values are labelled `computed: (offline; validate vs live)` and must be validated against a live render before being trusted. `--compute` is heavier than `--resolve` alone (it deep-captures the template scope, incl. accounts). Requires silverfin-ls (for the scope). Added `manifest` command. It prints a reconciliation template's static data scope: own customs, cross-template results/customs, period drop and prior-period depth, company drop, accounts, result echoes and involved files. Usage: `silverfin manifest -h ` or `-u ` (run from your templates repo). The STL analysis is delegated to silverfin-ls (the maintained tree-sitter language server) rather than duplicated here; set `SILVERFIN_LS_CMD` to point at a specific silverfin-ls binary/build. It is the first step of a deep-capture + default-resolution pipeline for resolving effective default values without a browser. From d5986ddd59403f11d05501674500054d304e6822 Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Wed, 1 Jul 2026 13:59:38 +0200 Subject: [PATCH 16/17] Read a sibling template by --handle + surface line-item collections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inspecting an upstream/underlying template (the ones that feed another's defaults) previously required its URL and didn't cover collection inputs, so an agent had to fall back to reading liquid. Now: - lib/targetResolver.js resolves a handle to its reconciliation instance id (and workflow id + URL) in the same company/period as a given URL. - get-results / describe-inputs / capture accept --handle to act on that sibling; new 'resolve-handle' command prints the resolved id + URL. - describe-inputs surfaces line-item COLLECTIONS (fori over custom.) as a 'collections' section with the item fields and the live items. Verified live on firm 1355: describe-inputs --handle liquidation_reserve now shows its custom.items line-items (date/amount/description) — the exact data needed to change a computed addition at its source. 636 tests pass. --- CHANGELOG.md | 2 + bin/cli.js | 26 ++++++++++-- lib/dataCapture.js | 22 ++++++++++ lib/inputDescriber.js | 70 ++++++++++++++++++++++++++++---- lib/resultsReader.js | 25 +++++++++++- lib/targetResolver.js | 70 ++++++++++++++++++++++++++++++++ tests/lib/inputDescriber.test.js | 36 +++++++++++++++- 7 files changed, 237 insertions(+), 14 deletions(-) create mode 100644 lib/targetResolver.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 41c20014..fa63e57a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +Added `--handle ` to the read commands (`get-results`, `describe-inputs`, `capture`) and a new `resolve-handle` command. These let you inspect a **sibling** reconciliation — an upstream/underlying template that feeds another's defaults — by handle in the same company/period, without reconstructing its URL (`resolve-handle` prints the resolved instance id, workflow id and URL). `describe-inputs` also now surfaces **line-item collections** (`{% fori item in custom. %}{% input item. %}`) as a `collections` section listing the item fields and the live items, so collection-driven inputs (e.g. a working paper's dated line amounts) are visible instead of appearing as "no inputs". + Added `doctor` command. It checks that silverfin-ls is installed and supports `data-scope` (required by `manifest`, `describe-inputs --resolve/--compute` and `set-default`) by probing it end-to-end against a throwaway template, and reports the working invocation or the exact failure with install steps. silverfin-ls is now auto-detected (`SILVERFIN_LS_CMD` → `silverfin-ls` on PATH → `npx --no-install silverfin-ls`), and every invocation has a timeout so an outdated silverfin-ls (which would ignore `data-scope` and start its LSP server) can no longer hang the CLI. Added `set-default` command. Instead of overriding a field, it sets an input's **default** by writing the **upstream** custom the default derives from — so the field stays un-overridden and the value flows in as it does in the real case. Usage: `silverfin set-default -u --input custom.. --value X [--dry-run]` (run from your templates repo). It reverse-traces the default's provenance (via `lib/provenanceTracer.js` + the offline evaluator) to a single settable custom; it auto-proceeds when the default is auto-invertible (a direct cross-template custom, or a result that statically echoes a custom) and prints a change table (target · level · namespace.key · old → new · why) with a blast-radius note when >1 template reads that source. When the default is computed / branch-gated / a dynamic result (no single custom to set), it writes nothing and prints the provenance chain, pointing to the upstream template + period to set directly. Adds a shared `lib/changeReport.js` (change table + JSON) and requires silverfin-ls. diff --git a/bin/cli.js b/bin/cli.js index 8a9493f6..29f7b7a7 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -27,6 +27,7 @@ const textPropertyUtils = require("../lib/utils/textPropertyUtils"); const liquidTestUtils = require("../lib/utils/liquidTestUtils"); const customWriter = require("../lib/customWriter"); const defaultSetter = require("../lib/defaultSetter"); +const targetResolver = require("../lib/targetResolver"); const firmIdDefault = cliUtils.loadDefaultFirmId(); cliUtils.handleUncaughtErrors(); @@ -540,9 +541,10 @@ program .command("get-results") .description("Fetch the computed results and custom data of a reconciliation or account in a live company file, printed as JSON") .requiredOption("-u, --url ", "Specify the full Silverfin URL of the reconciliation/account in the company file (mandatory)") + .option("--handle ", "Read a sibling reconciliation by handle in the same company/period (instead of the URL target)") .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") .action(async (options) => { - const data = await resultsReader.fetchResults(options.url); + const data = await resultsReader.fetchResults(options.url, { handle: options.handle }); if (!data) { process.exitCode = 1; return; @@ -557,15 +559,32 @@ program } }); +// RESOLVE-HANDLE — resolve a sibling reconciliation's instance id + URL from a handle +program + .command("resolve-handle") + .description("Resolve a reconciliation handle to its instance id and URL in the same company/period as the given URL, so you can point commands at a sibling (upstream) template") + .requiredOption("-u, --url ", "A Silverfin URL identifying the target company/period (mandatory)") + .requiredOption("--handle ", "The reconciliation handle to resolve (mandatory)") + .action(async (options) => { + const target = await targetResolver.resolveReconciliationTarget(options.url, options.handle); + if (target.error) { + consola.error(target.error); + process.exitCode = 1; + return; + } + console.log(JSON.stringify({ handle: target.handle, reconciliationId: target.reconciliationId, workflowId: target.workflowId, periodId: target.ledgerId, url: target.url }, null, 2)); + }); + // CAPTURE — snapshot a live company file's data as JSON program .command("capture") .description("Capture a live company file's data as JSON. Default: the template at the URL and its dependencies (scoped). Use --full to capture company/period/reconciliation customs and results across all periods") .requiredOption("-u, --url ", "Specify the full Silverfin URL of the reconciliation/account in the company file (mandatory)") .option("--full", "Capture the whole company file (all periods, workflows, reconciliations) instead of just the template's scope (optional)", false) + .option("--handle ", "Snapshot a sibling reconciliation by handle in the same company/period (its full custom incl. line-item collections + results)") .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") .action(async (options) => { - const data = await dataCapture.capture(options.url, { full: options.full }); + const data = await dataCapture.capture(options.url, { full: options.full, handle: options.handle }); if (!data) { process.exitCode = 1; return; @@ -587,9 +606,10 @@ program .requiredOption("-u, --url ", "Specify the full Silverfin URL of the reconciliation in the company file (mandatory)") .option("--resolve", "Resolve `unavailable` defaults that reference data created elsewhere, via a targeted deep capture of the live company file (extra API calls). Requires silverfin-ls", false) .option("--compute", "With --resolve: additionally compute variable-defaults that reduce to a lookup into captured live data, using a bounded offline STL evaluator. Heavier (deep-captures the template scope); values are labelled `computed:… (offline; validate vs live)` and MUST be validated against a live render", false) + .option("--handle ", "Describe a sibling reconciliation by handle in the same company/period (instead of the URL target)") .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") .action(async (options) => { - const data = await inputDescriber.describeInputs(options.url, { resolve: options.resolve, compute: options.compute }); + const data = await inputDescriber.describeInputs(options.url, { resolve: options.resolve, compute: options.compute, handle: options.handle }); if (!data) { process.exitCode = 1; return; diff --git a/lib/dataCapture.js b/lib/dataCapture.js index 9868a9d3..dc8f53ea 100644 --- a/lib/dataCapture.js +++ b/lib/dataCapture.js @@ -1,6 +1,7 @@ const SF = require("./api/sfApi"); const Utils = require("./utils/liquidTestUtils"); const liquidTestGenerator = require("./liquidTestGenerator"); +const { resolveReconciliationTarget } = require("./targetResolver"); const { consola } = require("consola"); const PER_PAGE = 200; @@ -24,9 +25,30 @@ const MAX_PAGES = 50; * @returns {Promise} */ async function capture(url, opts = {}) { + if (opts.handle) return captureHandle(url, opts.handle); return opts.full ? captureFull(url) : captureScoped(url); } +// --handle: a targeted snapshot of a SIBLING reconciliation (its full custom — +// including line-item collections — and results), from the URL's company/period. +async function captureHandle(url, handle) { + const target = await resolveReconciliationTarget(url, handle); + if (target.error) { + consola.error(target.error); + return null; + } + const customResponse = await SF.getReconciliationCustom("firm", target.firmId, target.companyId, target.ledgerId, target.reconciliationId); + const resultsResponse = await SF.getReconciliationResults("firm", target.firmId, target.companyId, target.ledgerId, target.reconciliationId); + return { + mode: "handle", + handle: target.handle, + reconciliationId: target.reconciliationId, + url: target.url, + custom: customResponse?.data ?? null, + results: resultsResponse?.data ?? null, + }; +} + async function captureScoped(url) { const built = await liquidTestGenerator.buildLiquidTest(url, "capture", true); if (!built) { diff --git a/lib/inputDescriber.js b/lib/inputDescriber.js index 3741743e..aee96c96 100644 --- a/lib/inputDescriber.js +++ b/lib/inputDescriber.js @@ -1,6 +1,7 @@ const SF = require("./api/sfApi"); const Utils = require("./utils/liquidTestUtils"); const { ReconciliationText } = require("./templates/reconciliationText"); +const { resolveReconciliationTarget } = require("./targetResolver"); const { consola } = require("consola"); /** @@ -99,6 +100,48 @@ function parseAdjacentEchoes(liquid) { return byPath; } +// Detect line-item COLLECTION inputs: `{% fori item in items %}{% input item.field %}…` +// where the collection is `custom.` (directly, or via `{% assign items = custom. %}`). +// Returns [{ namespace, collection, loopVar, fields }]. +function parseForiCollections(liquid) { + const varToNs = {}; + const assignRe = /\{%-?\s*assign\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*custom\.([a-zA-Z0-9_]+)/g; + let m; + while ((m = assignRe.exec(liquid)) !== null) varToNs[m[1]] = m[2]; + + const collections = []; + const foriRe = /\{%-?\s*fori\s+([a-zA-Z_][a-zA-Z0-9_]*)\s+in\s+([a-zA-Z_][a-zA-Z0-9_.]*)\s*-?%\}([\s\S]*?)\{%-?\s*endfori\s*-?%\}/g; + while ((m = foriRe.exec(liquid)) !== null) { + const loopVar = m[1]; + const source = m[2]; + const body = m[3]; + const direct = source.match(/^custom\.([a-zA-Z0-9_]+)/); + const ns = direct ? direct[1] : varToNs[source]; + if (!ns) continue; + const fieldRe = new RegExp(`\\{%-?\\s*input\\s+${loopVar}\\.([a-zA-Z0-9_]+)`, "g"); + const fields = new Set(); + let fm; + while ((fm = fieldRe.exec(body)) !== null) fields.add(fm[1]); + if (!fields.size) continue; + if (!collections.find((c) => c.namespace === ns)) { + collections.push({ namespace: ns, collection: `custom.${ns}`, loopVar, fields: [...fields] }); + } + } + return collections; +} + +// Attach the live items (from the fetched custom array) to each detected collection. +function describeCollections(collections, customArray) { + return (collections || []).map((col) => { + const items = []; + for (const c of customArray || []) { + if (!c || c.namespace !== col.namespace) continue; + items.push({ key: c.key, value: c.value }); + } + return { collection: col.collection, fields: col.fields, itemCount: items.length, items }; + }); +} + // Build { "namespace.key": value } from the API custom array. function storedMapFromCustom(customArray) { const map = {}; @@ -122,15 +165,21 @@ function literalValue(def) { async function describeInputs(url, opts = {}) { const parameters = Utils.extractURL(url); - if (parameters.templateType !== "reconciliationText") { - consola.error("describe-inputs currently supports reconciliation templates only."); + if (!opts.handle && parameters.templateType !== "reconciliationText") { + consola.error("describe-inputs currently supports reconciliation templates only (or pass --handle)."); return null; } - const detailsResponse = await SF.readReconciliationTextDetails("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); - const handle = detailsResponse?.data?.handle; + // --handle: describe a SIBLING reconciliation in the same company/period. + const target = await resolveReconciliationTarget(url, opts.handle); + if (target.error) { + consola.error(target.error); + return null; + } + const handle = target.handle; + const reconciliationId = target.reconciliationId; if (!handle) { - consola.error("Could not resolve the reconciliation handle from the URL."); + consola.error("Could not resolve the reconciliation handle."); return null; } @@ -144,11 +193,13 @@ async function describeInputs(url, opts = {}) { const inputs = parseInputs(liquid); const echoes = parseResultEchoes(liquid); const adjacentEchoes = parseAdjacentEchoes(liquid); + const collections = parseForiCollections(liquid); - const customResponse = await SF.getReconciliationCustom("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); - const resultsResponse = await SF.getReconciliationResults("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); + const customResponse = await SF.getReconciliationCustom("firm", target.firmId, target.companyId, target.ledgerId, reconciliationId); + const resultsResponse = await SF.getReconciliationResults("firm", target.firmId, target.companyId, target.ledgerId, reconciliationId); const stored = storedMapFromCustom(customResponse?.data); const results = resultsResponse?.data || {}; + const collectionsOut = describeCollections(collections, customResponse?.data); const rows = inputs.map((input) => { const nsKey = `${input.namespace}.${input.key}`; @@ -188,7 +239,8 @@ async function describeInputs(url, opts = {}) { }; }); - const output = { handle, reconciliationId: parameters.reconciliationId, inputs: rows, results }; + const output = { handle, reconciliationId, url: target.url || url, inputs: rows, results }; + if (collectionsOut.length) output.collections = collectionsOut; // --resolve: fill the `unavailable` rows whose default is a direct reference to // data created elsewhere (cross-template result/custom, period/company custom), @@ -216,4 +268,4 @@ async function describeInputs(url, opts = {}) { return output; } -module.exports = { describeInputs, parseInputs, parseResultEchoes, parseAdjacentEchoes, combineLiquid, storedMapFromCustom, literalValue }; +module.exports = { describeInputs, parseInputs, parseResultEchoes, parseAdjacentEchoes, parseForiCollections, describeCollections, combineLiquid, storedMapFromCustom, literalValue }; diff --git a/lib/resultsReader.js b/lib/resultsReader.js index 6a92079f..a2a884ae 100644 --- a/lib/resultsReader.js +++ b/lib/resultsReader.js @@ -1,5 +1,6 @@ const SF = require("./api/sfApi"); const Utils = require("./utils/liquidTestUtils"); +const { resolveReconciliationTarget } = require("./targetResolver"); const { consola } = require("consola"); /** @@ -12,9 +13,31 @@ const { consola } = require("consola"); * @param {String} url Full Silverfin URL of the reconciliation/account in the company file * @returns {Promise} { templateType, firmId, companyId, periodId, ... , results, custom } or null on failure */ -async function fetchResults(url) { +async function fetchResults(url, opts = {}) { const parameters = Utils.extractURL(url); + // --handle: read a SIBLING reconciliation in the same company/period. + if (opts.handle) { + const target = await resolveReconciliationTarget(url, opts.handle); + if (target.error) { + consola.error(target.error); + return null; + } + const customResponse = await SF.getReconciliationCustom("firm", target.firmId, target.companyId, target.ledgerId, target.reconciliationId); + const resultsResponse = await SF.getReconciliationResults("firm", target.firmId, target.companyId, target.ledgerId, target.reconciliationId); + return { + templateType: "reconciliationText", + firmId: target.firmId, + companyId: target.companyId, + periodId: target.ledgerId, + reconciliationId: target.reconciliationId, + handle: target.handle, + url: target.url, + results: resultsResponse?.data ?? null, + custom: customResponse?.data ?? null, + }; + } + switch (parameters.templateType) { case "reconciliationText": { const customResponse = await SF.getReconciliationCustom("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); diff --git a/lib/targetResolver.js b/lib/targetResolver.js new file mode 100644 index 00000000..58cc85aa --- /dev/null +++ b/lib/targetResolver.js @@ -0,0 +1,70 @@ +const SF = require("./api/sfApi"); +const Utils = require("./utils/liquidTestUtils"); + +/** + * Resolve which reconciliation a command should act on: the one in the URL, or a + * SIBLING in the same company/period identified by a handle (`--handle`). This lets + * read commands (get-results / describe-inputs / capture) inspect an upstream + * template without the user having to reconstruct its URL. + */ + +const PER_PAGE = 200; +const MAX_PAGES = 50; + +// handle -> { id, workflowId, name } for a period (loops workflows, returns the +// workflow so a URL can be reconstructed). Null if not found. +async function findReconciliationWithWorkflow(firmId, handle, companyId, periodId) { + const workflowsResponse = await SF.getWorkflows(firmId, companyId, periodId); + const workflows = workflowsResponse?.data ?? []; + for (const workflow of workflows) { + let page = 1; + while (page <= MAX_PAGES) { + const response = await SF.getWorkflowInformation(firmId, companyId, periodId, workflow.id, page); + const reconciliations = response?.data ?? []; + const match = reconciliations.find((r) => r.handle === handle); + if (match) return { id: match.id, workflowId: workflow.id, name: match.name }; + if (reconciliations.length < PER_PAGE) break; + page++; + } + } + return null; +} + +function buildReconciliationUrl(p) { + if (!p.workflowId || !p.reconciliationId) return null; + return `https://live.getsilverfin.com/f/${p.firmId}/${p.companyId}/ledgers/${p.ledgerId}/workflows/${p.workflowId}/reconciliation_texts/${p.reconciliationId}`; +} + +/** + * @param {String} url + * @param {String} [handleOverride] target a sibling reconciliation by handle instead of the URL's target + * @returns {Promise} URL parameters with handle/reconciliationId (and workflowId+url when resolved by handle), or { error } + */ +async function resolveReconciliationTarget(url, handleOverride) { + const parameters = Utils.extractURL(url); + + if (handleOverride) { + const found = await findReconciliationWithWorkflow(parameters.firmId, handleOverride, parameters.companyId, parameters.ledgerId); + if (!found) { + return { error: `Reconciliation "${handleOverride}" not found in period ${parameters.ledgerId} of company ${parameters.companyId}.` }; + } + const resolved = { + ...parameters, + templateType: "reconciliationText", + reconciliationId: found.id, + workflowId: found.workflowId, + handle: handleOverride, + }; + resolved.url = buildReconciliationUrl(resolved); + return resolved; + } + + // No override: resolve the URL target's handle (for reconciliations) so callers have it. + if (parameters.templateType === "reconciliationText") { + const details = await SF.readReconciliationTextDetails("firm", parameters.firmId, parameters.companyId, parameters.ledgerId, parameters.reconciliationId); + parameters.handle = details?.data?.handle || null; + } + return parameters; +} + +module.exports = { resolveReconciliationTarget, findReconciliationWithWorkflow, buildReconciliationUrl }; diff --git a/tests/lib/inputDescriber.test.js b/tests/lib/inputDescriber.test.js index 652944b9..0b73628e 100644 --- a/tests/lib/inputDescriber.test.js +++ b/tests/lib/inputDescriber.test.js @@ -6,7 +6,41 @@ jest.mock("consola"); const SF = require("../../lib/api/sfApi"); const Utils = require("../../lib/utils/liquidTestUtils"); const { ReconciliationText } = require("../../lib/templates/reconciliationText"); -const { describeInputs, parseInputs, parseResultEchoes, parseAdjacentEchoes, storedMapFromCustom, literalValue } = require("../../lib/inputDescriber"); +const { describeInputs, parseInputs, parseResultEchoes, parseAdjacentEchoes, parseForiCollections, describeCollections, storedMapFromCustom, literalValue } = require("../../lib/inputDescriber"); + +describe("parseForiCollections / describeCollections (line-item collections)", () => { + it("detects a fori collection assigned from a custom namespace + its item fields", () => { + const liquid = ` + {% assign items = custom.items | sort:"date" %} + {% fori item in items %} + {% input item.date as:date %} + {% input item.amount as:currency %} + {% input item.description %} + {% endfori %}`; + const cols = parseForiCollections(liquid); + expect(cols).toHaveLength(1); + expect(cols[0]).toMatchObject({ namespace: "items", collection: "custom.items", loopVar: "item" }); + expect(cols[0].fields.sort()).toEqual(["amount", "date", "description"]); + }); + + it("detects a fori iterating custom. directly", () => { + const cols = parseForiCollections(`{% fori x in custom.things %}{% input x.foo %}{% endfori %}`); + expect(cols).toEqual([{ namespace: "things", collection: "custom.things", loopVar: "x", fields: ["foo"] }]); + }); + + it("attaches the live items from the custom array", () => { + const cols = [{ namespace: "items", collection: "custom.items", loopVar: "item", fields: ["date", "amount"] }]; + const customArray = [ + { namespace: "items", key: "1", value: { date: "2024-12-31", amount: 1000 } }, + { namespace: "items", key: "2", value: { amount: 500 } }, + { namespace: "other", key: "x", value: 1 }, + ]; + const out = describeCollections(cols, customArray); + expect(out).toHaveLength(1); + expect(out[0]).toMatchObject({ collection: "custom.items", itemCount: 2 }); + expect(out[0].items.map((i) => i.key)).toEqual(["1", "2"]); + }); +}); describe("parseAdjacentEchoes (indirect result echoes)", () => { it("attributes a result echoing a variable to the nearest preceding input", () => { From f9b892f43b42ebba59c9079f9e9ceb45417e4f76 Mon Sep 17 00:00:00 2001 From: Benjamin Van Dam Date: Wed, 1 Jul 2026 15:24:52 +0200 Subject: [PATCH 17/17] Add render command + resultCount signal (empty results != cached) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes a real failure mode: an agent tried to verify template LOGIC (a display mode -> a note result) by poking live data on an empty shell company where the template is gated off, got 0 results, and wrongly concluded the API was cached. - get-results / describe-inputs now include resultCount (+ an explanatory note when 0): an empty results table means the template isn't computing in this company/period (gated / not set up), NOT that the API is cached and NOT something a custom write will force. - New 'render' command (lib/renderRunner.js) renders a template against its LOCAL liquid-test fixture via the render/test engine and returns a JSON outcome (rendered? / expectation mismatches) — the deterministic, fixture- driven way to verify template logic when a live company lacks the data. Verified live: empty vkt_6_8 -> resultCount 0 + note; liquidation -> 51; render of the liquidation template -> rendered:true, all expectations passed. 638 tests. --- CHANGELOG.md | 2 ++ bin/cli.js | 24 ++++++++++++++++++++ lib/inputDescriber.js | 3 ++- lib/renderRunner.js | 39 +++++++++++++++++++++++++++++++++ lib/resultsReader.js | 18 ++++++++++++++- tests/lib/resultsReader.test.js | 18 ++++++++++++++- 6 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 lib/renderRunner.js diff --git a/CHANGELOG.md b/CHANGELOG.md index fa63e57a..65691153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +Added `render` command — renders a reconciliation template against its LOCAL liquid-test fixture via the Silverfin render/test engine and returns the outcome as JSON (rendered? / expectation mismatches). Usage: `silverfin render -h [-t ] [-f ]`. It's the deterministic, fixture-driven way to verify template LOGIC/output — the right tool when a live company doesn't have the template's data set up (where `get-results` would just be empty). `get-results` and `describe-inputs` now also include a `resultCount` (and, when zero, an explanatory `note`) so an empty results table is self-explanatory — the template isn't computing in that company/period (gated / not set up), which is NOT a caching issue and won't be fixed by setting a custom. + Added `--handle ` to the read commands (`get-results`, `describe-inputs`, `capture`) and a new `resolve-handle` command. These let you inspect a **sibling** reconciliation — an upstream/underlying template that feeds another's defaults — by handle in the same company/period, without reconstructing its URL (`resolve-handle` prints the resolved instance id, workflow id and URL). `describe-inputs` also now surfaces **line-item collections** (`{% fori item in custom. %}{% input item. %}`) as a `collections` section listing the item fields and the live items, so collection-driven inputs (e.g. a working paper's dated line amounts) are visible instead of appearing as "no inputs". Added `doctor` command. It checks that silverfin-ls is installed and supports `data-scope` (required by `manifest`, `describe-inputs --resolve/--compute` and `set-default`) by probing it end-to-end against a throwaway template, and reports the working invocation or the exact failure with install steps. silverfin-ls is now auto-detected (`SILVERFIN_LS_CMD` → `silverfin-ls` on PATH → `npx --no-install silverfin-ls`), and every invocation has a timeout so an outdated silverfin-ls (which would ignore `data-scope` and start its LSP server) can no longer hang the CLI. diff --git a/bin/cli.js b/bin/cli.js index 29f7b7a7..a80f000e 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -28,6 +28,7 @@ const liquidTestUtils = require("../lib/utils/liquidTestUtils"); const customWriter = require("../lib/customWriter"); const defaultSetter = require("../lib/defaultSetter"); const targetResolver = require("../lib/targetResolver"); +const renderRunner = require("../lib/renderRunner"); const firmIdDefault = cliUtils.loadDefaultFirmId(); cliUtils.handleUncaughtErrors(); @@ -559,6 +560,29 @@ program } }); +// RENDER — deterministic, fixture-driven render of a template's logic (vs live get-results) +program + .command("render") + .description("Render a reconciliation template against its LOCAL liquid-test fixture (deterministic, fixture-driven) and return the outcome as JSON — the right tool for verifying template LOGIC/output. Use this (or run-test) instead of set-custom+get-results when a live company doesn't have the template's data set up (get-results would just be empty). Run from your templates repo") + .requiredOption("-h, --handle ", "Reconciliation handle to render (mandatory)") + .option("-f, --firm ", "Firm id (defaults to your configured firm)", firmIdDefault) + .option("-t, --test ", "Render a specific test in the template's YAML (default: all tests)", "") + .option("-o, --output ", "Write the JSON to a file instead of stdout (optional)") + .action(async (options) => { + const data = await renderRunner.renderTemplate(options.firm, options.handle, options.test); + if (!data) { + process.exitCode = 1; + return; + } + const json = JSON.stringify(data, null, 2); + if (options.output) { + require("fs").writeFileSync(options.output, json); + consola.success(`Wrote render outcome to ${options.output}`); + } else { + console.log(json); + } + }); + // RESOLVE-HANDLE — resolve a sibling reconciliation's instance id + URL from a handle program .command("resolve-handle") diff --git a/lib/inputDescriber.js b/lib/inputDescriber.js index aee96c96..5e797157 100644 --- a/lib/inputDescriber.js +++ b/lib/inputDescriber.js @@ -2,6 +2,7 @@ const SF = require("./api/sfApi"); const Utils = require("./utils/liquidTestUtils"); const { ReconciliationText } = require("./templates/reconciliationText"); const { resolveReconciliationTarget } = require("./targetResolver"); +const { resultSummary } = require("./resultsReader"); const { consola } = require("consola"); /** @@ -239,7 +240,7 @@ async function describeInputs(url, opts = {}) { }; }); - const output = { handle, reconciliationId, url: target.url || url, inputs: rows, results }; + const output = { handle, reconciliationId, url: target.url || url, inputs: rows, results, ...resultSummary(results) }; if (collectionsOut.length) output.collections = collectionsOut; // --resolve: fill the `unavailable` rows whose default is a direct reference to diff --git a/lib/renderRunner.js b/lib/renderRunner.js new file mode 100644 index 00000000..93c41683 --- /dev/null +++ b/lib/renderRunner.js @@ -0,0 +1,39 @@ +const liquidTestRunner = require("./liquidTestRunner"); + +/** + * Render a reconciliation template against its LOCAL liquid-test fixture via the + * Silverfin render/test engine and return a structured JSON outcome. + * + * This is the DETERMINISTIC, fixture-driven way to verify what a template's code + * produces — the right tool when a live company doesn't have the template's data + * set up (where `get-results` would just return an empty results table). Unlike + * `get-results` (which reflects a live company's actual state), this renders the + * template against a fixture you control. + * + * Note: the render/test API reports expectation MISMATCHES (got vs expected), not + * the full set of computed results — so to inspect a specific value, assert it in + * the test's `expectation` block and read the `got` on mismatch. + */ +async function renderTemplate(firmId, handle, testName = "") { + const result = await liquidTestRunner.runTests(firmId, "reconciliationText", handle, testName, false, "none", ""); + const testRun = result && result.testRun; + if (!testRun) return null; + + const failures = testRun.tests || {}; + const failedNames = Object.keys(failures); + const rendered = testRun.status === "completed" || testRun.status === "test_success"; + + return { + handle, + status: testRun.status, + rendered, + allExpectationsPassed: rendered && failedNames.length === 0, + failures, // per-test: { reconciled, results (got vs expected), rollforwards } + note: + failedNames.length === 0 + ? "Rendered against the local fixture. To inspect a specific result value, assert it in the test's `expectation` block — the render reports got-vs-expected on mismatch." + : `${failedNames.length} test(s) had expectation mismatches — see failures (got vs expected).`, + }; +} + +module.exports = { renderTemplate }; diff --git a/lib/resultsReader.js b/lib/resultsReader.js index a2a884ae..f37ab66a 100644 --- a/lib/resultsReader.js +++ b/lib/resultsReader.js @@ -3,6 +3,19 @@ const Utils = require("./utils/liquidTestUtils"); const { resolveReconciliationTarget } = require("./targetResolver"); const { consola } = require("consola"); +// A `resultCount` (+ an explanatory note when zero) so an empty results table is +// self-explanatory: it means the template isn't computing here (gated/not set up), +// NOT that the API is cached. Prevents the "set-custom didn't re-render" misread. +function resultSummary(results) { + const resultCount = results && typeof results === "object" ? Object.keys(results).length : 0; + const summary = { resultCount }; + if (resultCount === 0) { + summary.note = + "No results computed for this reconciliation in this company/period — the template is likely gated or not set up here (this is NOT a caching issue; live writes ARE reflected once the template actually computes). To verify template logic, use a liquid test (`run-test`) or the `render` command; to see live effects, use a company/period where it computes."; + } + return summary; +} + /** * Fetch the computed results and custom data of a reconciliation or account * in a LIVE company file, identified by its Silverfin URL. @@ -34,6 +47,7 @@ async function fetchResults(url, opts = {}) { handle: target.handle, url: target.url, results: resultsResponse?.data ?? null, + ...resultSummary(resultsResponse?.data), custom: customResponse?.data ?? null, }; } @@ -49,6 +63,7 @@ async function fetchResults(url, opts = {}) { periodId: parameters.ledgerId, reconciliationId: parameters.reconciliationId, results: resultsResponse?.data ?? null, + ...resultSummary(resultsResponse?.data), custom: customResponse?.data ?? null, }; } @@ -69,6 +84,7 @@ async function fetchResults(url, opts = {}) { accountNumber: account.account.number, accountId, results: resultsResponse?.data ?? null, + ...resultSummary(resultsResponse?.data), custom: customResponse?.data ?? null, }; } @@ -78,4 +94,4 @@ async function fetchResults(url, opts = {}) { } } -module.exports = { fetchResults }; +module.exports = { fetchResults, resultSummary }; diff --git a/tests/lib/resultsReader.test.js b/tests/lib/resultsReader.test.js index 054b9475..ed1ad72e 100644 --- a/tests/lib/resultsReader.test.js +++ b/tests/lib/resultsReader.test.js @@ -5,7 +5,22 @@ jest.mock("consola"); const SF = require("../../lib/api/sfApi"); const Utils = require("../../lib/utils/liquidTestUtils"); const { consola } = require("consola"); -const { fetchResults } = require("../../lib/resultsReader"); +const { fetchResults, resultSummary } = require("../../lib/resultsReader"); + +describe("resultsReader.resultSummary", () => { + it("counts results and adds no note when non-empty", () => { + const s = resultSummary({ a: 1, b: 2 }); + expect(s.resultCount).toBe(2); + expect(s.note).toBeUndefined(); + }); + + it("explains an empty results table (not a caching issue)", () => { + const s = resultSummary({}); + expect(s.resultCount).toBe(0); + expect(s.note).toMatch(/NOT a caching issue/); + expect(resultSummary(null).resultCount).toBe(0); + }); +}); describe("resultsReader.fetchResults", () => { beforeEach(() => { @@ -33,6 +48,7 @@ describe("resultsReader.fetchResults", () => { periodId: "200", reconciliationId: "300", results: { vol_1022_man: "5000.0" }, + resultCount: 1, custom: [{ namespace: "ns", key: "k", value: "v" }], }); });