From f06e484fa52cfb022a6cc810f4aa8c87d41c7a37 Mon Sep 17 00:00:00 2001 From: sid597 Date: Mon, 13 Jul 2026 00:03:52 +0530 Subject: [PATCH] [ENG-1857] Validate Roam-origin Obsidian imports --- apps/obsidian/package.json | 2 + .../src/components/ImportNodesModal.tsx | 13 +- apps/obsidian/src/utils/importNodes.ts | 153 +++++++++++------- .../src/utils/sharedNodeImport.test.ts | 90 +++++++++++ apps/obsidian/src/utils/sharedNodeImport.ts | 92 +++++++++++ pnpm-lock.yaml | 3 + 6 files changed, 292 insertions(+), 61 deletions(-) create mode 100644 apps/obsidian/src/utils/sharedNodeImport.test.ts create mode 100644 apps/obsidian/src/utils/sharedNodeImport.ts diff --git a/apps/obsidian/package.json b/apps/obsidian/package.json index 1b715d839..f0915ec1b 100644 --- a/apps/obsidian/package.json +++ b/apps/obsidian/package.json @@ -9,6 +9,7 @@ "build": "tsx scripts/build.ts", "lint": "eslint .", "lint:fix": "eslint . --fix", + "test:unit": "vitest run", "publish": "tsx scripts/publish.ts --version 0.1.0", "check-types": "tsc --noEmit --skipLibCheck" }, @@ -34,6 +35,7 @@ "tslib": "2.5.1", "tsx": "^4.19.2", "typescript": "5.5.4", + "vitest": "catalog:", "uuidv7": "1.1.0", "zod": "^3.24.1" }, diff --git a/apps/obsidian/src/components/ImportNodesModal.tsx b/apps/obsidian/src/components/ImportNodesModal.tsx index c84ef49e1..273c102af 100644 --- a/apps/obsidian/src/components/ImportNodesModal.tsx +++ b/apps/obsidian/src/components/ImportNodesModal.tsx @@ -8,11 +8,12 @@ import { getAvailableGroupIds } from "@repo/database/lib/groups"; import { fetchUserNames, getPublishedNodesForGroups, - getLocalNodeInstanceIds, + getImportedNodeKeys, getSpaceNameFromIds, getSpaceUris, importSelectedNodes, } from "~/utils/importNodes"; +import { getImportedNodeKey } from "~/utils/sharedNodeImport"; import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; import { computeImportPreview, @@ -71,11 +72,17 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => { currentSpaceId: context.spaceId, }); - const localNodeInstanceIds = getLocalNodeInstanceIds(plugin); + const importedNodeKeys = await getImportedNodeKeys({ plugin, client }); // Filter out nodes that already exist locally const importableNodes = publishedNodes.filter( - (node) => !localNodeInstanceIds.has(node.source_local_id), + (node) => + !importedNodeKeys.has( + getImportedNodeKey({ + spaceId: node.space_id, + sourceLocalId: node.source_local_id, + }), + ), ); const uniqueSpaceIds = [ diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 03eecab0a..cb663c41f 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -1,5 +1,4 @@ import type { Json } from "@repo/database/dbTypes"; -import matter from "gray-matter"; import { App, Notice, TFile } from "obsidian"; import type { DGSupabaseClient } from "@repo/database/lib/client"; import { listGroupSharedNodes } from "@repo/database/lib/sharedNodes"; @@ -21,6 +20,13 @@ import { } from "./importRelations"; import { createTemplateFile } from "./templates"; import { resolveFolderForSpaceUri } from "./importFolderMetadata"; +import { + buildImportedNodeFrontmatter, + buildSourceNodeTypeIdMap, + getAvailableImportPath, + type SourceNodeConcept, + type SourceNodeSchema, +} from "./sharedNodeImport"; type PublishedNode = { source_local_id: string; @@ -73,22 +79,20 @@ export const getPublishedNodesForGroups = async ({ }); }; -export const getLocalNodeInstanceIds = ( - plugin: DiscourseGraphPlugin, -): Set => { +export const getImportedNodeKeys = async ({ + plugin, + client, +}: { + plugin: DiscourseGraphPlugin; + client: DGSupabaseClient; +}): Promise> => { const queryEngine = new QueryEngine(plugin.app); - const files = queryEngine.getFilesWithNodeInstanceId(); - const nodeInstanceIds = new Set(); - - for (const file of files) { - const cache = plugin.app.metadataCache.getFileCache(file); - const frontmatter = cache?.frontmatter; - if (frontmatter?.nodeInstanceId) { - nodeInstanceIds.add(frontmatter.nodeInstanceId as string); - } - } - - return nodeInstanceIds; + const { nodeKeys } = await getImportedNodesInfo({ + queryEngine, + plugin, + client, + }); + return nodeKeys; }; /** @@ -325,6 +329,47 @@ const fetchNodeContentForImport = async ({ }; }; +const fetchSourceNodeTypeIds = async ({ + client, + spaceId, + nodeInstanceIds, +}: { + client: DGSupabaseClient; + spaceId: number; + nodeInstanceIds: string[]; +}): Promise> => { + const { data: conceptRows, error: conceptError } = await client + .from("my_concepts") + .select("source_local_id, schema_id") + .eq("space_id", spaceId) + .eq("is_schema", false) + .in("source_local_id", nodeInstanceIds); + if (conceptError || !conceptRows) return new Map(); + + const concepts = conceptRows as SourceNodeConcept[]; + const schemaIds = [ + ...new Set( + concepts + .map((concept) => concept.schema_id) + .filter((schemaId): schemaId is number => schemaId !== null), + ), + ]; + if (schemaIds.length === 0) return new Map(); + + const { data: schemaRows, error: schemaError } = await client + .from("my_concepts") + .select("id, source_local_id") + .eq("space_id", spaceId) + .eq("is_schema", true) + .in("id", schemaIds); + if (schemaError || !schemaRows) return new Map(); + + return buildSourceNodeTypeIdMap({ + concepts, + schemas: schemaRows as SourceNodeSchema[], + }); +}; + /** * Fetches created/last_modified from the source space Content (my_contents) for an imported node. * Used by the discourse context view to show "last modified in original vault". @@ -940,24 +985,6 @@ const sanitizePathForImport = (path: string): string => { .join("/"); }; -type ParsedFrontmatter = { - nodeTypeId?: string; - nodeInstanceId?: string; - publishedToGroups?: string[]; - authorId?: number; - [key: string]: unknown; -}; - -const parseFrontmatter = ( - content: string, -): { frontmatter: ParsedFrontmatter; body: string } => { - const { data, content: body } = matter(content); - return { - frontmatter: (data ?? {}) as ParsedFrontmatter, - body: body ?? "", - }; -}; - /** * Parse literal_content from a Concept schema into fields for DiscourseNode. * Handles both nested form { label, template, source_data: { format, color, tag } } @@ -1099,6 +1126,8 @@ const processFileContent = async ({ sourceSpaceId, sourceSpaceUri, rawContent, + sourceNodeId, + sourceNodeTypeId, originalFilePath, filePath, importedCreatedAt, @@ -1110,6 +1139,8 @@ const processFileContent = async ({ sourceSpaceId: number; sourceSpaceUri: string; rawContent: string; + sourceNodeId: string; + sourceNodeTypeId: string; originalFilePath?: string; filePath: string; importedCreatedAt?: number; @@ -1134,24 +1165,6 @@ const processFileContent = async ({ await plugin.app.vault.process(file, () => rawContent, stat); } - // 2. Parse frontmatter from rawContent (metadataCache is updated async and is - // often empty immediately after create/modify), then map nodeTypeId and update frontmatter. - const { frontmatter } = parseFrontmatter(rawContent); - const sourceNodeTypeId = frontmatter.nodeTypeId; - if (typeof sourceNodeTypeId !== "string") { - await plugin.app.vault.delete(file); - return { - error: "importedNode missing sourceNodeTypeId", - }; - } - const sourceNodeId = frontmatter.nodeInstanceId; - if (typeof sourceNodeId !== "string") { - await plugin.app.vault.delete(file); - return { - error: "importedNode missing nodeInstanceId", - }; - } - const mappedNodeTypeId = await mapNodeTypeIdToLocal({ plugin, client, @@ -1164,16 +1177,22 @@ const processFileContent = async ({ file, (fm) => { const record = fm as Record; - if (mappedNodeTypeId !== undefined) { - record.nodeTypeId = mappedNodeTypeId; - } - record.importedFromRid = spaceUriAndLocalIdToRid( + const importedFromRid = spaceUriAndLocalIdToRid( sourceSpaceUri, sourceNodeId, "note", ); - record.lastModified = importedModifiedAt; - if (authorId) record.authorId = authorId; + Object.assign( + record, + buildImportedNodeFrontmatter({ + existingFrontmatter: record, + sourceNodeId, + mappedNodeTypeId, + importedFromRid, + importedModifiedAt, + authorId, + }), + ); }, stat, ); @@ -1241,6 +1260,11 @@ export const importSelectedNodes = async ({ } const spaceName = spaceNames.get(spaceId) ?? `space-${spaceId}`; + const sourceNodeTypeIds = await fetchSourceNodeTypeIds({ + client, + spaceId, + nodeInstanceIds: nodes.map((node) => node.nodeInstanceId), + }); const importFolderPath = await resolveFolderForSpaceUri({ adapter: plugin.app.vault.adapter, spaceUri, @@ -1250,6 +1274,13 @@ export const importSelectedNodes = async ({ // Process each node in this space for (const node of nodes) { try { + const sourceNodeTypeId = sourceNodeTypeIds.get(node.nodeInstanceId); + if (!sourceNodeTypeId) { + failedCount++; + processedCount++; + onProgress?.(processedCount, totalNodes); + continue; + } const importedFromRid = spaceUriAndLocalIdToRid( spaceUri, node.nodeInstanceId, @@ -1302,6 +1333,10 @@ export const importSelectedNodes = async ({ ? sanitizePathForImport(contentFilePath) : `${sanitizedFileName}.md`; finalFilePath = `${importFolderPath}/${pathUnderImport}`; + finalFilePath = await getAvailableImportPath({ + desiredPath: finalFilePath, + pathExists: (path) => plugin.app.vault.adapter.exists(path), + }); // Ensure all parent folders exist (e.g. import/VaultName/Discourse Nodes/SubFolder) const dirParts = finalFilePath.split("/"); @@ -1321,6 +1356,8 @@ export const importSelectedNodes = async ({ sourceSpaceId: spaceId, sourceSpaceUri: spaceUri, rawContent: content, + sourceNodeId: node.nodeInstanceId, + sourceNodeTypeId, originalFilePath: contentFilePath, filePath: finalFilePath, importedCreatedAt: createdAt, diff --git a/apps/obsidian/src/utils/sharedNodeImport.test.ts b/apps/obsidian/src/utils/sharedNodeImport.test.ts new file mode 100644 index 000000000..48f2f2a35 --- /dev/null +++ b/apps/obsidian/src/utils/sharedNodeImport.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; +import { + buildImportedNodeFrontmatter, + buildSourceNodeTypeIdMap, + getAvailableImportPath, + getImportedNodeKey, +} from "./sharedNodeImport"; + +const ROAM_SOURCE_NODE_ID = "tgWb6JozF"; +const ROAM_SOURCE_NODE_TYPE_ID = "rCLM0schema"; +const ROAM_SOURCE_SPACE_ID = 42; + +const roamFullMarkdown = `# Sleep improves memory consolidation + +Multiple studies show that sleep after learning strengthens memory traces. + +- Supported by [[EVD]] - Rasch & Born 2013 +`; + +describe("Roam-origin shared node import", () => { + it("derives node type identity without relying on markdown frontmatter", () => { + const sourceNodeTypeIds = buildSourceNodeTypeIdMap({ + concepts: [ + { + source_local_id: ROAM_SOURCE_NODE_ID, + schema_id: 200, + }, + ], + schemas: [ + { + id: 200, + source_local_id: ROAM_SOURCE_NODE_TYPE_ID, + }, + ], + }); + + expect(roamFullMarkdown).not.toContain("nodeTypeId:"); + expect(sourceNodeTypeIds.get(ROAM_SOURCE_NODE_ID)).toBe( + ROAM_SOURCE_NODE_TYPE_ID, + ); + }); + + it("adds stable source identity while preserving existing metadata", () => { + expect( + buildImportedNodeFrontmatter({ + existingFrontmatter: { aliases: ["Sleep and memory"] }, + sourceNodeId: ROAM_SOURCE_NODE_ID, + mappedNodeTypeId: "local-claim-type", + importedFromRid: "https://roamresearch.com/#/app/MAPLab/tgWb6JozF", + importedModifiedAt: 1_781_275_600_000, + authorId: 17, + }), + ).toEqual({ + aliases: ["Sleep and memory"], + nodeInstanceId: ROAM_SOURCE_NODE_ID, + nodeTypeId: "local-claim-type", + importedFromRid: "https://roamresearch.com/#/app/MAPLab/tgWb6JozF", + lastModified: 1_781_275_600_000, + authorId: 17, + }); + }); + + it("scopes duplicate prevention to the source space and node identity", () => { + expect( + getImportedNodeKey({ + spaceId: ROAM_SOURCE_SPACE_ID, + sourceLocalId: ROAM_SOURCE_NODE_ID, + }), + ).toBe(`${ROAM_SOURCE_SPACE_ID}:${ROAM_SOURCE_NODE_ID}`); + expect( + getImportedNodeKey({ + spaceId: ROAM_SOURCE_SPACE_ID + 1, + sourceLocalId: ROAM_SOURCE_NODE_ID, + }), + ).not.toBe(`${ROAM_SOURCE_SPACE_ID}:${ROAM_SOURCE_NODE_ID}`); + }); + + it("keeps distinct same-title nodes in separate files", async () => { + const existingPaths = new Set([ + "import/Roam/Sleep improves memory consolidation.md", + ]); + + await expect( + getAvailableImportPath({ + desiredPath: "import/Roam/Sleep improves memory consolidation.md", + pathExists: (path) => Promise.resolve(existingPaths.has(path)), + }), + ).resolves.toBe("import/Roam/Sleep improves memory consolidation (1).md"); + }); +}); diff --git a/apps/obsidian/src/utils/sharedNodeImport.ts b/apps/obsidian/src/utils/sharedNodeImport.ts new file mode 100644 index 000000000..b0c0b71f6 --- /dev/null +++ b/apps/obsidian/src/utils/sharedNodeImport.ts @@ -0,0 +1,92 @@ +export type SourceNodeConcept = { + source_local_id: string; + schema_id: number | null; +}; + +export type SourceNodeSchema = { + id: number; + source_local_id: string; +}; + +export const getImportedNodeKey = ({ + spaceId, + sourceLocalId, +}: { + spaceId: number; + sourceLocalId: string; +}): string => `${spaceId}:${sourceLocalId}`; + +export const getAvailableImportPath = async ({ + desiredPath, + pathExists, +}: { + desiredPath: string; + pathExists: (path: string) => Promise; +}): Promise => { + if (!(await pathExists(desiredPath))) return desiredPath; + + const extensionIndex = desiredPath.lastIndexOf("."); + const basePath = + extensionIndex > desiredPath.lastIndexOf("/") + ? desiredPath.slice(0, extensionIndex) + : desiredPath; + const extension = + extensionIndex > desiredPath.lastIndexOf("/") + ? desiredPath.slice(extensionIndex) + : ""; + + let counter = 1; + let availablePath = `${basePath} (${counter})${extension}`; + while (await pathExists(availablePath)) { + counter++; + availablePath = `${basePath} (${counter})${extension}`; + } + return availablePath; +}; + +export const buildSourceNodeTypeIdMap = ({ + concepts, + schemas, +}: { + concepts: SourceNodeConcept[]; + schemas: SourceNodeSchema[]; +}): Map => { + const sourceNodeTypeIdBySchemaId = new Map( + schemas.map((schema) => [schema.id, schema.source_local_id]), + ); + + return new Map( + concepts.flatMap((concept): [string, string][] => { + if (concept.schema_id === null) return []; + const sourceNodeTypeId = sourceNodeTypeIdBySchemaId.get( + concept.schema_id, + ); + return sourceNodeTypeId + ? [[concept.source_local_id, sourceNodeTypeId]] + : []; + }), + ); +}; + +export const buildImportedNodeFrontmatter = ({ + existingFrontmatter, + sourceNodeId, + mappedNodeTypeId, + importedFromRid, + importedModifiedAt, + authorId, +}: { + existingFrontmatter: Record; + sourceNodeId: string; + mappedNodeTypeId: string; + importedFromRid: string; + importedModifiedAt?: number; + authorId?: number; +}): Record => ({ + ...existingFrontmatter, + nodeInstanceId: sourceNodeId, + nodeTypeId: mappedNodeTypeId, + importedFromRid, + lastModified: importedModifiedAt, + ...(authorId === undefined ? {} : { authorId }), +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ba8f12ba6..78b30460b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -215,6 +215,9 @@ importers: uuidv7: specifier: 1.1.0 version: 1.1.0 + vitest: + specifier: 'catalog:' + version: 4.1.6(@edge-runtime/vm@3.2.0)(@opentelemetry/api@1.9.0)(@types/node@22.20.0)(jsdom@20.0.3)(msw@2.11.1(@types/node@22.20.0)(typescript@5.5.4))(vite@7.3.3(@types/node@22.20.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) zod: specifier: ^3.24.1 version: 3.25.76