ENG-1857 Validate Obsidian importer with Roam-origin nodes

apps/obsidian/package.json

@@ -9,6 +9,7 @@
     "build": "tsx scripts/build.ts",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
+    "test:unit": "vitest run",
     "publish": "tsx scripts/publish.ts --version 0.1.0",
     "check-types": "tsc --noEmit --skipLibCheck"
   },
@@ -34,6 +35,7 @@
     "tslib": "2.5.1",
     "tsx": "^4.19.2",
     "typescript": "5.5.4",
+    "vitest": "catalog:",
     "uuidv7": "1.1.0",
     "zod": "^3.24.1"
   },

apps/obsidian/src/components/ImportNodesModal.tsx

@@ -8,11 +8,12 @@ import { getAvailableGroupIds } from "@repo/database/lib/groups";
 import {
   fetchUserNames,
   getPublishedNodesForGroups,
-  getLocalNodeInstanceIds,
+  getImportedNodeKeys,
   getSpaceNameFromIds,
   getSpaceUris,
   importSelectedNodes,
 } from "~/utils/importNodes";
+import { getImportedNodeKey } from "~/utils/sharedNodeImport";
 import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext";
 import {
   computeImportPreview,
@@ -71,11 +72,17 @@ const ImportNodesContent = ({ plugin, onClose }: ImportNodesModalProps) => {
         currentSpaceId: context.spaceId,
       });
 
-      const localNodeInstanceIds = getLocalNodeInstanceIds(plugin);
+      const importedNodeKeys = await getImportedNodeKeys({ plugin, client });
 
       // Filter out nodes that already exist locally
       const importableNodes = publishedNodes.filter(
-        (node) => !localNodeInstanceIds.has(node.source_local_id),
+        (node) =>
+          !importedNodeKeys.has(
+            getImportedNodeKey({
+              spaceId: node.space_id,
+              sourceLocalId: node.source_local_id,
+            }),
+          ),
       );
 
       const uniqueSpaceIds = [

apps/obsidian/src/utils/importNodes.ts

@@ -1,5 +1,4 @@
 import type { Json } from "@repo/database/dbTypes";
-import matter from "gray-matter";
 import { App, Notice, TFile } from "obsidian";
 import type { DGSupabaseClient } from "@repo/database/lib/client";
 import { listGroupSharedNodes } from "@repo/database/lib/sharedNodes";
@@ -21,6 +20,13 @@ import {
 } from "./importRelations";
 import { createTemplateFile } from "./templates";
 import { resolveFolderForSpaceUri } from "./importFolderMetadata";
+import {
+  buildImportedNodeFrontmatter,
+  buildSourceNodeTypeIdMap,
+  getAvailableImportPath,
+  type SourceNodeConcept,
+  type SourceNodeSchema,
+} from "./sharedNodeImport";
 
 type PublishedNode = {
   source_local_id: string;
@@ -73,22 +79,20 @@ export const getPublishedNodesForGroups = async ({
   });
 };
 
-export const getLocalNodeInstanceIds = (
-  plugin: DiscourseGraphPlugin,
-): Set<string> => {
+export const getImportedNodeKeys = async ({
+  plugin,
+  client,
+}: {
+  plugin: DiscourseGraphPlugin;
+  client: DGSupabaseClient;
+}): Promise<Set<string>> => {
   const queryEngine = new QueryEngine(plugin.app);
-  const files = queryEngine.getFilesWithNodeInstanceId();
-  const nodeInstanceIds = new Set<string>();
-
-  for (const file of files) {
-    const cache = plugin.app.metadataCache.getFileCache(file);
-    const frontmatter = cache?.frontmatter;
-    if (frontmatter?.nodeInstanceId) {
-      nodeInstanceIds.add(frontmatter.nodeInstanceId as string);
-    }
-  }
-
-  return nodeInstanceIds;
+  const { nodeKeys } = await getImportedNodesInfo({
+    queryEngine,
+    plugin,
+    client,
+  });
+  return nodeKeys;
 };
 
 /**
@@ -325,6 +329,47 @@ const fetchNodeContentForImport = async ({
   };
 };
 
+const fetchSourceNodeTypeIds = async ({
+  client,
+  spaceId,
+  nodeInstanceIds,
+}: {
+  client: DGSupabaseClient;
+  spaceId: number;
+  nodeInstanceIds: string[];
+}): Promise<Map<string, string>> => {
+  const { data: conceptRows, error: conceptError } = await client
+    .from("my_concepts")
+    .select("source_local_id, schema_id")
+    .eq("space_id", spaceId)
+    .eq("is_schema", false)
+    .in("source_local_id", nodeInstanceIds);
+  if (conceptError || !conceptRows) return new Map();
+
+  const concepts = conceptRows as SourceNodeConcept[];
+  const schemaIds = [
+    ...new Set(
+      concepts
+        .map((concept) => concept.schema_id)
+        .filter((schemaId): schemaId is number => schemaId !== null),
+    ),
+  ];
+  if (schemaIds.length === 0) return new Map();
+
+  const { data: schemaRows, error: schemaError } = await client
+    .from("my_concepts")
+    .select("id, source_local_id")
+    .eq("space_id", spaceId)
+    .eq("is_schema", true)
+    .in("id", schemaIds);
+  if (schemaError || !schemaRows) return new Map();
+
+  return buildSourceNodeTypeIdMap({
+    concepts,
+    schemas: schemaRows as SourceNodeSchema[],
+  });
+};
+
 /**
  * Fetches created/last_modified from the source space Content (my_contents) for an imported node.
  * Used by the discourse context view to show "last modified in original vault".
@@ -940,24 +985,6 @@ const sanitizePathForImport = (path: string): string => {
     .join("/");
 };
 
-type ParsedFrontmatter = {
-  nodeTypeId?: string;
-  nodeInstanceId?: string;
-  publishedToGroups?: string[];
-  authorId?: number;
-  [key: string]: unknown;
-};
-
-const parseFrontmatter = (
-  content: string,
-): { frontmatter: ParsedFrontmatter; body: string } => {
-  const { data, content: body } = matter(content);
-  return {
-    frontmatter: (data ?? {}) as ParsedFrontmatter,
-    body: body ?? "",
-  };
-};
-
 /**
  * Parse literal_content from a Concept schema into fields for DiscourseNode.
  * Handles both nested form { label, template, source_data: { format, color, tag } }
@@ -1099,6 +1126,8 @@ const processFileContent = async ({
   sourceSpaceId,
   sourceSpaceUri,
   rawContent,
+  sourceNodeId,
+  sourceNodeTypeId,
   originalFilePath,
   filePath,
   importedCreatedAt,
@@ -1110,6 +1139,8 @@ const processFileContent = async ({
   sourceSpaceId: number;
   sourceSpaceUri: string;
   rawContent: string;
+  sourceNodeId: string;
+  sourceNodeTypeId: string;
   originalFilePath?: string;
   filePath: string;
   importedCreatedAt?: number;
@@ -1134,24 +1165,6 @@ const processFileContent = async ({
     await plugin.app.vault.process(file, () => rawContent, stat);
   }
 
-  // 2. Parse frontmatter from rawContent (metadataCache is updated async and is
-  //    often empty immediately after create/modify), then map nodeTypeId and update frontmatter.
-  const { frontmatter } = parseFrontmatter(rawContent);
-  const sourceNodeTypeId = frontmatter.nodeTypeId;
-  if (typeof sourceNodeTypeId !== "string") {
-    await plugin.app.vault.delete(file);
-    return {
-      error: "importedNode missing sourceNodeTypeId",
-    };
-  }
-  const sourceNodeId = frontmatter.nodeInstanceId;
-  if (typeof sourceNodeId !== "string") {
-    await plugin.app.vault.delete(file);
-    return {
-      error: "importedNode missing nodeInstanceId",
-    };
-  }
-
   const mappedNodeTypeId = await mapNodeTypeIdToLocal({
     plugin,
     client,
@@ -1164,16 +1177,22 @@ const processFileContent = async ({
     file,
     (fm) => {
       const record = fm as Record<string, unknown>;
-      if (mappedNodeTypeId !== undefined) {
-        record.nodeTypeId = mappedNodeTypeId;
-      }
-      record.importedFromRid = spaceUriAndLocalIdToRid(
+      const importedFromRid = spaceUriAndLocalIdToRid(
         sourceSpaceUri,
         sourceNodeId,
         "note",
       );
-      record.lastModified = importedModifiedAt;
-      if (authorId) record.authorId = authorId;
+      Object.assign(
+        record,
+        buildImportedNodeFrontmatter({
+          existingFrontmatter: record,
+          sourceNodeId,
+          mappedNodeTypeId,
+          importedFromRid,
+          importedModifiedAt,
+          authorId,
+        }),
+      );
     },
     stat,
   );
@@ -1241,6 +1260,11 @@ export const importSelectedNodes = async ({
     }
 
     const spaceName = spaceNames.get(spaceId) ?? `space-${spaceId}`;
+    const sourceNodeTypeIds = await fetchSourceNodeTypeIds({
+      client,
+      spaceId,
+      nodeInstanceIds: nodes.map((node) => node.nodeInstanceId),
+    });
     const importFolderPath = await resolveFolderForSpaceUri({
       adapter: plugin.app.vault.adapter,
       spaceUri,
@@ -1250,6 +1274,13 @@ export const importSelectedNodes = async ({
     // Process each node in this space
     for (const node of nodes) {
       try {
+        const sourceNodeTypeId = sourceNodeTypeIds.get(node.nodeInstanceId);
+        if (!sourceNodeTypeId) {
+          failedCount++;
+          processedCount++;
+          onProgress?.(processedCount, totalNodes);
+          continue;
+        }
         const importedFromRid = spaceUriAndLocalIdToRid(
           spaceUri,
           node.nodeInstanceId,
@@ -1302,6 +1333,10 @@ export const importSelectedNodes = async ({
               ? sanitizePathForImport(contentFilePath)
               : `${sanitizedFileName}.md`;
           finalFilePath = `${importFolderPath}/${pathUnderImport}`;
+          finalFilePath = await getAvailableImportPath({
+            desiredPath: finalFilePath,
+            pathExists: (path) => plugin.app.vault.adapter.exists(path),
+          });
 
           // Ensure all parent folders exist (e.g. import/VaultName/Discourse Nodes/SubFolder)
           const dirParts = finalFilePath.split("/");
@@ -1321,6 +1356,8 @@ export const importSelectedNodes = async ({
           sourceSpaceId: spaceId,
           sourceSpaceUri: spaceUri,
           rawContent: content,
+          sourceNodeId: node.nodeInstanceId,
+          sourceNodeTypeId,
           originalFilePath: contentFilePath,
           filePath: finalFilePath,
           importedCreatedAt: createdAt,

apps/obsidian/src/utils/sharedNodeImport.test.ts

@@ -0,0 +1,90 @@
+import { describe, expect, it } from "vitest";
+import {
+  buildImportedNodeFrontmatter,
+  buildSourceNodeTypeIdMap,
+  getAvailableImportPath,
+  getImportedNodeKey,
+} from "./sharedNodeImport";
+
+const ROAM_SOURCE_NODE_ID = "tgWb6JozF";
+const ROAM_SOURCE_NODE_TYPE_ID = "rCLM0schema";
+const ROAM_SOURCE_SPACE_ID = 42;
+
+const roamFullMarkdown = `# Sleep improves memory consolidation
+
+Multiple studies show that sleep after learning strengthens memory traces.
+
+- Supported by [[EVD]] - Rasch & Born 2013
+`;
+
+describe("Roam-origin shared node import", () => {
+  it("derives node type identity without relying on markdown frontmatter", () => {
+    const sourceNodeTypeIds = buildSourceNodeTypeIdMap({
+      concepts: [
+        {
+          source_local_id: ROAM_SOURCE_NODE_ID,
+          schema_id: 200,
+        },
+      ],
+      schemas: [
+        {
+          id: 200,
+          source_local_id: ROAM_SOURCE_NODE_TYPE_ID,
+        },
+      ],
+    });
+
+    expect(roamFullMarkdown).not.toContain("nodeTypeId:");
+    expect(sourceNodeTypeIds.get(ROAM_SOURCE_NODE_ID)).toBe(
+      ROAM_SOURCE_NODE_TYPE_ID,
+    );
+  });
+
+  it("adds stable source identity while preserving existing metadata", () => {
+    expect(
+      buildImportedNodeFrontmatter({
+        existingFrontmatter: { aliases: ["Sleep and memory"] },
+        sourceNodeId: ROAM_SOURCE_NODE_ID,
+        mappedNodeTypeId: "local-claim-type",
+        importedFromRid: "https://roamresearch.com/#/app/MAPLab/tgWb6JozF",
+        importedModifiedAt: 1_781_275_600_000,
+        authorId: 17,
+      }),
+    ).toEqual({
+      aliases: ["Sleep and memory"],
+      nodeInstanceId: ROAM_SOURCE_NODE_ID,
+      nodeTypeId: "local-claim-type",
+      importedFromRid: "https://roamresearch.com/#/app/MAPLab/tgWb6JozF",
+      lastModified: 1_781_275_600_000,
+      authorId: 17,
+    });
+  });
+
+  it("scopes duplicate prevention to the source space and node identity", () => {
+    expect(
+      getImportedNodeKey({
+        spaceId: ROAM_SOURCE_SPACE_ID,
+        sourceLocalId: ROAM_SOURCE_NODE_ID,
+      }),
+    ).toBe(`${ROAM_SOURCE_SPACE_ID}:${ROAM_SOURCE_NODE_ID}`);
+    expect(
+      getImportedNodeKey({
+        spaceId: ROAM_SOURCE_SPACE_ID + 1,
+        sourceLocalId: ROAM_SOURCE_NODE_ID,
+      }),
+    ).not.toBe(`${ROAM_SOURCE_SPACE_ID}:${ROAM_SOURCE_NODE_ID}`);
+  });
+
+  it("keeps distinct same-title nodes in separate files", async () => {
+    const existingPaths = new Set([
+      "import/Roam/Sleep improves memory consolidation.md",
+    ]);
+
+    await expect(
+      getAvailableImportPath({
+        desiredPath: "import/Roam/Sleep improves memory consolidation.md",
+        pathExists: (path) => Promise.resolve(existingPaths.has(path)),
+      }),
+    ).resolves.toBe("import/Roam/Sleep improves memory consolidation (1).md");
+  });
+});