From af28f37da8ea04be13e8ff85f946bbb54f3ef974 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:12:45 +0000 Subject: [PATCH 1/2] fix(aio): validate skill reference links resolve in the built bundle The skill build strips the .j2 suffix from every rendered template, so a SKILL.md that links to a `.md.j2` reference points at a path the bundle never contains. Add a blocking lint check that resolves every references/ and scripts/ link in each SKILL.md against the shipped bundle, and fix the one skill that linked a template path. Generated-By: PostHog Desktop Task-Id: 51745b3a-bba0-431b-8b69-fc0e4170d7c2 --- .../skills/exploring-llm-traces/SKILL.md | 2 +- products/posthog_ai/scripts/build_skills.py | 59 +++++++++++++++++++ .../scripts/test/test_build_skills.py | 36 +++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/products/ai_observability/skills/exploring-llm-traces/SKILL.md b/products/ai_observability/skills/exploring-llm-traces/SKILL.md index e636e2b17184..c4e9e79afbca 100644 --- a/products/ai_observability/skills/exploring-llm-traces/SKILL.md +++ b/products/ai_observability/skills/exploring-llm-traces/SKILL.md @@ -246,7 +246,7 @@ posthog:query-llm-traces-list For more complex SQL patterns, read these references: -- [Single trace retrieval](./references/example-llm-trace.md.j2) — fetches a single trace by ID with all events and properties (renders the `TraceQuery` HogQL) +- [Single trace retrieval](./references/example-llm-trace.md) — fetches a single trace by ID with all events and properties (renders the `TraceQuery` HogQL) - [Traces list with aggregated metrics](./references/example-llm-traces-list.md) — two-phase query: find trace IDs first, then fetch aggregated latency, tokens, costs, and error counts ## Parsing large trace results diff --git a/products/posthog_ai/scripts/build_skills.py b/products/posthog_ai/scripts/build_skills.py index 2f55c184aa25..6089b7013c06 100755 --- a/products/posthog_ai/scripts/build_skills.py +++ b/products/posthog_ai/scripts/build_skills.py @@ -266,6 +266,62 @@ def _assert_text_file(file_path: Path) -> None: ) +_MARKDOWN_LINK_RE = re.compile(r"\]\(([^)\s]+)\)") + + +def _check_reference_links(skill_dir: Path, repo_root: Path) -> list[str]: + """Find markdown links to references/ or scripts/ files that will not exist in the built bundle. + + A skill ships its SKILL.md, references/, and scripts/ files, and the build strips the .j2 suffix + from every rendered template. So a link to `references/x.md` resolves when the source holds either + `references/x.md` or `references/x.md.j2`, but a link to `references/x.md.j2` never resolves, + because the bundle only has the stripped `references/x.md`. + """ + bundle: set[str] = {"SKILL.md"} + entry = skill_dir / "SKILL.md.j2" + if not entry.exists(): + entry = skill_dir / "SKILL.md" + if not entry.exists(): + return [] + for subdir_name in sorted(_ALLOWED_SUBDIRS): + subdir = skill_dir / subdir_name + if not subdir.is_dir(): + continue + for root, dirs, filenames in os.walk(subdir): + dirs[:] = sorted(dirs) + for filename in sorted(filenames): + # The build strips .j2 when it renders a template, so record the shipped path. + rel = str((Path(root) / filename).relative_to(skill_dir)) + bundle.add(rel.removesuffix(".j2")) + + # Only the SKILL.md entry point is scanned. Reference files link to each other with paths + # relative to the skill root, which do not resolve from inside references/, and their code + # snippets hold `](...)` fragments that are not links. + errors: list[str] = [] + text = entry.read_text() + source_label = str(entry.relative_to(repo_root)) + for match in _MARKDOWN_LINK_RE.finditer(text): + target = match.group(1).split("#", 1)[0].split("?", 1)[0] + if not target or "://" in target or target.startswith("mailto:"): + continue + resolved = (skill_dir / target).resolve() + try: + rel_to_skill = resolved.relative_to(skill_dir.resolve()) + except ValueError: + continue # Link points outside the skill; not part of the bundle. + if not rel_to_skill.parts or rel_to_skill.parts[0] not in _ALLOWED_SUBDIRS: + continue # Only references/ and scripts/ files ship in the bundle. + if resolved.is_dir(): + continue # A directory ships through its files. + if str(rel_to_skill) not in bundle: + line, _col = _line_col(text, match.start(1)) + errors.append( + f"Broken reference link in {source_label}:{line}: '{target}' does not resolve to a " + f"bundled file. Link to the built '.md' path, not the '.md.j2' template." + ) + return errors + + class SkillFrontmatter(BaseModel): name: str description: str = Field(max_length=_MAX_SKILL_DESCRIPTION_LENGTH) @@ -612,6 +668,9 @@ def lint_all(self) -> bool: for skill in skills: lint_files = self._collect_lint_files(skill) + if skill.depth == 1: + errors.extend(_check_reference_links(skill.source_file.parent, self.repo_root)) + for file_path in lint_files: source_label = str(file_path.relative_to(self.repo_root)) diff --git a/products/posthog_ai/scripts/test/test_build_skills.py b/products/posthog_ai/scripts/test/test_build_skills.py index 9776d59a2403..0741fdefff50 100644 --- a/products/posthog_ai/scripts/test/test_build_skills.py +++ b/products/posthog_ai/scripts/test/test_build_skills.py @@ -504,6 +504,42 @@ def test_lint_all_catches_bad_jinja2_in_subdirectory(tmp_path: Path) -> None: assert builder.lint_all() is False +def test_lint_all_catches_md_j2_reference_link(tmp_path: Path) -> None: + skill_dir = tmp_path / "products" / "alpha" / "skills" / "bad-link" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + "---\nname: bad-link\ndescription: D\n---\nSee [payload](references/payload.md.j2).\n" + ) + refs = skill_dir / "references" + refs.mkdir() + (refs / "payload.md.j2").write_text("# {{ 'rendered' }}\n") + + builder = SkillBuilder( + repo_root=tmp_path, + products_dir=tmp_path / "products", + output_dir=tmp_path / "output", + ) + assert builder.lint_all() is False + + +def test_lint_all_passes_md_link_to_rendered_template(tmp_path: Path) -> None: + skill_dir = tmp_path / "products" / "alpha" / "skills" / "good-link" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + "---\nname: good-link\ndescription: D\n---\nSee [payload](references/payload.md).\n" + ) + refs = skill_dir / "references" + refs.mkdir() + (refs / "payload.md.j2").write_text("# {{ 'rendered' }}\n") + + builder = SkillBuilder( + repo_root=tmp_path, + products_dir=tmp_path / "products", + output_dir=tmp_path / "output", + ) + assert builder.lint_all() is True + + def test_lint_all_catches_duplicate_skill_names(tmp_path: Path) -> None: for product in ("alpha", "beta"): skill_dir = tmp_path / "products" / product / "skills" / "same-name" From 1482c68b6c8dc56315bdbb2647aa00e7dd182e3a Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Thu, 27 Aug 2026 11:49:46 +0200 Subject: [PATCH 2/2] chore(aio): name the real cause when a skill link has no bundled file --- products/posthog_ai/scripts/build_skills.py | 20 +++++------ .../scripts/test/test_build_skills.py | 36 +++++++------------ 2 files changed, 22 insertions(+), 34 deletions(-) diff --git a/products/posthog_ai/scripts/build_skills.py b/products/posthog_ai/scripts/build_skills.py index 6089b7013c06..147d51ebc1d1 100755 --- a/products/posthog_ai/scripts/build_skills.py +++ b/products/posthog_ai/scripts/build_skills.py @@ -269,7 +269,7 @@ def _assert_text_file(file_path: Path) -> None: _MARKDOWN_LINK_RE = re.compile(r"\]\(([^)\s]+)\)") -def _check_reference_links(skill_dir: Path, repo_root: Path) -> list[str]: +def _check_reference_links(entry: Path, repo_root: Path) -> list[str]: """Find markdown links to references/ or scripts/ files that will not exist in the built bundle. A skill ships its SKILL.md, references/, and scripts/ files, and the build strips the .j2 suffix @@ -277,12 +277,8 @@ def _check_reference_links(skill_dir: Path, repo_root: Path) -> list[str]: `references/x.md` or `references/x.md.j2`, but a link to `references/x.md.j2` never resolves, because the bundle only has the stripped `references/x.md`. """ - bundle: set[str] = {"SKILL.md"} - entry = skill_dir / "SKILL.md.j2" - if not entry.exists(): - entry = skill_dir / "SKILL.md" - if not entry.exists(): - return [] + skill_dir = entry.parent + bundle: set[str] = set() for subdir_name in sorted(_ALLOWED_SUBDIRS): subdir = skill_dir / subdir_name if not subdir.is_dir(): @@ -315,9 +311,13 @@ def _check_reference_links(skill_dir: Path, repo_root: Path) -> list[str]: continue # A directory ships through its files. if str(rel_to_skill) not in bundle: line, _col = _line_col(text, match.start(1)) + hint = ( + "Link to the built '.md' path, not the '.md.j2' template." + if target.endswith(".j2") + else "No file of that name ships in the skill." + ) errors.append( - f"Broken reference link in {source_label}:{line}: '{target}' does not resolve to a " - f"bundled file. Link to the built '.md' path, not the '.md.j2' template." + f"Broken reference link in {source_label}:{line}: '{target}' does not resolve to a bundled file. {hint}" ) return errors @@ -669,7 +669,7 @@ def lint_all(self) -> bool: lint_files = self._collect_lint_files(skill) if skill.depth == 1: - errors.extend(_check_reference_links(skill.source_file.parent, self.repo_root)) + errors.extend(_check_reference_links(skill.source_file, self.repo_root)) for file_path in lint_files: source_label = str(file_path.relative_to(self.repo_root)) diff --git a/products/posthog_ai/scripts/test/test_build_skills.py b/products/posthog_ai/scripts/test/test_build_skills.py index 0741fdefff50..8211f1b2b566 100644 --- a/products/posthog_ai/scripts/test/test_build_skills.py +++ b/products/posthog_ai/scripts/test/test_build_skills.py @@ -504,30 +504,18 @@ def test_lint_all_catches_bad_jinja2_in_subdirectory(tmp_path: Path) -> None: assert builder.lint_all() is False -def test_lint_all_catches_md_j2_reference_link(tmp_path: Path) -> None: - skill_dir = tmp_path / "products" / "alpha" / "skills" / "bad-link" - skill_dir.mkdir(parents=True) - (skill_dir / "SKILL.md").write_text( - "---\nname: bad-link\ndescription: D\n---\nSee [payload](references/payload.md.j2).\n" - ) - refs = skill_dir / "references" - refs.mkdir() - (refs / "payload.md.j2").write_text("# {{ 'rendered' }}\n") - - builder = SkillBuilder( - repo_root=tmp_path, - products_dir=tmp_path / "products", - output_dir=tmp_path / "output", - ) - assert builder.lint_all() is False - - -def test_lint_all_passes_md_link_to_rendered_template(tmp_path: Path) -> None: - skill_dir = tmp_path / "products" / "alpha" / "skills" / "good-link" +@pytest.mark.parametrize( + "link_target,expected", + [ + ("references/payload.md.j2", False), + ("references/payload.md", True), + ("references/missing.md", False), + ], +) +def test_lint_all_checks_reference_links_against_the_bundle(tmp_path: Path, link_target: str, expected: bool) -> None: + skill_dir = tmp_path / "products" / "alpha" / "skills" / "linker" skill_dir.mkdir(parents=True) - (skill_dir / "SKILL.md").write_text( - "---\nname: good-link\ndescription: D\n---\nSee [payload](references/payload.md).\n" - ) + (skill_dir / "SKILL.md").write_text(f"---\nname: linker\ndescription: D\n---\nSee [payload]({link_target}).\n") refs = skill_dir / "references" refs.mkdir() (refs / "payload.md.j2").write_text("# {{ 'rendered' }}\n") @@ -537,7 +525,7 @@ def test_lint_all_passes_md_link_to_rendered_template(tmp_path: Path) -> None: products_dir=tmp_path / "products", output_dir=tmp_path / "output", ) - assert builder.lint_all() is True + assert builder.lint_all() is expected def test_lint_all_catches_duplicate_skill_names(tmp_path: Path) -> None: