diff --git a/products/ai_observability/skills/exploring-llm-traces/SKILL.md b/products/ai_observability/skills/exploring-llm-traces/SKILL.md index 020bc004db68..b7835fd08e41 100644 --- a/products/ai_observability/skills/exploring-llm-traces/SKILL.md +++ b/products/ai_observability/skills/exploring-llm-traces/SKILL.md @@ -246,7 +246,7 @@ posthog:query-llm-traces-list For more complex SQL patterns, read these references: -- [Single trace retrieval](./references/example-llm-trace.md.j2) — fetches a single trace by ID with all events and properties (renders the `TraceQuery` HogQL) +- [Single trace retrieval](./references/example-llm-trace.md) — fetches a single trace by ID with all events and properties (renders the `TraceQuery` HogQL) - [Traces list with aggregated metrics](./references/example-llm-traces-list.md) — two-phase query: find trace IDs first, then fetch aggregated latency, tokens, costs, and error counts ## Parsing large trace results diff --git a/products/posthog_ai/scripts/build_skills.py b/products/posthog_ai/scripts/build_skills.py index 2f55c184aa25..147d51ebc1d1 100755 --- a/products/posthog_ai/scripts/build_skills.py +++ b/products/posthog_ai/scripts/build_skills.py @@ -266,6 +266,62 @@ def _assert_text_file(file_path: Path) -> None: ) +_MARKDOWN_LINK_RE = re.compile(r"\]\(([^)\s]+)\)") + + +def _check_reference_links(entry: Path, repo_root: Path) -> list[str]: + """Find markdown links to references/ or scripts/ files that will not exist in the built bundle. + + A skill ships its SKILL.md, references/, and scripts/ files, and the build strips the .j2 suffix + from every rendered template. So a link to `references/x.md` resolves when the source holds either + `references/x.md` or `references/x.md.j2`, but a link to `references/x.md.j2` never resolves, + because the bundle only has the stripped `references/x.md`. + """ + skill_dir = entry.parent + bundle: set[str] = set() + for subdir_name in sorted(_ALLOWED_SUBDIRS): + subdir = skill_dir / subdir_name + if not subdir.is_dir(): + continue + for root, dirs, filenames in os.walk(subdir): + dirs[:] = sorted(dirs) + for filename in sorted(filenames): + # The build strips .j2 when it renders a template, so record the shipped path. + rel = str((Path(root) / filename).relative_to(skill_dir)) + bundle.add(rel.removesuffix(".j2")) + + # Only the SKILL.md entry point is scanned. Reference files link to each other with paths + # relative to the skill root, which do not resolve from inside references/, and their code + # snippets hold `](...)` fragments that are not links. + errors: list[str] = [] + text = entry.read_text() + source_label = str(entry.relative_to(repo_root)) + for match in _MARKDOWN_LINK_RE.finditer(text): + target = match.group(1).split("#", 1)[0].split("?", 1)[0] + if not target or "://" in target or target.startswith("mailto:"): + continue + resolved = (skill_dir / target).resolve() + try: + rel_to_skill = resolved.relative_to(skill_dir.resolve()) + except ValueError: + continue # Link points outside the skill; not part of the bundle. + if not rel_to_skill.parts or rel_to_skill.parts[0] not in _ALLOWED_SUBDIRS: + continue # Only references/ and scripts/ files ship in the bundle. + if resolved.is_dir(): + continue # A directory ships through its files. + if str(rel_to_skill) not in bundle: + line, _col = _line_col(text, match.start(1)) + hint = ( + "Link to the built '.md' path, not the '.md.j2' template." + if target.endswith(".j2") + else "No file of that name ships in the skill." + ) + errors.append( + f"Broken reference link in {source_label}:{line}: '{target}' does not resolve to a bundled file. {hint}" + ) + return errors + + class SkillFrontmatter(BaseModel): name: str description: str = Field(max_length=_MAX_SKILL_DESCRIPTION_LENGTH) @@ -612,6 +668,9 @@ def lint_all(self) -> bool: for skill in skills: lint_files = self._collect_lint_files(skill) + if skill.depth == 1: + errors.extend(_check_reference_links(skill.source_file, self.repo_root)) + for file_path in lint_files: source_label = str(file_path.relative_to(self.repo_root)) diff --git a/products/posthog_ai/scripts/test/test_build_skills.py b/products/posthog_ai/scripts/test/test_build_skills.py index 9776d59a2403..8211f1b2b566 100644 --- a/products/posthog_ai/scripts/test/test_build_skills.py +++ b/products/posthog_ai/scripts/test/test_build_skills.py @@ -504,6 +504,30 @@ def test_lint_all_catches_bad_jinja2_in_subdirectory(tmp_path: Path) -> None: assert builder.lint_all() is False +@pytest.mark.parametrize( + "link_target,expected", + [ + ("references/payload.md.j2", False), + ("references/payload.md", True), + ("references/missing.md", False), + ], +) +def test_lint_all_checks_reference_links_against_the_bundle(tmp_path: Path, link_target: str, expected: bool) -> None: + skill_dir = tmp_path / "products" / "alpha" / "skills" / "linker" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text(f"---\nname: linker\ndescription: D\n---\nSee [payload]({link_target}).\n") + refs = skill_dir / "references" + refs.mkdir() + (refs / "payload.md.j2").write_text("# {{ 'rendered' }}\n") + + builder = SkillBuilder( + repo_root=tmp_path, + products_dir=tmp_path / "products", + output_dir=tmp_path / "output", + ) + assert builder.lint_all() is expected + + def test_lint_all_catches_duplicate_skill_names(tmp_path: Path) -> None: for product in ("alpha", "beta"): skill_dir = tmp_path / "products" / product / "skills" / "same-name"