From 1c5d371077e9c6fc8bd3c7e4354cb3995ecd04bb Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Mon, 3 Aug 2026 14:45:41 +0800 Subject: [PATCH 01/14] feat(knowledge): a code wiki is a knowledge base written by an agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A code wiki used to be its own world: its own tables, its own reader, its own permission rules, its own list. It is now a knowledge base whose content an agent generates from a source repository — which means retrieval, sharing, permissions and the document reader all apply to it without being rebuilt. **Versions, and a projection.** The agent writes into `wiki_generations` / `wiki_contents`, which no reader ever touches; publishing projects the chosen version into the knowledge base. The point is where the atomicity boundary sits: around a deterministic projection that takes seconds and can be retried, rather than around an LLM run that takes hours and fails in the middle. `spec.publishedGenerationId` is the sole authority for which version is live — never inferred from the newest completed generation, because a generation can finish and still be refused. **A page's path is its identity.** Documents are keyed by it, so rewording a heading revises the page instead of deleting and recreating it — which would change the document id that the RAG index and every stored citation depend on, and pay to re-embed the content. **Incremental versions are seeded before the agent starts**, so every version is a complete snapshot and orphans are a plain set difference rather than a reconciliation. **Ordering is fixed and asserted.** Blobs are written before the transaction commits and deleted after it: a failure before the commit leaves litter, while deleting first would destroy live content on a rollback. Attachments live in object storage and cannot join the transaction, so the ordering is the only thing protecting them. **The publish gate is what makes agent-declared deletion safe.** Removal is measured over the set of published paths a version no longer contains, not over page counts — a same-sized version under different paths is a mass deletion that counting cannot see. A refused version stays in the store with its verdict attached, and the published one is untouched. **Content ownership.** `origin` defaults to `user`, so anything the projection did not create is excluded from it. Getting that backwards would let a regeneration delete content nobody can restore. Generation is triggered directly for now; the repository's HEAD and diff are read from the provider so an unchanged repository costs one comparison instead of a full pass through the model. Every such read degrades to "unknown", which means a rebuild — a partial diff mistaken for a complete one would pick an incremental run for a change that reshaped the repository. Co-Authored-By: Claude Opus 5 --- backend/.env.example | 5 + ..._add_knowledge_content_origin_and_wiki_.py | 77 +++ backend/app/api/endpoints/knowledge.py | 156 ++++++ backend/app/core/celery_app.py | 7 + backend/app/core/wiki_config.py | 7 + backend/app/models/knowledge.py | 32 ++ backend/app/models/wiki.py | 5 + backend/app/repository/gitea_provider.py | 77 +++ backend/app/repository/github_provider.py | 103 +++- backend/app/repository/gitlab_provider.py | 80 +++ backend/app/schemas/kind.py | 21 +- backend/app/schemas/knowledge.py | 133 ++++- backend/app/schemas/wiki.py | 14 + .../knowledge/code_wiki_generation.py | 266 +++++++++ .../services/knowledge/code_wiki_page_path.py | 156 ++++++ .../knowledge/code_wiki_projection.py | 342 ++++++++++++ .../knowledge/code_wiki_projection_plan.py | 141 +++++ .../services/knowledge/code_wiki_prompts.py | 256 +++++++++ .../knowledge/code_wiki_publish_gate.py | 146 +++++ .../services/knowledge/code_wiki_publisher.py | 333 +++++++++++ .../knowledge/code_wiki_repo_state.py | 151 +++++ .../services/knowledge/code_wiki_run_mode.py | 224 ++++++++ .../services/knowledge/code_wiki_runner.py | 474 ++++++++++++++++ .../knowledge/code_wiki_side_effects.py | 145 +++++ .../services/knowledge/code_wiki_source.py | 261 +++++++++ .../knowledge/code_wiki_version_store.py | 337 +++++++++++ .../app/services/knowledge/content_scope.py | 76 +++ .../services/knowledge/knowledge_service.py | 38 +- .../app/services/knowledge/mermaid_check.py | 219 ++++++++ .../app/services/knowledge/orchestrator.py | 8 +- backend/app/services/wiki_service.py | 163 +++++- backend/app/tasks/knowledge_tasks.py | 16 + backend/init_data/02-public-resources.yaml | 104 ++++ backend/init_data/skills/wiki_submit/SKILL.md | 69 ++- .../skills/wiki_submit/wiki_submit.js | 94 +++- backend/tests/api/test_knowledge_code_wiki.py | 241 ++++++++ .../repository/test_repository_state_reads.py | 235 ++++++++ .../knowledge/test_code_wiki_cleanup_sweep.py | 202 +++++++ .../knowledge/test_code_wiki_content_write.py | 289 ++++++++++ .../knowledge/test_code_wiki_generation.py | 349 ++++++++++++ .../knowledge/test_code_wiki_page_path.py | 147 +++++ .../knowledge/test_code_wiki_projection.py | 389 +++++++++++++ .../test_code_wiki_projection_plan.py | 139 +++++ .../knowledge/test_code_wiki_prompts.py | 169 ++++++ .../test_code_wiki_publish_end_to_end.py | 322 +++++++++++ .../knowledge/test_code_wiki_publish_gate.py | 177 ++++++ .../knowledge/test_code_wiki_publisher.py | 402 ++++++++++++++ .../knowledge/test_code_wiki_repo_state.py | 202 +++++++ .../knowledge/test_code_wiki_run_mode.py | 257 +++++++++ .../knowledge/test_code_wiki_runner.py | 524 ++++++++++++++++++ .../knowledge/test_code_wiki_source.py | 246 ++++++++ .../test_code_wiki_submit_to_publish.py | 266 +++++++++ .../knowledge/test_code_wiki_version_store.py | 448 +++++++++++++++ .../services/knowledge/test_content_scope.py | 150 +++++ .../services/knowledge/test_mermaid_check.py | 194 +++++++ 55 files changed, 10018 insertions(+), 66 deletions(-) create mode 100644 backend/alembic/versions/20260731_bd9c871a93d2_add_knowledge_content_origin_and_wiki_.py create mode 100644 backend/app/services/knowledge/code_wiki_generation.py create mode 100644 backend/app/services/knowledge/code_wiki_page_path.py create mode 100644 backend/app/services/knowledge/code_wiki_projection.py create mode 100644 backend/app/services/knowledge/code_wiki_projection_plan.py create mode 100644 backend/app/services/knowledge/code_wiki_prompts.py create mode 100644 backend/app/services/knowledge/code_wiki_publish_gate.py create mode 100644 backend/app/services/knowledge/code_wiki_publisher.py create mode 100644 backend/app/services/knowledge/code_wiki_repo_state.py create mode 100644 backend/app/services/knowledge/code_wiki_run_mode.py create mode 100644 backend/app/services/knowledge/code_wiki_runner.py create mode 100644 backend/app/services/knowledge/code_wiki_side_effects.py create mode 100644 backend/app/services/knowledge/code_wiki_source.py create mode 100644 backend/app/services/knowledge/code_wiki_version_store.py create mode 100644 backend/app/services/knowledge/content_scope.py create mode 100644 backend/app/services/knowledge/mermaid_check.py create mode 100644 backend/tests/api/test_knowledge_code_wiki.py create mode 100644 backend/tests/repository/test_repository_state_reads.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_cleanup_sweep.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_content_write.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_generation.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_page_path.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_projection.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_projection_plan.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_prompts.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_publish_end_to_end.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_publish_gate.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_publisher.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_repo_state.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_run_mode.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_runner.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_source.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_submit_to_publish.py create mode 100644 backend/tests/services/knowledge/test_code_wiki_version_store.py create mode 100644 backend/tests/services/knowledge/test_content_scope.py create mode 100644 backend/tests/services/knowledge/test_mermaid_check.py diff --git a/backend/.env.example b/backend/.env.example index fb12ed109f..f956864b47 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -213,6 +213,11 @@ WIKI_ENABLED=True # The team's bot MUST have a model configured (bind_model or custom config) # If no model is configured, wiki generation will be disabled in the frontend WIKI_DEFAULT_TEAM_NAME=wiki-team +# Team that runs knowledge-base code wikis (matches init_data/02-public-resources.yaml) +# Separate from WIKI_DEFAULT_TEAM_NAME: the two paths give their agents different +# instructions and different submission rules, so they cannot share one team. +# Its bot MUST have a model configured (bind_model or custom config) +WIKI_CODE_WIKI_TEAM_NAME=code-wiki-team # Default agent type for wiki generation WIKI_DEFAULT_AGENT_TYPE=ClaudeCode # Default language for wiki documentation generation (en = English, zh = Chinese) diff --git a/backend/alembic/versions/20260731_bd9c871a93d2_add_knowledge_content_origin_and_wiki_.py b/backend/alembic/versions/20260731_bd9c871a93d2_add_knowledge_content_origin_and_wiki_.py new file mode 100644 index 0000000000..fb36d68e6c --- /dev/null +++ b/backend/alembic/versions/20260731_bd9c871a93d2_add_knowledge_content_origin_and_wiki_.py @@ -0,0 +1,77 @@ +"""add knowledge content origin and wiki generation kb link + +Revision ID: bd9c871a93d2 +Revises: b9c0d1e2f3a4 + +Adds the two things a code wiki needs to coexist with ordinary knowledge content: + +- ``origin`` on documents and folders, marking whether a row is agent-generated or + user-owned. It defaults to ``user`` so that every existing row, and every row this + service does not create itself, is excluded from the generated-content projection. + Getting this backwards would let a regeneration delete content nobody can restore. +- ``kind_id`` on wiki generations, binding a version line to a knowledge base. The + versions previously hung off ``wiki_projects``, whose ``source_url`` is globally + unique; leaving them there would share one version line between knowledge bases + tracking the same repository. ``0`` marks rows that predate code wikis. + +All three columns are NOT NULL with a server default, as required for existing rows to +backfill without a nullable intermediate state. +""" + +from collections.abc import Sequence +from typing import Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "bd9c871a93d2" +down_revision: Union[str, Sequence[str], None] = "b9c0d1e2f3a4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.add_column( + "knowledge_documents", + sa.Column( + "origin", + sa.String(length=20), + nullable=False, + server_default="user", + comment="Content ownership: 'generated' (agent-owned) or 'user'", + ), + ) + op.add_column( + "knowledge_folders", + sa.Column( + "origin", + sa.String(length=20), + nullable=False, + server_default="user", + comment="Content ownership: 'generated' (agent-owned) or 'user'", + ), + ) + op.add_column( + "wiki_generations", + sa.Column( + "kind_id", + sa.Integer(), + nullable=False, + server_default="0", + comment="Knowledge base this version line belongs to; 0 = legacy row", + ), + ) + op.create_index( + "ix_wiki_generations_kind_id", "wiki_generations", ["kind_id"], unique=False + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index("ix_wiki_generations_kind_id", table_name="wiki_generations") + op.drop_column("wiki_generations", "kind_id") + op.drop_column("knowledge_folders", "origin") + op.drop_column("knowledge_documents", "origin") diff --git a/backend/app/api/endpoints/knowledge.py b/backend/app/api/endpoints/knowledge.py index 69d25c6848..c1491abaa9 100644 --- a/backend/app/api/endpoints/knowledge.py +++ b/backend/app/api/endpoints/knowledge.py @@ -35,6 +35,9 @@ AllGroupedKnowledgeResponse, BatchDocumentIds, BatchOperationResult, + CodeWikiCreate, + CodeWikiRunCreate, + CodeWikiRunResponse, DocumentContentUpdate, DocumentDetailResponse, DocumentMoveRequest, @@ -42,6 +45,7 @@ KnowledgeBaseCreate, KnowledgeBaseListResponse, KnowledgeBaseResponse, + KnowledgeBaseType, KnowledgeBaseTypeUpdate, KnowledgeBaseUpdate, KnowledgeDocumentCreate, @@ -64,6 +68,14 @@ KnowledgeService, knowledge_base_qa_service, ) +from app.services.knowledge.code_wiki_generation import GenerationInFlight +from app.services.knowledge.code_wiki_run_mode import ChangedPath +from app.services.knowledge.code_wiki_runner import CodeWikiRunError, start_run +from app.services.knowledge.code_wiki_source import ( + SourceAccessDenied, + SourceRepository, + assert_user_can_read_source, +) from app.services.knowledge.orchestrator import ( DEFAULT_KNOWLEDGE_LIST_LIMIT, MAX_DOCUMENT_READ_LIMIT, @@ -424,6 +436,18 @@ def create_knowledge_base( - **namespace=**: Team knowledge base (requires Maintainer+ permission) - **members**: Optional initial members to add after creation """ + # A code wiki must be created through its own endpoint, which binds a repository + # and checks the caller can read it. Refusing here rather than quietly ignoring + # the field keeps this endpoint from becoming a way around that check. + if data.kb_type == KnowledgeBaseType.CODE_WIKI: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Use POST /knowledge-bases/code-wikis to create a " + "'code_wiki' knowledge base" + ), + ) + try: # Use Orchestrator for unified business logic (REST API and MCP tools share the same logic) result = knowledge_orchestrator.create_knowledge_base( @@ -468,6 +492,138 @@ def create_knowledge_base( ) +@router.post( + "/code-wikis", + response_model=KnowledgeBaseResponse, + status_code=status.HTTP_201_CREATED, +) +@trace_sync("create_code_wiki", "knowledge.api") +def create_code_wiki( + data: CodeWikiCreate, + current_user: User = Depends(security.get_current_user), + db: Session = Depends(get_db), +): + """Create a code wiki bound to a source repository. + + The requester must be able to read the repository, so that a wiki cannot be built + for a private repository they have no access to. Reading the resulting wiki is + then governed by knowledge-base permissions alone: place it in an organization + namespace to make it readable by everyone signed in, or keep it in a restricted + namespace and share it explicitly. + """ + try: + source = SourceRepository.from_url(data.source_type, data.source_url) + assert_user_can_read_source(db, current_user.id, source) + except SourceAccessDenied as e: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) from e + + try: + result = knowledge_orchestrator.create_knowledge_base( + db=db, + user=current_user, + name=data.name, + description=data.description, + namespace=data.namespace or "default", + kb_type=KnowledgeBaseType.CODE_WIKI.value, + source=source, + ) + add_span_event( + "knowledge.code_wiki.created", + { + "kb_id": str(result.id), + "project_name": source.project_name, + "namespace": data.namespace or "default", + "user_id": str(current_user.id), + }, + ) + return result + except IntegrityError as e: + db.rollback() + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Knowledge base with name '{data.name}' already exists in this namespace", + ) from e + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) + ) from e + + +@router.post( + "/{knowledge_base_id}/code-wiki/generations", + response_model=CodeWikiRunResponse, + status_code=status.HTTP_202_ACCEPTED, +) +@trace_sync("start_code_wiki_run", "knowledge.api") +def start_code_wiki_run( + knowledge_base_id: int, + data: CodeWikiRunCreate, + current_user: User = Depends(security.get_current_user), + db: Session = Depends(get_db), +): + """Regenerate a code wiki now, without waiting for its schedule. + + Managing the knowledge base is required rather than merely reading it: a run + rewrites every page in it, so this is closer to replacing its content than to + viewing it. + + Answers 202 even when no run was needed. "The repository has not changed since the + published version" is a successful outcome, not a failure, and the response says + which it was. + """ + knowledge_base = KnowledgeService._get_knowledge_base_record(db, knowledge_base_id) + if knowledge_base is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Knowledge base not found" + ) + if not KnowledgeService.can_manage_knowledge_base( + db, knowledge_base_id, current_user.id + ): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="You do not have permission to regenerate this code wiki", + ) + + try: + started = start_run( + db, + knowledge_base=knowledge_base, + user=current_user, + head_commit=data.head_commit, + changed_paths=( + None + if data.changed_paths is None + else [ + ChangedPath(path=item.path, status=item.change_type) + for item in data.changed_paths + ] + ), + ) + except GenerationInFlight as e: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) from e + except CodeWikiRunError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) + ) from e + + add_span_event( + "knowledge.code_wiki.run_requested", + { + "kb_id": str(knowledge_base_id), + "started": str(started.started), + "mode": started.mode, + "user_id": str(current_user.id), + }, + ) + return CodeWikiRunResponse( + started=started.started, + mode=started.mode, + reason=started.reason, + generation_id=started.generation.id if started.generation else 0, + task_id=started.task_id, + ) + + @router.get("/{knowledge_base_id}", response_model=KnowledgeBaseResponse) @trace_sync("get_knowledge_base", "knowledge.api") def get_knowledge_base( diff --git a/backend/app/core/celery_app.py b/backend/app/core/celery_app.py index 9aefb32e3d..721a37ecd7 100644 --- a/backend/app/core/celery_app.py +++ b/backend/app/core/celery_app.py @@ -84,6 +84,13 @@ "task": "app.tasks.knowledge_tasks.scan_stale_index_tasks", "schedule": 5 * 60, # every 5 minutes }, + "sweep-code-wiki-index-cleanup": { + "task": "app.tasks.knowledge_tasks.sweep_code_wiki_index_cleanup", + # Less often than the stale scan: this drains a list that only grows when + # the vector store is already failing, and retrying a broken one every + # five minutes adds load exactly when it is least welcome. + "schedule": 15 * 60, + }, }, # Beat scheduler class - Use default PersistentScheduler (file-based) # Note: Only run ONE Celery Beat instance in production diff --git a/backend/app/core/wiki_config.py b/backend/app/core/wiki_config.py index d8858ddc82..6c2a0229a5 100644 --- a/backend/app/core/wiki_config.py +++ b/backend/app/core/wiki_config.py @@ -19,6 +19,13 @@ class WikiSettings(BaseSettings): "wiki-team" # Default execution team name (matches init_data/01-default-resources.yaml) ) DEFAULT_AGENT_TYPE: str = "ClaudeCode" # Default agent type + # Team that runs knowledge-base code wikis (env var: WIKI_CODE_WIKI_TEAM_NAME). + # Separate from DEFAULT_TEAM_NAME because the two paths hand their agents + # different instructions and different submission rules; pointing both at one team + # would give the legacy wiki the page-path write contract it cannot satisfy. + CODE_WIKI_TEAM_NAME: str = ( + "code-wiki-team" # Matches init_data/02-public-resources.yaml + ) DEFAULT_USER_ID: int = 0 # Default user ID for task creation (0 = use current user) DEFAULT_LANGUAGE: str = ( "en" # Default language for wiki documentation generation (en/zh) diff --git a/backend/app/models/knowledge.py b/backend/app/models/knowledge.py index 1f23ca7b60..36cac6f850 100644 --- a/backend/app/models/knowledge.py +++ b/backend/app/models/knowledge.py @@ -47,6 +47,22 @@ class DocumentSourceType(str, PyEnum): TEXT = "text" # Pasted text TABLE = "table" # External table (DingTalk, Feishu, etc.) WEB = "web" # Web page (scraped URL) + CODE = "code" # Source file indexed for retrieval, not a browsable document + + +class ContentOrigin(str, PyEnum): + """Ownership of knowledge content, deciding whether regeneration may touch it. + + ``GENERATED`` content is owned by the generating agent: it may be overwritten by + a later run and removed by generation-set reconciliation. ``USER`` content is + owned by a person and is never touched by regeneration. + + ``USER`` is the safe default: mislabelling generated content as user-owned only + stops automatic cleanup, while the reverse would expose user content to deletion. + """ + + GENERATED = "generated" + USER = "user" class DocumentIndexStatus(str, PyEnum): @@ -131,6 +147,14 @@ class KnowledgeDocument(Base): source_config = Column( JSON, nullable=False, default={} ) # Source configuration (e.g., {"url": "..."} for table) + # Content ownership; see ContentOrigin. Defaults to ``user`` so that any row this + # service did not create is protected from the generated-content projection. + origin = Column( + String(20), + nullable=False, + default=ContentOrigin.USER.value, + server_default=ContentOrigin.USER.value, + ) # --- Helper properties for converted attachment reference --- @@ -210,6 +234,14 @@ class KnowledgeFolder(Base): # Self-referencing parent folder, 0 = root level parent_id = Column(Integer, nullable=False, default=0) name = Column(String(255), nullable=False) + # Content ownership; a top-level folder decides ownership for its subtree. + # Defaults to ``user`` for the same fail-safe reason as on documents. + origin = Column( + String(20), + nullable=False, + default=ContentOrigin.USER.value, + server_default=ContentOrigin.USER.value, + ) created_at = Column(DateTime, nullable=False, default=func.now()) updated_at = Column( DateTime, nullable=False, default=func.now(), onupdate=func.now() diff --git a/backend/app/models/wiki.py b/backend/app/models/wiki.py index d653b50e41..c3ef86f1db 100644 --- a/backend/app/models/wiki.py +++ b/backend/app/models/wiki.py @@ -82,6 +82,11 @@ class WikiGeneration(WikiBase): nullable=False, index=True, ) + # Knowledge base this version line belongs to (references kinds.id, no FK). + # Versions are owned by the KB rather than by the project: wiki_projects.source_url + # is globally unique, so project-scoped versions would be shared between knowledge + # bases tracking the same repository. 0 marks a row predating code_wiki. + kind_id = Column(Integer, nullable=False, default=0, server_default="0", index=True) user_id = Column(Integer, nullable=False, index=True) task_id = Column(big_integer_id_type(), nullable=False, default=0, index=True) team_id = Column(Integer, nullable=False) diff --git a/backend/app/repository/gitea_provider.py b/backend/app/repository/gitea_provider.py index 298297fe72..d2eda90f1d 100644 --- a/backend/app/repository/gitea_provider.py +++ b/backend/app/repository/gitea_provider.py @@ -22,6 +22,22 @@ from shared.utils.sensitive_data_masker import mask_string from shared.utils.url_util import build_url +# Reading repository state happens on a scheduled path, but an unresponsive instance +# must still fail rather than hold the worker. +REPO_STATE_TIMEOUT_SECONDS = 15 + +# Compare statuses mapped to git's name-status letters, which is what the code wiki +# run-mode rules are written against. +_GITEA_FILE_STATUS = { + "added": "A", + "removed": "D", + "deleted": "D", + "modified": "M", + "renamed": "R", + "copied": "A", + "changed": "M", +} + class GiteaProvider(RepositoryProvider): """ @@ -850,3 +866,64 @@ def check_user_project_access( except requests.exceptions.RequestException as e: self.logger.error(f"Failed to check Gitea repository access: {str(e)}") raise HTTPException(status_code=502, detail=f"Gitea API error: {str(e)}") + + # ---- repository state, for deciding whether a code wiki needs regenerating ---- + + def get_default_branch_head( + self, token: str, git_domain: str, repo_name: str + ) -> Dict[str, str]: + """Return the default branch and the commit it points at.""" + api_base_url = self._get_api_base_url(git_domain) + headers = self._build_headers(token) + + repo = requests.get( + f"{api_base_url}/repos/{repo_name}", + headers=headers, + timeout=REPO_STATE_TIMEOUT_SECONDS, + ) + repo.raise_for_status() + branch_name = (repo.json() or {}).get("default_branch") or "main" + + branch = requests.get( + f"{api_base_url}/repos/{repo_name}/branches/{branch_name}", + headers=headers, + timeout=REPO_STATE_TIMEOUT_SECONDS, + ) + branch.raise_for_status() + commit = (branch.json() or {}).get("commit") or {} + return {"branch": branch_name, "commit": commit.get("id", "")} + + def get_changed_files( + self, token: str, git_domain: str, repo_name: str, base: str, head: str + ) -> Optional[List[Dict[str, str]]]: + """List the files that changed between two commits. + + Returns: + One entry per file, or ``None`` when this Gitea cannot answer. The + compare endpoint arrived in Gitea 1.17 and self-hosted instances lag, so + an older one reports the diff as unknown and gets a full rebuild rather + than an error. + """ + api_base_url = self._get_api_base_url(git_domain) + response = requests.get( + f"{api_base_url}/repos/{repo_name}/compare/{base}...{head}", + headers=self._build_headers(token), + timeout=REPO_STATE_TIMEOUT_SECONDS, + ) + if response.status_code == 404: + self.logger.info( + "Gitea at %s has no compare endpoint; reporting the diff as unknown", + git_domain, + ) + return None + response.raise_for_status() + + payload = response.json() or {} + return [ + { + "path": entry.get("filename", ""), + "status": _GITEA_FILE_STATUS.get(entry.get("status", ""), "M"), + } + for entry in (payload.get("files") or []) + if entry.get("filename") + ] diff --git a/backend/app/repository/github_provider.py b/backend/app/repository/github_provider.py index 4b6d684ba0..316efb738e 100644 --- a/backend/app/repository/github_provider.py +++ b/backend/app/repository/github_provider.py @@ -22,6 +22,26 @@ from shared.utils.sensitive_data_masker import mask_string from shared.utils.url_util import build_url +# Repository access checks run inside a user-facing request (creating a code wiki), +# so an unresponsive provider must fail rather than hold the worker. Only this check +# is bounded here; the other calls in this module remain as they were. +ACCESS_CHECK_TIMEOUT_SECONDS = 15 + +# GitHub's compare endpoint returns at most this many files and gives no reliable +# signal that it truncated, so hitting it is treated as "the diff is unknown". +GITHUB_COMPARE_FILE_LIMIT = 300 + +# Compare statuses mapped to git's name-status letters, which is what the code wiki +# run-mode rules are written against. +_GITHUB_FILE_STATUS = { + "added": "A", + "removed": "D", + "modified": "M", + "renamed": "R", + "copied": "A", + "changed": "M", +} + class GitHubProvider(RepositoryProvider): """ @@ -859,7 +879,11 @@ def check_user_project_access( # First get the current user info from the token try: - user_response = requests.get(f"{api_base_url}/user", headers=headers) + user_response = requests.get( + f"{api_base_url}/user", + headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) if user_response.status_code == 401: return { "has_access": False, @@ -880,6 +904,7 @@ def check_user_project_access( permission_response = requests.get( f"{api_base_url}/repos/{repo_name}/collaborators/{username}/permission", headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, ) if permission_response.status_code == 404: @@ -924,6 +949,7 @@ def check_user_project_access( repo_response = requests.get( f"{api_base_url}/repos/{repo_name}", headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, ) if repo_response.status_code == 200: return { @@ -942,3 +968,78 @@ def check_user_project_access( } self.logger.error(f"Failed to check repository access: {str(e)}") raise HTTPException(status_code=502, detail=f"GitHub API error: {str(e)}") + + # ---- repository state, for deciding whether a code wiki needs regenerating ---- + + def get_default_branch_head( + self, token: str, git_domain: str, repo_name: str + ) -> Dict[str, str]: + """Return the default branch and the commit it points at. + + Two targeted calls rather than ``get_branches``, which pages through the + whole branch list to answer a question about one branch. + """ + api_base_url = self._get_api_base_url(git_domain) + headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + } + + repo = requests.get( + f"{api_base_url}/repos/{repo_name}", + headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + repo.raise_for_status() + branch_name = repo.json().get("default_branch") or "main" + + branch = requests.get( + f"{api_base_url}/repos/{repo_name}/branches/{branch_name}", + headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + branch.raise_for_status() + commit = (branch.json() or {}).get("commit") or {} + return {"branch": branch_name, "commit": commit.get("sha", "")} + + def get_changed_files( + self, token: str, git_domain: str, repo_name: str, base: str, head: str + ) -> Optional[List[Dict[str, str]]]: + """List the files that changed between two commits. + + Returns: + One entry per file, or ``None`` when the answer would be incomplete. + GitHub truncates ``compare`` at 300 files, and a partial diff read as a + complete one would pick an incremental run for a change big enough to + need a rebuild. + """ + api_base_url = self._get_api_base_url(git_domain) + response = requests.get( + f"{api_base_url}/repos/{repo_name}/compare/{base}...{head}", + headers={ + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + }, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + response.raise_for_status() + payload = response.json() or {} + + files = payload.get("files") or [] + if len(files) >= GITHUB_COMPARE_FILE_LIMIT: + self.logger.info( + "Compare of %s is at GitHub's %s file limit; reporting the diff as " + "unknown rather than truncated", + repo_name, + GITHUB_COMPARE_FILE_LIMIT, + ) + return None + + return [ + { + "path": entry.get("filename", ""), + "status": _GITHUB_FILE_STATUS.get(entry.get("status", ""), "M"), + } + for entry in files + if entry.get("filename") + ] diff --git a/backend/app/repository/gitlab_provider.py b/backend/app/repository/gitlab_provider.py index 85c8126c94..c58e7cb3e2 100644 --- a/backend/app/repository/gitlab_provider.py +++ b/backend/app/repository/gitlab_provider.py @@ -9,6 +9,7 @@ import asyncio import logging from typing import Any, Dict, List, Optional +from urllib.parse import quote import requests from fastapi import HTTPException @@ -20,6 +21,11 @@ from app.schemas.github import Branch, Repository from shared.utils.url_util import build_url +# Repository access checks run inside a user-facing request (creating a code wiki), +# so an unresponsive provider must fail rather than hold the worker. Only this check +# is bounded here; the other calls in this module remain as they were. +ACCESS_CHECK_TIMEOUT_SECONDS = 15 + class GitLabProvider(RepositoryProvider): """ @@ -894,6 +900,7 @@ def check_user_project_access( method="GET", url=f"{api_base_url}/user", token=decrypt_token, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, ) user_data = user_response.json() user_id = user_data.get("id") @@ -920,6 +927,7 @@ def check_user_project_access( method="GET", url=f"{api_base_url}/projects/{encoded_project_id}/members/all/{user_id}", token=decrypt_token, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, ) member_data = member_response.json() access_level = member_data.get("access_level", 0) @@ -949,3 +957,75 @@ def check_user_project_access( } self.logger.error(f"Failed to check project access: {str(e)}") raise HTTPException(status_code=502, detail=f"GitLab API error: {str(e)}") + + # ---- repository state, for deciding whether a code wiki needs regenerating ---- + + def get_default_branch_head( + self, token: str, git_domain: str, repo_name: str + ) -> Dict[str, str]: + """Return the default branch and the commit it points at.""" + api_base_url = self._get_api_base_url(git_domain) + encoded = quote(repo_name, safe="") + + project = self._make_request_with_auth_retry( + method="GET", + url=f"{api_base_url}/projects/{encoded}", + token=token, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + branch_name = (project.json() or {}).get("default_branch") or "main" + + branch = self._make_request_with_auth_retry( + method="GET", + url=f"{api_base_url}/projects/{encoded}/repository/branches/" + f"{quote(branch_name, safe='')}", + token=token, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + commit = (branch.json() or {}).get("commit") or {} + return {"branch": branch_name, "commit": commit.get("id", "")} + + def get_changed_files( + self, token: str, git_domain: str, repo_name: str, base: str, head: str + ) -> Optional[List[Dict[str, str]]]: + """List the files that changed between two commits. + + Returns: + One entry per file, or ``None`` when GitLab reports that it gave up + computing the diff. A partial diff read as a complete one would pick an + incremental run for a change big enough to need a rebuild. + """ + api_base_url = self._get_api_base_url(git_domain) + encoded = quote(repo_name, safe="") + + response = self._make_request_with_auth_retry( + method="GET", + url=f"{api_base_url}/projects/{encoded}/repository/compare", + token=token, + params={"from": base, "to": head}, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + payload = response.json() or {} + + if payload.get("compare_timeout"): + self.logger.info( + "GitLab timed out comparing %s; reporting the diff as unknown", + repo_name, + ) + return None + + changed: List[Dict[str, str]] = [] + for entry in payload.get("diffs") or []: + path = entry.get("new_path") or entry.get("old_path") or "" + if not path: + continue + if entry.get("new_file"): + status = "A" + elif entry.get("deleted_file"): + status = "D" + elif entry.get("renamed_file"): + status = "R" + else: + status = "M" + changed.append({"path": path, "status": status}) + return changed diff --git a/backend/app/schemas/kind.py b/backend/app/schemas/kind.py index e9dc763df2..f2352883d2 100644 --- a/backend/app/schemas/kind.py +++ b/backend/app/schemas/kind.py @@ -502,7 +502,7 @@ class TeamSpec(QuickPhraseMixin): """Team specification""" members: List[TeamMember] - collaborationModel: str # pipeline、route、coordinate、collaborate + collaborationModel: str # solo、pipeline、route、coordinate、collaborate bind_mode: Optional[List[str]] = None # ['chat', 'code'] or empty list for none description: Optional[str] = None # Team description icon: Optional[str] = None # Icon ID from preset icon library @@ -1008,7 +1008,24 @@ class KnowledgeBaseSpec(BaseModel): ) kbType: Optional[str] = Field( "notebook", - description="Default opening view: 'notebook' opens Notebook view by default, 'classic' opens document view by default", + description=( + "What this knowledge base is: 'notebook' or 'classic' select the default " + "opening view and may be switched freely; 'code_wiki' binds the knowledge " + "base to a source repository and is fixed at creation." + ), + ) + source: Optional[Dict[str, Any]] = Field( + None, + description="Source repository a code wiki is generated from", + ) + publishedGenerationId: int = Field( + 0, + description=( + "Generation whose content is currently projected into this knowledge base; " + "0 means nothing has been published yet. Sole authority for which version " + "is live — never infer it from the latest completed generation, because a " + "generation that finished but failed its publish gate is not published." + ), ) document_count: Optional[int] = Field( default=0, description="Cached document count" diff --git a/backend/app/schemas/knowledge.py b/backend/app/schemas/knowledge.py index 35eb41d663..67ad5fc820 100644 --- a/backend/app/schemas/knowledge.py +++ b/backend/app/schemas/knowledge.py @@ -54,6 +54,9 @@ class DocumentSourceType(str, Enum): TABLE = "table" WEB = "web" ATTACHMENT = "attachment" + # Source file indexed for retrieval rather than a browsable document. Declared so + # ensure_source_type_enum does not silently coerce it to FILE. + CODE = "code" class DocumentIndexStatus(str, Enum): @@ -77,6 +80,25 @@ class ResourceScope(str, Enum): ALL = "all" +class KnowledgeBaseType(str, Enum): + """What a knowledge base is, stored at ``spec.kbType``. + + ``notebook`` and ``classic`` differ only in the default opening view and can be + switched freely. ``code_wiki`` is a different thing altogether: it is bound to a + source repository, generated and maintained by an agent, and published by the + server rather than edited by hand. + + The three are mutually exclusive, so one field carries all of them. A knowledge + base may move between ``notebook`` and ``classic``, but **never** into or out of + ``code_wiki``: doing so would either orphan a repository binding and its version + history, or produce a code wiki with no repository at all. + """ + + NOTEBOOK = "notebook" + CLASSIC = "classic" + CODE_WIKI = "code_wiki" + + # ============== Knowledge Base Schemas ============== # Note: RetrieverRef, EmbeddingModelRef, HybridWeights, RetrievalConfig # are imported from app.schemas.kind to maintain single source of truth @@ -134,9 +156,17 @@ class KnowledgeBaseCreate(MultimodalAnalysisFieldsMixin): default="read", description="Minimum capability required for direct knowledge base access", ) - kb_type: Optional[str] = Field( - "notebook", - description="Default opening view: 'notebook' opens Notebook view by default, 'classic' opens document view by default", + kb_type: KnowledgeBaseType = Field( + KnowledgeBaseType.NOTEBOOK, + description=( + "'notebook' opens Notebook view by default, 'classic' opens document view. " + "'code_wiki' is a repository-backed, agent-generated wiki and can only be " + "created through the dedicated code wiki endpoint." + ), + ) + source: Optional[Dict[str, Any]] = Field( + None, + description="Source repository a code wiki is generated from (code wikis only)", ) retrieval_config: Optional[RetrievalConfigCreate] = Field( None, description="Retrieval configuration" @@ -267,8 +297,95 @@ def validate_guided_questions(cls, v): return v +class CodeWikiCreate(BaseModel): + """Request to create a code wiki bound to a source repository. + + Kept separate from ``KnowledgeBaseCreate`` because a code wiki needs a repository + and passes a repository-access gate, neither of which applies to other knowledge + bases. Its kind is set by the endpoint, not by the caller. + """ + + name: str = Field(..., min_length=1, max_length=100) + description: Optional[str] = Field(None, max_length=500) + namespace: str = Field( + default="default", + max_length=255, + description=( + "Namespace to create the wiki in. Use an organization namespace to make " + "it readable by everyone signed in." + ), + ) + source_type: Literal["github", "gitlab", "gitea"] = Field( + ..., + description=( + "Which platform hosts the repository. Required because a self-hosted " + "GitLab or Gitea cannot be told apart by its domain." + ), + ) + source_url: str = Field( + ..., + min_length=1, + max_length=500, + description=( + "Repository URL. The host and project are derived from it, so the " + "repository checked for access is necessarily the one that gets cloned." + ), + ) + + +class CodeWikiChangedPath(BaseModel): + """One file the repository changed since the published commit.""" + + path: str = Field(..., min_length=1, max_length=1000) + change_type: Literal["A", "M", "D", "R"] = Field( + "M", + description="Git status letter: A added, M modified, D deleted, R renamed", + ) + + +class CodeWikiRunCreate(BaseModel): + """Request to regenerate a code wiki now. + + Both fields are optional and describe the repository's current state. Supplying + neither is safe but expensive: with no commit to compare against, the run cannot + tell what changed and rebuilds the whole wiki rather than guessing. + """ + + head_commit: str = Field( + "", + max_length=64, + description=( + "Commit the repository is at now. Compared against the published commit " + "to decide whether anything needs regenerating at all." + ), + ) + changed_paths: Optional[List[CodeWikiChangedPath]] = Field( + None, + description=( + "Files changed since the published commit. Absent means unknown, which " + "forces a full rebuild; an empty list means nothing changed." + ), + ) + + +class CodeWikiRunResponse(BaseModel): + """What happened when a code wiki was asked to regenerate.""" + + started: bool = Field(..., description="Whether a run was actually created") + mode: str = Field("", description="Run mode chosen: full, incremental or skip") + reason: str = Field("", description="Why that mode was chosen") + generation_id: int = Field(0, description="The version being written, when started") + task_id: int = Field(0, description="Task running the agent, when started") + + class KnowledgeBaseTypeUpdate(BaseModel): - """Schema for updating the default opening view.""" + """Schema for updating the default opening view. + + The pattern deliberately excludes ``code_wiki``: it is not a view preference but a + binding to a source repository, so it can be neither adopted nor abandoned by + toggling a view. Turning a code wiki into a notebook would orphan its repository + and version history; the reverse would produce a code wiki with no repository. + """ kb_type: str = Field( ..., @@ -286,9 +403,9 @@ class KnowledgeBaseResponse(MultimodalAnalysisResponseFieldsMixin): user_id: int namespace: str direct_access_requirement: Literal["read", "edit"] = "read" - kb_type: Optional[str] = Field( - "notebook", - description="Default opening view: 'notebook' opens Notebook view by default, 'classic' opens document view by default", + kb_type: KnowledgeBaseType = Field( + KnowledgeBaseType.NOTEBOOK, + description="What this knowledge base is; see KnowledgeBaseType", ) document_count: int is_active: bool @@ -354,7 +471,7 @@ def from_kind(cls, kind, document_count: int = 0): # Extract summary_model_ref from spec summary_model_ref = spec.get("summaryModelRef") # Extract kb_type from spec, default to 'notebook' for backward compatibility - kb_type = spec.get("kbType", "notebook") + kb_type = spec.get("kbType", KnowledgeBaseType.NOTEBOOK.value) # Extract guided questions from spec guided_questions = spec.get("guidedQuestions") diff --git a/backend/app/schemas/wiki.py b/backend/app/schemas/wiki.py index 7c144bbe3d..a8d00776e6 100644 --- a/backend/app/schemas/wiki.py +++ b/backend/app/schemas/wiki.py @@ -136,6 +136,11 @@ class WikiContentSection(BaseModel): content: str parent_id: Optional[int] = None ext: Optional[Dict[str, Any]] = None + # Stable identity of the page, e.g. "architecture/backend". Pages are matched on + # it rather than on the title, so that rewording a heading revises the existing + # page instead of replacing it — which would change the document id the RAG index + # and stored citations depend on. Optional only for the legacy write path. + path: Optional[str] = None class WikiContentSummary(BaseModel): @@ -146,6 +151,11 @@ class WikiContentSummary(BaseModel): model: Optional[str] = None tokens_used: Optional[int] = None structure_order: Optional[List[str]] = None + # Commit the agent actually documented. It read the working tree, whereas the + # trigger only knew what it was told, and this is the value the next run's mode + # decision compares against — so a wrong one here costs a needless full rebuild + # or, worse, a skipped set of changes. + head_commit: Optional[str] = None class WikiContentWriteRequest(BaseModel): @@ -154,6 +164,10 @@ class WikiContentWriteRequest(BaseModel): generation_id: int sections: List[WikiContentSection] summary: Optional[WikiContentSummary] = None + # Pages the agent is declaring gone. Only it knows which page covered a module + # that no longer exists, and an incremental version starts as a copy of the + # published one, so not writing a page cannot mean removing it. + removed_paths: List[str] = Field(default_factory=list) class WikiContentCreate(BaseModel): diff --git a/backend/app/services/knowledge/code_wiki_generation.py b/backend/app/services/knowledge/code_wiki_generation.py new file mode 100644 index 0000000000..5c1d63fe68 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_generation.py @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Starting and finishing a code wiki generation run. + +This is the spine the rest of the pieces hang from. Everything else decides something +— which mode, which pages changed, whether a version may be published — but nothing +happens until a run is started and later concluded, and those are the two moments +where the invariants have to be enforced together: + +**Starting** picks a mode, refuses to begin while another run is genuinely in flight, +reclaims one whose worker is gone, creates the version and seeds it. Seeding belongs +here rather than in the agent because a version that is not a complete snapshot would +be projected as one, and every page the run did not touch would read as an orphan. + +**Finishing** records how the run ended and, only for a run that succeeded, offers the +version to the publish gate. A failed run leaves the published pointer where it was, +which is what makes the next scheduled run pick the work up again rather than skip it. +""" + +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Optional, Sequence + +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.user import User +from app.models.wiki import WikiGeneration, WikiGenerationStatus, WikiGenerationType +from app.services.knowledge.code_wiki_projection import ProjectionSideEffects +from app.services.knowledge.code_wiki_publish_gate import PublishPolicy +from app.services.knowledge.code_wiki_publisher import ( + PublishResult, + publish_generation, + published_generation_id, + read_version_pages, +) +from app.services.knowledge.code_wiki_run_mode import ( + ChangedPath, + RunMode, + RunModeDecision, + RunModePolicy, + decide_run_mode, +) +from app.services.knowledge.code_wiki_version_store import ( + reclaim_stale_generations, + seed_from_published, +) + +logger = logging.getLogger(__name__) + +SOURCE_COMMIT_KEY = "commit" + + +class GenerationInFlight(RuntimeError): + """Raised when a run is already working on this wiki.""" + + +@dataclass(frozen=True) +class StartedGeneration: + """A run that has been created and is ready for the agent.""" + + generation: Optional[WikiGeneration] + decision: RunModeDecision + seeded_pages: int = 0 + + @property + def started(self) -> bool: + return self.generation is not None + + +def published_commit(db: Session, knowledge_base: Kind) -> str: + """Commit the currently published wiki was generated from.""" + current = published_generation_id(knowledge_base) + if not current: + return "" + generation = db.get(WikiGeneration, current) + if generation is None: + return "" + return str((generation.source_snapshot or {}).get(SOURCE_COMMIT_KEY, "") or "") + + +def start_generation( + db: Session, + *, + knowledge_base: Kind, + user: User, + head_commit: str, + changed_paths: Optional[Sequence[ChangedPath]] = None, + total_source_files: Optional[int] = None, + team_id: int = 0, + task_id: int = 0, + policy: Optional[RunModePolicy] = None, + now: Optional[datetime] = None, +) -> StartedGeneration: + """Begin a run, unless there is nothing to do or one is already going. + + Args: + db: Session. + knowledge_base: The code wiki. + user: Identity the run executes under. + head_commit: Repository HEAD the run would document. + changed_paths: Diff since the published commit, or ``None`` when unknown — + in which case a full rebuild is chosen rather than a guess. + total_source_files: Repository size, used by the change-ratio threshold. + team_id: Team the generation task belongs to. + task_id: Task driving the run, when one exists yet. + policy: Thresholds promoting an incremental run to a full one. + now: Reference time, for tests. + + Returns: + The started run, or a decision explaining why none was needed. + + Raises: + GenerationInFlight: If a live run already owns this wiki. + """ + reclaimed = reclaim_stale_generations(db, kind_id=knowledge_base.id, now=now) + if reclaimed: + logger.warning( + "[code_wiki] kb %s had %s abandoned run(s) before starting", + knowledge_base.id, + len(reclaimed), + ) + + # Locked before the in-flight query, because that query locks only the rows it + # returns. With no live run there are no rows, so two concurrent starts would + # both find nothing and both insert — two agents documenting one repository, two + # seeded versions, and whichever finishes last silently overwrites the other. + # The knowledge base row always exists, so locking it serialises starts per wiki + # without depending on gap-lock behaviour that the isolation level can turn off. + db.query(Kind).filter(Kind.id == knowledge_base.id).with_for_update().first() + + in_flight = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == knowledge_base.id, + WikiGeneration.status.in_( + [WikiGenerationStatus.PENDING, WikiGenerationStatus.RUNNING] + ), + ) + .with_for_update() + .first() + ) + if in_flight is not None: + raise GenerationInFlight( + f"generation {in_flight.id} is already running for this wiki" + ) + + last_commit = published_commit(db, knowledge_base) + decision = decide_run_mode( + head_commit=head_commit, + last_commit=last_commit or None, + changed_paths=changed_paths, + total_source_files=total_source_files, + # Passed only when given: the callee defaults it, and forwarding ``None`` + # would replace that default with nothing. + **({"policy": policy} if policy is not None else {}), + ) + + if RunMode(decision.mode) == RunMode.SKIP: + logger.info( + "[code_wiki] kb %s needs no run: %s", knowledge_base.id, decision.reason + ) + return StartedGeneration(generation=None, decision=decision) + + generation = WikiGeneration( + project_id=0, + kind_id=knowledge_base.id, + user_id=user.id, + task_id=task_id, + team_id=team_id, + generation_type=( + WikiGenerationType.FULL + if RunMode(decision.mode) == RunMode.FULL + else WikiGenerationType.INCREMENTAL + ), + source_snapshot={SOURCE_COMMIT_KEY: head_commit}, + status=WikiGenerationStatus.RUNNING, + ext={"runModeReason": decision.reason}, + completed_at=datetime(1970, 1, 1), + ) + db.add(generation) + db.flush() + + seeded = 0 + if decision.seeds_from_published: + # Before the agent starts, so that it revises a complete snapshot rather than + # producing a partial one the projection would read as a mass deletion. + outcome = seed_from_published( + db, + target_generation_id=generation.id, + published_generation_id=published_generation_id(knowledge_base), + ) + seeded = outcome.copied_pages + + logger.info( + "[code_wiki] started %s generation %s for kb %s (%s), seeded %s pages", + decision.mode, + generation.id, + knowledge_base.id, + decision.reason, + seeded, + ) + return StartedGeneration( + generation=generation, decision=decision, seeded_pages=seeded + ) + + +def finish_generation( + db: Session, + *, + knowledge_base: Kind, + generation: WikiGeneration, + user: User, + effects: ProjectionSideEffects, + succeeded: bool, + error_message: str = "", + policy: Optional[PublishPolicy] = None, + now: Optional[datetime] = None, +) -> Optional[PublishResult]: + """Conclude a run and, if it succeeded, offer its version for publishing. + + A failed run deliberately leaves the published pointer alone. The next scheduled + run then still sees the repository as undocumented at its current commit and does + the work again, rather than skipping changes nobody has written up. + + Returns: + The publish outcome, or ``None`` when the run failed. + """ + finished = (now or datetime.now(timezone.utc)).replace(tzinfo=None) + + if not succeeded: + generation.status = WikiGenerationStatus.FAILED + generation.completed_at = finished + ext = dict(generation.ext or {}) + if error_message: + ext["errorMessage"] = error_message + generation.ext = ext + db.commit() + logger.warning( + "[code_wiki] generation %s failed for kb %s: %s", + generation.id, + knowledge_base.id, + error_message or "no reason given", + ) + return None + + generation.status = WikiGenerationStatus.COMPLETED + generation.completed_at = finished + db.flush() + + return publish_generation( + db, + knowledge_base=knowledge_base, + generation=generation, + user_id=user.id, + effects=effects, + policy=policy, + ) + + +def version_page_count(db: Session, generation_id: int) -> int: + """How many pages a version holds, for status displays.""" + return len(read_version_pages(db, generation_id)) diff --git a/backend/app/services/knowledge/code_wiki_page_path.py b/backend/app/services/knowledge/code_wiki_page_path.py new file mode 100644 index 0000000000..8433095809 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_page_path.py @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""The stable identity of a wiki page. + +A page is identified by its path, not by its title. The distinction matters because +the projection matches a published version against the knowledge base by path: a page +whose path is unchanged keeps its ``KnowledgeDocument`` row, and with it the document +id that the RAG index is keyed on and that stored citations point at. Matching on +title instead — as the original write API did — would turn every reworded heading into +a delete plus an insert, re-embedding the page and breaking references to it. + +Paths are also what the projection turns into folders, so they are validated here +against the same limits the folder tree enforces. Rejecting a bad path when the agent +writes it keeps the failure next to its cause; discovering it at publish time would +fail a whole version for one malformed entry. +""" + +import re +import unicodedata +from typing import Iterable + +from app.services.knowledge.folder_policy import MAX_FOLDER_DEPTH + +# Each segment becomes a folder or document name, both String(255). +MAX_SEGMENT_LENGTH = 255 + +# Bound on the whole path. Generous next to the depth and segment limits; it exists to +# stop pathological input rather than to constrain real page layouts. +MAX_PATH_LENGTH = 500 + +# Directory segments a path may have. The leaf is the document, so it does not count: +# ``a/b/c/d/page`` places a document in the deepest folder the tree allows. +MAX_DIRECTORY_DEPTH = MAX_FOLDER_DEPTH + +# Extension the projection appends when it materialises the page. Accepted on input and +# stripped, because a model writing markdown will naturally include it. +MARKDOWN_SUFFIX = ".md" + +_CONTROL_CHARACTERS = re.compile(r"[\x00-\x1f\x7f]") + +# Reserved by common filesystems and by our own path grammar. +_FORBIDDEN_IN_SEGMENT = set('\\:*?"<>|') + + +class InvalidPagePath(ValueError): + """Raised when a page path cannot be used as an identity.""" + + +def normalize_page_path(raw: str) -> str: + """Return the canonical form of ``raw``, or raise :class:`InvalidPagePath`. + + Normalization is deliberately limited to differences that carry no meaning — + surrounding whitespace, repeated separators, a trailing ``.md``. Anything + ambiguous is rejected rather than guessed at, so that a malformed path surfaces as + an error the agent can correct instead of silently becoming a different page. + """ + if not isinstance(raw, str): + raise InvalidPagePath("Page path must be a string") + + path = unicodedata.normalize("NFC", raw).strip() + if not path: + raise InvalidPagePath("Page path must not be empty") + + if _CONTROL_CHARACTERS.search(path): + raise InvalidPagePath("Page path must not contain control characters") + + if path.startswith("/"): + raise InvalidPagePath(f"Page path must be relative, got '{raw}'") + + if "\\" in path: + raise InvalidPagePath( + f"Page path must use '/' as its separator, got '{raw}'", + ) + + segments = [segment.strip() for segment in path.split("/")] + segments = [segment for segment in segments if segment] + if not segments: + raise InvalidPagePath(f"Page path has no usable segments: '{raw}'") + + # Strip the suffix before validating, so that every segment checked below is the + # one that ends up in the result. Validating first and stripping afterwards lets + # a leaf like "...md" pass the relative-segment check and then become ".." — the + # very thing the check exists to reject. + leaf = segments[-1] + if leaf.lower().endswith(MARKDOWN_SUFFIX): + leaf = leaf[: -len(MARKDOWN_SUFFIX)].strip() + if not leaf: + raise InvalidPagePath(f"Page path has an empty file name: '{raw}'") + segments[-1] = leaf + + for segment in segments: + if segment in (".", ".."): + raise InvalidPagePath( + f"Page path must not contain relative segments, got '{raw}'", + ) + if _FORBIDDEN_IN_SEGMENT & set(segment): + raise InvalidPagePath( + f"Page path segment '{segment}' contains a reserved character", + ) + if len(segment) > MAX_SEGMENT_LENGTH: + raise InvalidPagePath( + f"Page path segment exceeds {MAX_SEGMENT_LENGTH} characters", + ) + + if len(segments) - 1 > MAX_DIRECTORY_DEPTH: + raise InvalidPagePath( + f"Page path nests {len(segments) - 1} folders deep, over the " + f"{MAX_DIRECTORY_DEPTH} the folder tree allows: '{raw}'", + ) + + normalized = "/".join(segments) + if len(normalized) > MAX_PATH_LENGTH: + raise InvalidPagePath(f"Page path exceeds {MAX_PATH_LENGTH} characters") + + return normalized + + +def split_page_path(path: str) -> tuple[tuple[str, ...], str]: + """Split a normalized path into its folder segments and document name.""" + segments = path.split("/") + return tuple(segments[:-1]), segments[-1] + + +def collation_key(path: str) -> str: + """Return the key under which two paths count as the same page. + + Comparison is case-insensitive because the knowledge tables collate that way + (``utf8mb4_unicode_ci``): ``Architecture`` and ``architecture`` would resolve to + one folder in the database, so treating them as distinct pages in a version would + produce a collision the projection could not honour. + """ + return path.casefold() + + +def assert_unique_within_version(paths: Iterable[str]) -> None: + """Raise if any two paths in one version identify the same page. + + Args: + paths: Normalized paths making up a single version. + + Raises: + InvalidPagePath: If two entries collide, naming both originals. + """ + seen: dict[str, str] = {} + for path in paths: + key = collation_key(path) + if key in seen and seen[key] != path: + raise InvalidPagePath( + f"Page paths '{seen[key]}' and '{path}' differ only by case and " + "would resolve to the same page", + ) + if key in seen: + raise InvalidPagePath(f"Page path '{path}' appears more than once") + seen[key] = path diff --git a/backend/app/services/knowledge/code_wiki_projection.py b/backend/app/services/knowledge/code_wiki_projection.py new file mode 100644 index 0000000000..027b5c00fd --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_projection.py @@ -0,0 +1,342 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Applying a projection plan to the knowledge base. + +The ordering here is the whole point, and it is not negotiable: + + before the transaction write the new attachments (row and object) + inside the transaction repoint, insert, delete rows, drop empty folders + after the transaction delete superseded attachments, clear RAG, reindex + +Attachment bytes live in object storage in production, so they are outside the +transaction. Writing them first costs an orphaned object when a publish fails, which a +sweep collects. Deleting them first would destroy live content that a rollback then +cannot bring back — the same act, reordered, is the difference between litter and data +loss. + +The same reasoning forbids overwriting an attachment in place for an updated page: the +overwrite happens before the commit, so a failed transaction would leave the document +pointing at content that was never published and the previous content gone. Updates +therefore write a new attachment and repoint. + +``KnowledgeService.delete_document`` is deliberately not used, because it commits +internally — calling it per page would turn one transaction into many. Its cascade +still has to happen, though: deleting a document row without clearing the RAG index +leaves the retrieval layer answering from a page that no longer exists and citing an +id that resolves to nothing. Since the vector store cannot join the transaction, those +deletions are recorded and retried instead. +""" + +import logging +from dataclasses import dataclass, field +from typing import Callable, Optional, Protocol, Sequence + +from sqlalchemy.orm import Session + +from app.models.knowledge import ContentOrigin, KnowledgeDocument +from app.services.knowledge.code_wiki_page_path import split_page_path +from app.services.knowledge.code_wiki_projection_plan import ( + CONTENT_HASH_KEY, + PAGE_PATH_KEY, + PageSource, + ProjectionPlan, +) + +logger = logging.getLogger(__name__) + +# Key under which doc_refs awaiting RAG deletion are parked on the knowledge base. +# The vector store is external, so its cleanup cannot be part of the transaction; a +# failure there must survive a restart rather than vanish into a log line. +PENDING_INDEX_CLEANUP_KEY = "pendingIndexCleanup" + +DOCUMENT_EXTENSION = "md" + + +class AttachmentWriter(Protocol): + """Creates attachment content. Called only before the transaction commits.""" + + def __call__(self, *, filename: str, content: str) -> int: + """Return the id of a newly created attachment.""" + + +@dataclass +class ProjectionSideEffects: + """Everything the projection does outside its own transaction. + + Injected rather than imported so the ordering rules above can be asserted on + directly: a test can record the sequence of calls and prove that no attachment is + deleted before the commit. + """ + + write_attachment: AttachmentWriter + delete_attachment: Callable[[int], None] + delete_rag_document: Callable[[int], None] + enqueue_reindex: Callable[[int], None] + + +@dataclass(frozen=True) +class ProjectionOutcome: + """What the projection did.""" + + plan: ProjectionPlan + created_document_ids: tuple[int, ...] = () + updated_document_ids: tuple[int, ...] = () + deleted_document_ids: tuple[int, ...] = () + # doc_refs whose RAG deletion did not succeed and must be retried. + unfinished_index_cleanup: tuple[str, ...] = field(default=()) + + +def _folder_resolver(db: Session, kind_id: int, user_id: int): + """Return a function creating (or finding) the folder chain for a page path.""" + from app.models.knowledge import KnowledgeFolder + + cache: dict[tuple[int, str], int] = {} + + def resolve(segments: Sequence[str]) -> int: + parent_id = 0 + for segment in segments: + key = (parent_id, segment.casefold()) + if key in cache: + parent_id = cache[key] + continue + existing = ( + db.query(KnowledgeFolder) + .filter( + KnowledgeFolder.kind_id == kind_id, + KnowledgeFolder.parent_id == parent_id, + KnowledgeFolder.name == segment, + ) + .first() + ) + if existing is None: + existing = KnowledgeFolder( + kind_id=kind_id, + parent_id=parent_id, + name=segment, + origin=ContentOrigin.GENERATED.value, + ) + db.add(existing) + db.flush() + cache[key] = existing.id + parent_id = existing.id + return parent_id + + return resolve + + +def _stamp(document: KnowledgeDocument, source: PageSource) -> None: + """Record the page identity and fingerprint the next plan compares against.""" + config = dict(document.source_config or {}) + config[PAGE_PATH_KEY] = source.path + config[CONTENT_HASH_KEY] = source.fingerprint + document.source_config = config + + +def apply_projection_plan( + db: Session, + *, + kind_id: int, + user_id: int, + plan: ProjectionPlan, + effects: ProjectionSideEffects, +) -> ProjectionOutcome: + """Write a plan into the knowledge base. + + The caller owns the transaction and must commit; this function flushes but never + commits, so that a failure anywhere leaves the knowledge base exactly as it was. + + Args: + db: Session holding the projection transaction. + kind_id: Knowledge base being projected into. + user_id: Owner recorded on documents the projection creates. + plan: What to do, from ``compute_projection_plan``. + effects: Side effects outside the transaction. + + Returns: + What was done, including any RAG cleanup left to retry. + """ + resolve_folder = _folder_resolver(db, kind_id, user_id) + + # --- before the transaction commits: content into storage ------------------- + # Written first on purpose. An orphaned object is litter; content deleted before + # a rollback is gone. + new_attachments: dict[str, int] = {} + for source in (*plan.adds, *(update.source for update in plan.updates)): + _, leaf = split_page_path(source.path) + new_attachments[source.path] = effects.write_attachment( + filename=f"{leaf}.{DOCUMENT_EXTENSION}", + content=source.content, + ) + + def add_document(source: PageSource) -> int: + """Create the row for a page that is not in the knowledge base.""" + folders, leaf = split_page_path(source.path) + document = KnowledgeDocument( + kind_id=kind_id, + attachment_id=new_attachments[source.path], + name=leaf, + file_extension=DOCUMENT_EXTENSION, + file_size=len(source.content.encode("utf-8")), + user_id=user_id, + folder_id=resolve_folder(folders), + origin=ContentOrigin.GENERATED.value, + # Left inactive; the existing indexing state machine turns a document on + # once its index succeeds, which is how every other document behaves. + is_active=False, + ) + _stamp(document, source) + db.add(document) + db.flush() + return document.id + + # --- inside the transaction: rows only -------------------------------------- + created: list[int] = [] + for source in plan.adds: + created.append(add_document(source)) + + superseded_attachments: list[int] = [] + updated: list[int] = [] + for update in plan.updates: + document = db.get(KnowledgeDocument, update.existing.document_id) + if document is None: + # The row went away between planning and here. Skipping would leave the + # published version missing a page the plan accounted for, and strand the + # attachment already written for it, so it is added instead. + logger.warning( + "[code_wiki] document %s vanished before projection; adding it back", + update.existing.document_id, + ) + created.append(add_document(update.source)) + continue + if document.attachment_id: + superseded_attachments.append(document.attachment_id) + document.attachment_id = new_attachments[update.source.path] + document.file_size = len(update.source.content.encode("utf-8")) + _stamp(document, update.source) + updated.append(document.id) + + removed_refs: list[str] = [] + deleted: list[int] = [] + for page in plan.deletes: + document = db.get(KnowledgeDocument, page.document_id) + if document is None: + continue + if document.attachment_id: + superseded_attachments.append(document.attachment_id) + converted = document.converted_attachment_id + if converted: + superseded_attachments.append(converted) + # The RAG index keys documents by their id as a string; capture it before the + # row is gone, because after the commit there is nothing left to derive it from. + removed_refs.append(str(document.id)) + deleted.append(document.id) + db.delete(document) + + db.flush() + _remove_emptied_generated_folders(db, kind_id) + db.flush() + + return ProjectionOutcome( + plan=plan, + created_document_ids=tuple(created), + updated_document_ids=tuple(updated), + deleted_document_ids=tuple(deleted), + unfinished_index_cleanup=tuple(removed_refs), + ) + + +def finish_projection( + outcome: ProjectionOutcome, + *, + superseded_attachment_ids: Sequence[int], + effects: ProjectionSideEffects, +) -> tuple[str, ...]: + """Run the work that must wait until the transaction has committed. + + Every step here is retriable and none of it can undo the publish, so a failure is + reported rather than raised: the pages are already correct, and refusing to + acknowledge that would only cause the whole version to be produced again. + + Returns: + doc_refs whose RAG deletion failed and must be retried. + """ + for attachment_id in superseded_attachment_ids: + try: + effects.delete_attachment(attachment_id) + except Exception as exc: # pragma: no cover - defensive + logger.warning( + "[code_wiki] superseded attachment %s not deleted: %s", + attachment_id, + exc, + ) + + unfinished: list[str] = [] + for doc_ref in outcome.unfinished_index_cleanup: + try: + effects.delete_rag_document(int(doc_ref)) + except Exception as exc: + logger.warning( + "[code_wiki] RAG cleanup for doc_ref %s failed, will retry: %s", + doc_ref, + exc, + ) + unfinished.append(doc_ref) + + for document_id in ( + *outcome.created_document_ids, + *outcome.updated_document_ids, + ): + try: + effects.enqueue_reindex(document_id) + except Exception as exc: # pragma: no cover - defensive + logger.warning( + "[code_wiki] reindex not enqueued for document %s: %s", + document_id, + exc, + ) + + return tuple(unfinished) + + +def _remove_emptied_generated_folders(db: Session, kind_id: int) -> None: + """Drop generated folders left with no documents and no children. + + Scoped to generated folders: a user folder that happens to be empty is theirs to + keep, and the projection has no business tidying it away. + """ + from app.models.knowledge import KnowledgeFolder + + # Repeated because emptying a leaf can empty its parent. + while True: + folders = ( + db.query(KnowledgeFolder) + .filter( + KnowledgeFolder.kind_id == kind_id, + KnowledgeFolder.origin == ContentOrigin.GENERATED.value, + ) + .all() + ) + if not folders: + return + + occupied_parents = {folder.parent_id for folder in folders if folder.parent_id} + used_folder_ids = { + row[0] + for row in db.query(KnowledgeDocument.folder_id) + .filter(KnowledgeDocument.kind_id == kind_id) + .distinct() + .all() + } + + empties = [ + folder + for folder in folders + if folder.id not in used_folder_ids and folder.id not in occupied_parents + ] + if not empties: + return + for folder in empties: + db.delete(folder) + db.flush() diff --git a/backend/app/services/knowledge/code_wiki_projection_plan.py b/backend/app/services/knowledge/code_wiki_projection_plan.py new file mode 100644 index 0000000000..11583593d6 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_projection_plan.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Working out what projecting a version would change. + +The plan is computed and returned before anything is written, so that the decision +and its execution are separable: it can be logged, asserted on in tests, and run as a +dry run. "This publish removes 37 pages" is the kind of fact that should be visible +and refusable, which it cannot be if the deletions only exist as they happen. + +Matching is by page path, never by title, because the path is what keeps a page's +``KnowledgeDocument`` id — and with it its RAG index entry and any stored citation — +stable across regenerations. + +Both sides are complete snapshots: a version is seeded so that it holds every page, +not only the ones a run revised. That is what makes a deletion a plain set difference +rather than a guess about whether the run's output was authoritative. +""" + +import hashlib +from dataclasses import dataclass +from typing import Iterable, Mapping + +from app.services.knowledge.code_wiki_page_path import collation_key + +# Keys under which the projection records a page's identity and content fingerprint on +# the document it owns. They live in ``source_config`` (an existing JSON column) rather +# than in new columns: the path is derivable from the folder tree and the hash from the +# attachment, so neither is worth a migration. +PAGE_PATH_KEY = "wiki_page_path" +CONTENT_HASH_KEY = "wiki_content_hash" + + +def content_fingerprint(content: str) -> str: + """Return the fingerprint used to decide whether a page needs rewriting.""" + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class PageSource: + """A page as the version holds it.""" + + path: str + title: str + content: str + + @property + def fingerprint(self) -> str: + return content_fingerprint(self.content) + + +@dataclass(frozen=True) +class ProjectedPage: + """A page as the knowledge base currently holds it.""" + + document_id: int + path: str + content_hash: str + + +@dataclass(frozen=True) +class PageUpdate: + """A page whose content changed.""" + + existing: ProjectedPage + source: PageSource + + +@dataclass(frozen=True) +class ProjectionPlan: + """What projecting a version would do, before any of it is done.""" + + adds: tuple[PageSource, ...] = () + updates: tuple[PageUpdate, ...] = () + skips: tuple[str, ...] = () + deletes: tuple[ProjectedPage, ...] = () + + @property + def is_empty(self) -> bool: + """Whether the knowledge base already matches the version.""" + return not (self.adds or self.updates or self.deletes) + + @property + def touched_pages(self) -> int: + return len(self.adds) + len(self.updates) + len(self.deletes) + + def describe(self) -> str: + """One line for logs and for showing a publish decision to a person.""" + return ( + f"{len(self.adds)} added, {len(self.updates)} updated, " + f"{len(self.deletes)} removed, {len(self.skips)} unchanged" + ) + + +def compute_projection_plan( + desired: Iterable[PageSource], + existing: Iterable[ProjectedPage], +) -> ProjectionPlan: + """Compare a version against the knowledge base. + + Args: + desired: Every page in the version being published. + existing: Every generated wiki page currently in the knowledge base. Callers + must scope this to content the projection owns; user content and code + targets appear in neither snapshot and would otherwise be read as orphans + and deleted. + + Returns: + The plan. Pages present on both sides with an equal fingerprint are skipped + entirely — no attachment written, no reindex, no row touched — which is where + nearly all of an incremental run's savings come from. + """ + existing_by_key: Mapping[str, ProjectedPage] = { + collation_key(page.path): page for page in existing + } + + adds: list[PageSource] = [] + updates: list[PageUpdate] = [] + skips: list[str] = [] + seen: set[str] = set() + + for source in desired: + key = collation_key(source.path) + seen.add(key) + current = existing_by_key.get(key) + if current is None: + adds.append(source) + elif current.content_hash == source.fingerprint: + skips.append(source.path) + else: + updates.append(PageUpdate(existing=current, source=source)) + + deletes = [page for key, page in existing_by_key.items() if key not in seen] + + return ProjectionPlan( + adds=tuple(adds), + updates=tuple(updates), + skips=tuple(skips), + deletes=tuple(deletes), + ) diff --git a/backend/app/services/knowledge/code_wiki_prompts.py b/backend/app/services/knowledge/code_wiki_prompts.py new file mode 100644 index 0000000000..c0a773e821 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_prompts.py @@ -0,0 +1,256 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Instructions for the agent that writes a code wiki. + +The previous prompt asked the model to "analyse the repository structure and generate +documentation", which is roughly the whole problem restated. The wiki it produced was +the generic kind that tells a reader nothing they could not get faster by opening the +code — which is what this refactor set out to fix, and no amount of storage work +fixes it. + +So the instructions here are constraints rather than encouragement. Each section +forbids a specific failure that generated wikis actually exhibit: + +- **Grounding** — inventing modules and APIs that sound plausible. +- **Discovery** — reading the whole repository, then running out of budget before + writing anything useful. +- **Planning** — writing pages one at a time so that nothing links to anything. +- **Git** — describing what the code is, when what a reader cannot recover by + reading it is *why* it became that. +- **Selection** — padding pages with restatements of the obvious. + +The write contract is stated as strictly as the content rules, because the projection +depends on it: a page's path is its identity, and a page that arrives without one, or +under a path that shifts between runs, is republished as a delete plus an insert. +""" + +from dataclasses import dataclass +from typing import Optional, Sequence + +# Coverage a wiki is expected to provide. Adapted from the openwiki repository guide, +# whose set has been through real use. +REQUIRED_COVERAGE: tuple[str, ...] = ( + "a concise quickstart: how to run, test and debug this repository", + "an architecture overview: the pieces and how they fit", + "a source map: which directory holds what, and where to start reading", + "key workflows: what happens end to end for the operations that matter", + "domain concepts: the vocabulary this codebase assumes you know", + "operations notes: how it is deployed, configured and recovered", + "testing guidance: what is covered, how to run it, what to add", + "integration points: what it talks to, and what talks to it", +) + +_GROUNDING = """\ +## Grounding + +- Do not invent files, modules, APIs, configuration keys or behaviour. Every claim of + substance must come from a file you have opened or from git history you have read. +- Where the evidence is thin, say so plainly in the page. A sentence admitting that a + workflow is undocumented is worth more than a confident paragraph that is wrong, + because a reader can act on the first and is misled by the second. +- Do not describe intent you cannot support. "This is designed for X" needs a comment, + a commit message or a document behind it.\ +""" + +_DISCOVERY = """\ +## Discovery + +- Do not read every file. Budget spent reading is budget not spent writing. +- Start from the repository tree, dependency manifests, entry points, routing and + schema files, then read representatively within each area you intend to document. +- Prefer search over reading: locate with grep-like tools, then open only the regions + that matter. Never glob the entire tree. +- Skip vendored code, generated output, lockfiles and build artefacts entirely.\ +""" + +_PLANNING = """\ +## Planning + +- Before writing any page, decide the full set of pages and their paths, and record + for each one the evidence it will rest on. +- Decide the links between pages at the same time, as `concept -> relationship -> + concept`. Cross-links designed after the fact do not get written, and a wiki whose + pages do not reference each other is a pile of documents. +- Prefer the smallest set of pages that explains the repository. A first pass that is + accurate and navigable beats a broad one that is thin everywhere.\ +""" + +_GIT_HISTORY = """\ +## Git history + +- Use history to explain *why* the code is the way it is. That is the one thing a + reader cannot recover by reading the code itself, and it is what makes a wiki worth + maintaining at all. +- Look at how the important workflows and entry points arrived at their current shape. + A constraint that was added deliberately is worth documenting; one that is + incidental is worth not documenting.\ +""" + +_SELECTION = """\ +## What to keep out + +- Put every sentence through one test: **would this change what someone does here?** + If not, delete it. +- Record only what cannot be recovered by reading the code. Do not restate the + obvious structure, and do not narrate the happy path of a function that reads + clearly. +- Draw only conclusions the evidence supports. Where there is none, say there is + none. Do not fill the gap.\ +""" + + +def _write_contract(deletion_allowed: bool) -> str: + """Rules for handing pages back, which the projection depends on.""" + deletion = ( + """\ +- To remove a page, declare its removal explicitly by path. Simply not writing a page + does **not** remove it in this mode, because your version begins as a copy of the + published one.\ +""" + if deletion_allowed + else """\ +- Your version begins empty. A page you do not write is not in the wiki, so write + every page the wiki should contain, including the ones you did not change.\ +""" + ) + + return f"""\ +## How to hand back pages + +- Every page has a **path**, which is its identity: `architecture/backend`, + `modules/indexing`. Lowercase, `/`-separated, no file extension. +- **Keep a page's path stable across runs.** The path is what lets an unchanged page + keep its place, its links and its search index. Changing it republishes the page as + a deletion plus an insertion, so reword titles freely and move paths rarely. +- Paths nest at most 4 folders deep, and two paths may not differ only by case. +- Write each page's **complete content** every time. There is no patch format; what + you send replaces the page. +- Write `index` as the overview page: a short list of links, each with one line + saying what the page covers and when to read it, followed by the sections. +{deletion}\ +""" + + +@dataclass(frozen=True) +class WikiRunContext: + """What the agent is being asked to document.""" + + project_name: str + generation_id: int + head_commit: str = "" + language: str = "English" + # Incremental runs only. + previous_commit: str = "" + changed_paths: Sequence[str] = () + existing_pages: Sequence[str] = () + + +def build_full_prompt(context: WikiRunContext) -> str: + """Instructions for rebuilding a wiki from nothing.""" + coverage = "\n".join(f"- {item}" for item in REQUIRED_COVERAGE) + return f"""\ +You are documenting the repository **{context.project_name}** for engineers who have +to work in it, and for agents that will answer questions about it. + +Write for practical navigation. A page earns its place by helping someone find the +part of the system they need and understand what they must know before changing it. + +## Coverage + +Aim to cover, in as few pages as does the job: +{coverage} + +{_GROUNDING} + +{_DISCOVERY} + +{_PLANNING} + +{_GIT_HISTORY} + +{_SELECTION} + +{_write_contract(deletion_allowed=False)} + +## This run + +- Generation: `{context.generation_id}` +- Commit: `{context.head_commit or "current HEAD"}` +- Language: {context.language} +- This is a **full rebuild**: the wiki is being written from scratch.\ +""" + + +def build_incremental_prompt(context: WikiRunContext) -> str: + """Instructions for revising a wiki after a set of changes.""" + changed = ( + "\n".join(f"- {path}" for path in context.changed_paths) + or "- (no file list was available)" + ) + existing = ( + "\n".join(f"- {path}" for path in context.existing_pages) + or "- (the wiki is currently empty)" + ) + return f"""\ +You are updating the wiki for **{context.project_name}** to account for recent +changes. The wiki already exists and is mostly correct; your job is to bring the +parts the changes affect back in line, not to rewrite it. + +## What changed + +Between `{context.previous_commit or "the last documented commit"}` and +`{context.head_commit or "current HEAD"}`: +{changed} + +## Pages that exist now + +{existing} + +## What to do + +- Revise only the pages the changes actually affect. Leave the rest untouched — an + unchanged page costs nothing to keep and is expensive to rewrite. +- Refresh the `index` overview if the set of pages changed. +- If a change makes a page's subject disappear, remove that page explicitly. +- Add pages only where the changes introduced something the wiki has no place for. + +{_GROUNDING} + +{_DISCOVERY} + +{_GIT_HISTORY} + +{_SELECTION} + +{_write_contract(deletion_allowed=True)} + +## This run + +- Generation: `{context.generation_id}` +- Language: {context.language} +- This is an **incremental update**: your version starts as a copy of the published + wiki, so pages you do not touch are carried over unchanged.\ +""" + + +def build_prompt(context: WikiRunContext, *, full: bool) -> str: + """Instructions for a run, chosen by mode.""" + return build_full_prompt(context) if full else build_incremental_prompt(context) + + +def build_diagram_correction(warnings: Sequence[str]) -> Optional[str]: + """Ask the agent to fix diagrams that will not render. + + Sent back as a follow-up rather than treated as a failure: a broken diagram is a + local display fault, and a version is not worth discarding over one. + """ + if not warnings: + return None + listed = "\n".join(f"- {warning}" for warning in warnings) + return ( + "Some Mermaid diagrams in the pages you wrote will not render. Fix them and " + "write those pages again, keeping their paths unchanged:\n" + f"{listed}" + ) diff --git a/backend/app/services/knowledge/code_wiki_publish_gate.py b/backend/app/services/knowledge/code_wiki_publish_gate.py new file mode 100644 index 0000000000..0b05259d3d --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_publish_gate.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Deciding whether a finished version may be published. + +A version being a complete snapshot is guaranteed by construction — seeding puts every +page there — but not by the agent's honesty. An agent that writes four pages and +reports success produces a version holding four pages, and the projection would +faithfully delete the rest. + +So the version is checked before the published pointer moves. What makes this workable +is that the thing being checked is a complete, inspectable, retained snapshot rather +than a knowledge base already half-rewritten: a rejected version stays in the store +with its verdict attached, and the published one is still whatever it was. + +Removal is the substantive check. It is what makes agent-declared deletion safe to +allow at all: deletions are permitted, mass deletion is not. + +It is measured against the *set of published paths the version no longer contains*, +not against the two page counts. Counting would miss the case that matters most: a +version holding the same number of pages under mostly different paths is a mass +deletion plus a mass insertion, every affected page loses its document id, and every +stored citation and index entry pointing at it breaks. The count says nothing changed. +""" + +import logging +from dataclasses import dataclass, field +from typing import Optional, Sequence + +from app.services.knowledge.code_wiki_page_path import collation_key +from app.services.knowledge.code_wiki_projection_plan import PageSource +from app.services.knowledge.mermaid_check import check_mermaid_blocks, describe_warnings + +logger = logging.getLogger(__name__) + +# Key under which the verdict is recorded on the generation. +PUBLISH_GATE_EXT_KEY = "publishGate" + + +@dataclass(frozen=True) +class PublishPolicy: + """Limits a version must respect to be published.""" + + # Share of the previously published pages that may disappear in one publish. + # Deleting pages is legitimate; deleting most of them is a malfunction, and the + # difference cannot be told apart by inspecting any single page. + max_removed_share: float = 0.5 + # A published wiki always has at least an overview page. A version with none is + # not an empty repository, it is a run that produced nothing. + min_pages: int = 1 + # Mermaid problems are reported, never blocking: a diagram that will not render is + # a local display fault, and holding a whole version for one is a poor trade. + block_on_mermaid: bool = False + + +DEFAULT_POLICY = PublishPolicy() + + +@dataclass(frozen=True) +class GateVerdict: + """Whether a version may be published, and what was noticed.""" + + passed: bool + reason: str = "" + warnings: tuple[str, ...] = field(default=()) + + def to_ext(self, checked_at: str) -> dict: + """Render for storage on the generation. + + Recorded to explain *why* a version is not live. It never decides that — + the published pointer alone does — because a second source of truth for + "which version is published" is a second thing that can be wrong. + """ + return { + "result": "passed" if self.passed else "rejected", + "reason": self.reason, + "warnings": list(self.warnings), + "checkedAt": checked_at, + } + + +def evaluate_publish_gate( + pages: Sequence[PageSource], + *, + published_paths: Sequence[str], + policy: Optional[PublishPolicy] = None, +) -> GateVerdict: + """Judge a finished version against the currently published one. + + Args: + pages: Every page in the version being considered. + published_paths: Paths currently published, empty when nothing has been. + Paths rather than a count: publishing a same-sized version under + different paths deletes every page it renamed, and a count cannot see it. + policy: Limits to apply. + + Returns: + The verdict. Warnings never cause a rejection on their own. + """ + policy = policy or DEFAULT_POLICY + warnings = _collect_warnings(pages) + + if len(pages) < policy.min_pages: + return GateVerdict( + passed=False, + reason=( + f"version has {len(pages)} pages, fewer than the minimum " + f"{policy.min_pages}; the run produced nothing usable" + ), + warnings=warnings, + ) + + if published_paths: + kept = {collation_key(page.path) for page in pages} + removed = [path for path in published_paths if collation_key(path) not in kept] + removed_share = len(removed) / len(published_paths) + if removed_share > policy.max_removed_share: + return GateVerdict( + passed=False, + reason=( + f"version removes {removed_share:.0%} of the published pages " + f"({len(removed)} of {len(published_paths)}), over the " + f"{policy.max_removed_share:.0%} limit" + ), + warnings=warnings, + ) + + if warnings and policy.block_on_mermaid: + return GateVerdict( + passed=False, + reason="diagram problems were found and the policy blocks on them", + warnings=warnings, + ) + + return GateVerdict(passed=True, warnings=warnings) + + +def _collect_warnings(pages: Sequence[PageSource]) -> tuple[str, ...]: + """Gather non-blocking content warnings across the version.""" + collected: list[str] = [] + for page in pages: + mermaid_warnings = check_mermaid_blocks(page.content) + if mermaid_warnings: + collected.append(f"{page.path}: {describe_warnings(mermaid_warnings)}") + return tuple(collected) diff --git a/backend/app/services/knowledge/code_wiki_publisher.py b/backend/app/services/knowledge/code_wiki_publisher.py new file mode 100644 index 0000000000..a06a29575e --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_publisher.py @@ -0,0 +1,333 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Publishing a code wiki version. + +This is the one place that moves ``spec.publishedGenerationId``. Everything else about +"which version is live" is derived from that pointer; nothing infers it from the newest +completed generation, because a generation can finish and still be rejected. + +The sequence is deliberately linear: + +1. read the version and the pages the knowledge base currently holds, +2. judge the version against the published one, +3. plan the difference, +4. apply the rows, advance the pointer, and commit both together, +5. clean up what could not be part of that transaction. + +The pointer moves *inside* the transaction that writes the rows, not after it. They +describe one fact — which version the knowledge base holds — and committing them +separately would leave a window where the pointer and the content disagree, in a +process that can die at any point. Step 5 is outside because the vector store and the +indexing queue cannot join a database transaction; what it cannot finish is parked and +swept later. + +A rollback is the same operation with an older version as its input, which is why no +separate machinery exists for it. +""" + +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Optional, Sequence + +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import KnowledgeDocument +from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.services.knowledge.code_wiki_projection import ( + PENDING_INDEX_CLEANUP_KEY, + ProjectionSideEffects, + apply_projection_plan, + finish_projection, +) +from app.services.knowledge.code_wiki_projection_plan import ( + CONTENT_HASH_KEY, + PAGE_PATH_KEY, + PageSource, + ProjectedPage, + ProjectionPlan, + compute_projection_plan, +) +from app.services.knowledge.code_wiki_publish_gate import ( + PUBLISH_GATE_EXT_KEY, + GateVerdict, + PublishPolicy, + evaluate_publish_gate, +) +from app.services.knowledge.code_wiki_version_store import page_path_of +from app.services.knowledge.content_scope import generated_wiki_pages + +logger = logging.getLogger(__name__) + +PUBLISHED_GENERATION_KEY = "publishedGenerationId" + + +@dataclass(frozen=True) +class PublishResult: + """What a publish attempt did.""" + + published: bool + verdict: GateVerdict + plan: Optional[ProjectionPlan] = None + reason: str = "" + + +def read_version_pages(db: Session, generation_id: int) -> tuple[PageSource, ...]: + """Read a version as the projection wants to see it. + + Entries without a page path predate page identity and are skipped rather than + guessed at: projecting one would create a document the next run could not match, + and so would delete and recreate on every publish. + """ + pages: list[PageSource] = [] + for entry in ( + db.query(WikiContent).filter(WikiContent.generation_id == generation_id).all() + ): + path = page_path_of(entry) + if not path: + logger.warning( + "[code_wiki] version entry %s has no page path; not projected", + entry.id, + ) + continue + pages.append(PageSource(path=path, title=entry.title, content=entry.content)) + return tuple(pages) + + +def read_projected_pages(db: Session, kind_id: int) -> tuple[ProjectedPage, ...]: + """Read the generated wiki pages the knowledge base currently holds. + + Scoped through ``generated_wiki_pages`` rather than by an inline filter: user + content and code targets must never enter this comparison, because anything + missing from the version is treated as an orphan and deleted. + """ + query = generated_wiki_pages( + db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == kind_id) + ) + + pages: list[ProjectedPage] = [] + for document in query.all(): + config = document.source_config or {} + path = config.get(PAGE_PATH_KEY) + if not path: + # A generated page the projection did not create. Leaving it out means it + # is never deleted, which is the safe direction to be wrong in. + logger.warning( + "[code_wiki] generated document %s has no page path; left alone", + document.id, + ) + continue + pages.append( + ProjectedPage( + document_id=document.id, + path=path, + content_hash=config.get(CONTENT_HASH_KEY, ""), + ) + ) + return tuple(pages) + + +def published_generation_id(knowledge_base: Kind) -> int: + spec = (knowledge_base.json or {}).get("spec", {}) + try: + return int(spec.get(PUBLISHED_GENERATION_KEY, 0) or 0) + except (TypeError, ValueError): + return 0 + + +def _update_spec(knowledge_base: Kind, **values) -> None: + payload = dict(knowledge_base.json or {}) + spec = dict(payload.get("spec", {})) + spec.update(values) + payload["spec"] = spec + knowledge_base.json = payload + + +def _record_verdict(generation: WikiGeneration, verdict: GateVerdict) -> None: + ext = dict(generation.ext or {}) + ext[PUBLISH_GATE_EXT_KEY] = verdict.to_ext( + datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + ) + generation.ext = ext + + +def publish_generation( + db: Session, + *, + knowledge_base: Kind, + generation: WikiGeneration, + user_id: int, + effects: ProjectionSideEffects, + policy: Optional[PublishPolicy] = None, + require_completed: bool = True, +) -> PublishResult: + """Project a version into the knowledge base and make it the published one. + + Args: + db: Session. Committed here once the rows are in place. + knowledge_base: The code wiki's Kind. + generation: Version to publish. + user_id: Owner recorded on documents the projection creates. + effects: Side effects outside the transaction. + policy: Publish gate limits. + require_completed: Whether the generation must have finished successfully. + Set false to re-project an older version, which is what a rollback is. + + Returns: + Whether the version was published, with the gate's verdict either way. + """ + if require_completed and generation.status != WikiGenerationStatus.COMPLETED: + verdict = GateVerdict( + passed=False, + reason=f"generation is {generation.status}, not completed", + ) + _record_verdict(generation, verdict) + db.commit() + return PublishResult(published=False, verdict=verdict, reason=verdict.reason) + + desired = read_version_pages(db, generation.id) + existing = read_projected_pages(db, knowledge_base.id) + + # The gate is asked what readers would lose, so it is given what readers can see + # now — the projected pages — rather than the published version's own page list. + # Those agree unless the knowledge base has drifted, and where they disagree the + # projection is the one that decides what actually gets deleted. + verdict = evaluate_publish_gate( + desired, + published_paths=[page.path for page in existing], + policy=policy, + ) + if not verdict.passed: + logger.warning( + "[code_wiki] generation %s rejected for kb %s: %s", + generation.id, + knowledge_base.id, + verdict.reason, + ) + _record_verdict(generation, verdict) + db.commit() + return PublishResult(published=False, verdict=verdict, reason=verdict.reason) + + plan = compute_projection_plan(desired, existing) + logger.info( + "[code_wiki] publishing generation %s into kb %s: %s", + generation.id, + knowledge_base.id, + plan.describe(), + ) + + superseded = _attachments_being_replaced(db, plan) + outcome = apply_projection_plan( + db, + kind_id=knowledge_base.id, + user_id=user_id, + plan=plan, + effects=effects, + ) + + _record_verdict(generation, verdict) + _update_spec(knowledge_base, **{PUBLISHED_GENERATION_KEY: generation.id}) + db.commit() + + # Past this point the version is live. Nothing below may raise: the pages are + # already correct, and failing now would only cause the run to be repeated. + unfinished = finish_projection( + outcome, superseded_attachment_ids=superseded, effects=effects + ) + if unfinished: + _park_unfinished_cleanup(db, knowledge_base, unfinished) + db.commit() + + return PublishResult(published=True, verdict=verdict, plan=plan) + + +def _attachments_being_replaced(db: Session, plan: ProjectionPlan) -> tuple[int, ...]: + """Ids of attachments the plan supersedes, read before the rows change. + + Includes each page's converted attachment: overwriting the source without it + leaves a stale conversion that the document still points at. + """ + doomed: list[int] = [] + touched = [update.existing.document_id for update in plan.updates] + touched += [page.document_id for page in plan.deletes] + + for document_id in touched: + document = db.get(KnowledgeDocument, document_id) + if document is None: + continue + if document.attachment_id: + doomed.append(document.attachment_id) + converted = document.converted_attachment_id + if converted: + doomed.append(converted) + return tuple(doomed) + + +def _park_unfinished_cleanup( + db: Session, knowledge_base: Kind, doc_refs: Sequence[str] +) -> None: + """Record index deletions still owed, so a sweep can finish them. + + The vector store is external and cannot be rolled into the transaction, so a + failure has to outlive the process rather than disappear into a log line: a + document row that is gone while its chunks remain leaves retrieval answering + from a page that no longer exists. + """ + # Stored as strings throughout. The retry reads them back and writes them out + # again, so mixing in an int here would produce an entry that no longer matches + # the membership test below and the ref would be parked twice. + pending = [ + str(ref) + for ref in (knowledge_base.json or {}) + .get("spec", {}) + .get(PENDING_INDEX_CLEANUP_KEY) + or [] + ] + pending.extend(str(ref) for ref in doc_refs if str(ref) not in pending) + _update_spec(knowledge_base, **{PENDING_INDEX_CLEANUP_KEY: pending}) + logger.warning( + "[code_wiki] kb %s has %s index deletions awaiting retry", + knowledge_base.id, + len(pending), + ) + + +def retry_pending_index_cleanup( + db: Session, *, knowledge_base: Kind, effects: ProjectionSideEffects +) -> tuple[str, ...]: + """Attempt the index deletions a previous publish could not finish. + + Returns: + The refs still outstanding. + """ + spec = (knowledge_base.json or {}).get("spec", {}) + pending = [str(ref) for ref in (spec.get(PENDING_INDEX_CLEANUP_KEY) or [])] + if not pending: + return () + + still_pending: list[str] = [] + for doc_ref in pending: + if not doc_ref.isdigit(): + # Dropped rather than retried. A ref that is not a document id can never + # be deleted, so keeping it means every sweep from here on fails on it + # and the list never drains. + logger.error( + "[code_wiki] kb %s parked an unusable index ref %r; dropping it", + knowledge_base.id, + doc_ref, + ) + continue + try: + effects.delete_rag_document(int(doc_ref)) + except Exception as exc: + logger.warning( + "[code_wiki] index cleanup still failing for %s: %s", doc_ref, exc + ) + still_pending.append(doc_ref) + + _update_spec(knowledge_base, **{PENDING_INDEX_CLEANUP_KEY: still_pending}) + db.commit() + return tuple(still_pending) diff --git a/backend/app/services/knowledge/code_wiki_repo_state.py b/backend/app/services/knowledge/code_wiki_repo_state.py new file mode 100644 index 0000000000..8f7ff6f4f7 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_repo_state.py @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Reading what a repository is at right now, so a run can decide whether to happen. + +Without this, ``decide_run_mode`` never sees a HEAD commit, and two of its three +answers become unreachable: it cannot recognise an unchanged repository, and with no +diff it must assume the worst and rebuild. A weekly schedule over a quiet repository +then pays a full pass through the model every week to conclude nothing happened. + +The mode has to be chosen *before* the agent starts — it decides whether the version +is seeded and which instructions are sent — so the agent's own clone cannot answer +this. It has to be read from the provider first. + +Everything here degrades to "unknown", never to a guess. A provider that is down, an +older self-hosted instance without a compare endpoint, a diff too large to return +whole: each of those yields ``None``, which ``decide_run_mode`` reads as *the extent +of the change is unknown* and answers with a full rebuild. That is the expensive +answer, and it is the only safe one — a partial diff mistaken for a complete one +picks an incremental run for a change that reshaped the repository. +""" + +import logging +from dataclasses import dataclass +from typing import Optional + +from sqlalchemy.orm import Session + +from app.services.git_skill.utils import get_user_git_info +from app.services.knowledge.code_wiki_run_mode import ChangedPath +from app.services.knowledge.code_wiki_source import SourceRepository, provider_for + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class RepositoryState: + """What the repository looks like now, as far as could be determined.""" + + head_commit: str = "" + branch: str = "" + # ``None`` means the diff is unknown, which is different from "nothing changed". + changed_paths: Optional[tuple[ChangedPath, ...]] = None + + +def read_repository_state( + db: Session, + *, + user_id: int, + source: SourceRepository, + since_commit: str = "", +) -> RepositoryState: + """Read the default branch's HEAD, and the diff since ``since_commit``. + + Args: + db: Session, for looking up the caller's credentials. + user_id: Whose token to read the repository with. + source: The repository the wiki is bound to. + since_commit: Commit the published wiki documents. Empty on a first run, in + which case no diff is asked for — there is nothing to compare against. + + Returns: + What could be read. Every field is best-effort: a failure anywhere leaves the + state less certain, never wrong. + """ + provider = provider_for(source.source_type) + if provider is None: + logger.info( + "[code_wiki] no provider for '%s'; repository state unknown", + source.source_type, + ) + return RepositoryState() + + git_info = get_user_git_info(user_id=user_id, domain=source.source_domain, db=db) + token = (git_info or {}).get("token") + if not token: + logger.info( + "[code_wiki] user %s has no credentials for %s; repository state unknown", + user_id, + source.source_domain, + ) + return RepositoryState() + + head = _read_head(provider, token, source) + if not head.head_commit or not since_commit: + return head + + return RepositoryState( + head_commit=head.head_commit, + branch=head.branch, + changed_paths=_read_changed_paths( + provider, token, source, since_commit, head.head_commit + ), + ) + + +def _read_head(provider, token: str, source: SourceRepository) -> RepositoryState: + """The default branch and its commit, or an empty state when it cannot be read.""" + try: + result = provider.get_default_branch_head( + token=token, + git_domain=source.source_domain, + repo_name=source.project_name, + ) + except Exception as exc: + logger.warning( + "[code_wiki] could not read HEAD of %s: %s", source.project_name, exc + ) + return RepositoryState() + + return RepositoryState( + head_commit=str(result.get("commit", "") or ""), + branch=str(result.get("branch", "") or ""), + ) + + +def _read_changed_paths( + provider, token: str, source: SourceRepository, base: str, head: str +) -> Optional[tuple[ChangedPath, ...]]: + """The diff between two commits, or ``None`` when it cannot be trusted.""" + if base == head: + # Nothing to compare. Returning an empty diff rather than ``None`` matters: + # it is what lets the run be skipped instead of rebuilt. + return () + + try: + entries = provider.get_changed_files( + token=token, + git_domain=source.source_domain, + repo_name=source.project_name, + base=base, + head=head, + ) + except Exception as exc: + logger.warning( + "[code_wiki] could not diff %s..%s in %s: %s", + base, + head, + source.project_name, + exc, + ) + return None + + if entries is None: + return None + return tuple( + ChangedPath(path=entry["path"], status=entry.get("status", "M")) + for entry in entries + if entry.get("path") + ) diff --git a/backend/app/services/knowledge/code_wiki_run_mode.py b/backend/app/services/knowledge/code_wiki_run_mode.py new file mode 100644 index 0000000000..37c088a31d --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_run_mode.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Deciding how much of a code wiki a run should rebuild. + +The mode decides **how a version is built**, and nothing else: + +- ``SKIP`` — the repository has not moved, so no version is created. +- ``INCREMENTAL`` — the default. The version is seeded from the published one and the + agent revises only the pages the changes affect. +- ``FULL`` — the version starts empty and the agent writes every page. + +Both modes end with a complete snapshot, so **publishing does not depend on the mode**. +An earlier design let the mode decide whether the run's reported page set could be +used to delete pages, which needed a matching set of heuristics to keep an incremental +run from deleting every page it had not touched. Seeding removes that question: the +projection always compares complete snapshots, so orphans are a plain set difference. + +A full rebuild is still forced periodically. An incremental run reworking the page +layout can leave pages nothing points at any more, and neither the agent nor the diff +has the whole picture; starting from an empty version is what clears them. +""" + +from dataclasses import dataclass, field +from enum import Enum +from fnmatch import fnmatchcase +from typing import Optional, Sequence + + +class RunMode(str, Enum): + """How much of the wiki a single run rebuilds.""" + + SKIP = "skip" + INCREMENTAL = "incremental" + FULL = "full" + + +@dataclass(frozen=True) +class ChangedPath: + """One entry from the diff between the last documented commit and HEAD.""" + + path: str + # Git name-status letter: "A" added, "M" modified, "D" deleted, "R" renamed. + status: str + + @property + def is_structural_move(self) -> bool: + """Whether this entry adds or removes a file rather than editing one.""" + return self.status.upper().startswith(("A", "D", "R")) + + +# Files that describe how the project is built or what it depends on. A change here +# usually reshapes the architecture the wiki describes, so it earns a full rebuild +# even when only one file moved. +DEFAULT_MANIFEST_PATTERNS: tuple[str, ...] = ( + "package.json", + "pnpm-workspace.yaml", + "pyproject.toml", + "requirements*.txt", + "go.mod", + "Cargo.toml", + "pom.xml", + "build.gradle*", + "Makefile", + "Dockerfile", + "docker-compose*.yml", + "*/package.json", + "*/pyproject.toml", + "*/go.mod", + "*/Cargo.toml", +) + + +@dataclass(frozen=True) +class RunModePolicy: + """Thresholds that promote an incremental run to a full rebuild.""" + + max_changed_files: int = 50 + max_structural_moves: int = 15 + max_changed_ratio: float = 0.25 + max_incrementals_since_full: int = 10 + max_days_since_full: float = 30.0 + manifest_patterns: tuple[str, ...] = field(default=DEFAULT_MANIFEST_PATTERNS) + + +DEFAULT_POLICY = RunModePolicy() + + +@dataclass(frozen=True) +class RunModeDecision: + """The chosen mode and why, recorded on the run for troubleshooting.""" + + mode: RunMode + reason: str + + @property + def seeds_from_published(self) -> bool: + """Whether the new version starts as a copy of the published one. + + Compared by value, not identity: ``RunMode`` is a ``str`` enum so that a mode + survives a round trip through a task payload, and ``"incremental" is + RunMode.INCREMENTAL`` is false. An identity test would quietly stop seeding, + and an unseeded incremental version is a partial snapshot — the projection + would read every page the run did not touch as an orphan and delete it. + """ + return RunMode(self.mode) == RunMode.INCREMENTAL + + +def _matches_manifest(path: str, patterns: Sequence[str]) -> bool: + """Match case-sensitively, so behaviour does not depend on the host platform.""" + return any(fnmatchcase(path, pattern) for pattern in patterns) + + +def decide_run_mode( + *, + head_commit: str, + last_commit: Optional[str] = None, + changed_paths: Optional[Sequence[ChangedPath]] = None, + incrementals_since_full: int = 0, + days_since_full: Optional[float] = None, + previous_top_level_dirs: Optional[frozenset[str]] = None, + current_top_level_dirs: Optional[frozenset[str]] = None, + policy: RunModePolicy = DEFAULT_POLICY, + total_source_files: Optional[int] = None, +) -> RunModeDecision: + """Choose the mode for one run. + + Args: + head_commit: Commit the repository is at now. + last_commit: Commit the wiki was last generated from; absent on a first run. + changed_paths: Diff between ``last_commit`` and ``head_commit``. When absent + the extent of the change is unknown, so a full rebuild is chosen. + incrementals_since_full: Incremental runs completed since the last full one. + days_since_full: Days since the last full run, if one has happened. + previous_top_level_dirs: Top-level directories at ``last_commit``. Compared + with ``current_top_level_dirs`` to spot modules appearing or disappearing; + skipped when either side is unknown. + current_top_level_dirs: Top-level directories at ``head_commit``. + policy: Thresholds to apply. + total_source_files: Files under consideration at ``head_commit``, used for the + proportional threshold; skipped when unknown. + """ + if not last_commit: + return RunModeDecision(RunMode.FULL, "first run for this repository") + + if last_commit == head_commit: + return RunModeDecision(RunMode.SKIP, "repository unchanged since last run") + + if changed_paths is None: + return RunModeDecision( + RunMode.FULL, "extent of changes unknown, rebuilding to stay correct" + ) + + # Checked before the empty-diff shortcut: the diff may be filtered to documented + # file types while the directory sets come from the whole tree, so removing a + # module of, say, protos or assets can show up here and nowhere else. Skipping on + # an empty diff first would orphan that module's pages with nothing left to catch + # it, since the periodic rules also sit behind the skip. + if previous_top_level_dirs is not None and current_top_level_dirs is not None: + appeared = current_top_level_dirs - previous_top_level_dirs + disappeared = previous_top_level_dirs - current_top_level_dirs + if appeared or disappeared: + moved = ", ".join(sorted(appeared | disappeared)) + return RunModeDecision(RunMode.FULL, f"top-level modules changed: {moved}") + + if not changed_paths: + # The commit moved but nothing we document did — treat as unchanged rather + # than paying for a rebuild. + return RunModeDecision(RunMode.SKIP, "no documented files changed") + + manifests = [ + change.path + for change in changed_paths + if _matches_manifest(change.path, policy.manifest_patterns) + ] + if manifests: + return RunModeDecision( + RunMode.FULL, f"build or dependency manifest changed: {manifests[0]}" + ) + + # Files appearing, disappearing or moving reshape what the wiki documents far more + # than edits do, so a burst of them earns a rebuild sooner than plain edits. + structural = [change for change in changed_paths if change.is_structural_move] + if len(structural) > policy.max_structural_moves: + return RunModeDecision( + RunMode.FULL, + f"{len(structural)} files added, removed or renamed, over the limit of " + f"{policy.max_structural_moves}", + ) + + if len(changed_paths) > policy.max_changed_files: + return RunModeDecision( + RunMode.FULL, + f"{len(changed_paths)} files changed, over the limit of " + f"{policy.max_changed_files}", + ) + + if total_source_files and total_source_files > 0: + ratio = len(changed_paths) / total_source_files + if ratio > policy.max_changed_ratio: + return RunModeDecision( + RunMode.FULL, + f"{ratio:.0%} of files changed, over the limit of " + f"{policy.max_changed_ratio:.0%}", + ) + + # Periodic rebuild: incremental runs cannot see pages orphaned by restructuring + # or by the agent relaying out the wiki, so those only get cleaned up here. + if incrementals_since_full >= policy.max_incrementals_since_full: + return RunModeDecision( + RunMode.FULL, + f"{incrementals_since_full} incremental runs since the last full rebuild", + ) + + if days_since_full is not None and days_since_full >= policy.max_days_since_full: + return RunModeDecision( + RunMode.FULL, + f"{days_since_full:.0f} days since the last full rebuild", + ) + + return RunModeDecision( + RunMode.INCREMENTAL, f"{len(changed_paths)} files changed since last run" + ) diff --git a/backend/app/services/knowledge/code_wiki_runner.py b/backend/app/services/knowledge/code_wiki_runner.py new file mode 100644 index 0000000000..ae5b3d37c5 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_runner.py @@ -0,0 +1,474 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Driving a code wiki run from a knowledge base to an agent and back. + +Everything either side of this module already exists and is tested on its own: the +version store decides what a run may do, the prompts say what the agent must produce, +and the publisher turns a finished version into knowledge base content. What was +missing is the part that actually starts one — resolving the repository, choosing the +mode, writing the instructions and handing them to a Task. + +Two ordering decisions here are deliberate: + +**The generation is committed before the task is created.** A crash between the two +then leaves a run with no task, which the staleness sweep reclaims after six hours. The +alternative ordering — task first — leaves a task with no run to report into, which +nothing reclaims and which would write its pages into a version that does not exist. + +**A task that cannot be created fails the run immediately** rather than leaving it +RUNNING for the sweep to find. A wiki that refuses to regenerate for six hours because +of a misconfigured team is a much worse failure than one that says so at once. +""" + +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Optional, Sequence + +from sqlalchemy import func +from sqlalchemy.orm import Session + +from app.core.wiki_config import wiki_settings +from app.models.kind import Kind +from app.models.user import User +from app.models.wiki import WikiGeneration, WikiGenerationStatus +from app.schemas.knowledge import KnowledgeBaseType +from app.schemas.task import TaskCreate +from app.services.knowledge.code_wiki_generation import ( + SOURCE_COMMIT_KEY, + finish_generation, + published_commit, + start_generation, +) +from app.services.knowledge.code_wiki_prompts import WikiRunContext, build_prompt +from app.services.knowledge.code_wiki_publisher import PublishResult, read_version_pages +from app.services.knowledge.code_wiki_repo_state import read_repository_state +from app.services.knowledge.code_wiki_run_mode import ChangedPath, RunMode +from app.services.knowledge.code_wiki_side_effects import build_projection_side_effects +from app.services.knowledge.code_wiki_source import SourceRepository + +logger = logging.getLogger(__name__) + +# What the model is told to write in, keyed by the configured language code. The +# prompt states this in prose, so it needs a language name rather than a code. +_LANGUAGE_NAMES = {"en": "English", "zh": "Chinese (Simplified)"} + + +class CodeWikiRunError(RuntimeError): + """Raised when a run cannot be set up. The message is shown to the caller.""" + + +@dataclass(frozen=True) +class StartedRun: + """The outcome of asking a code wiki to regenerate.""" + + generation: Optional[WikiGeneration] + reason: str + mode: str = "" + task_id: int = 0 + + @property + def started(self) -> bool: + return self.generation is not None + + +def source_of(knowledge_base: Kind) -> SourceRepository: + """Read the repository a code wiki is bound to. + + Raises: + CodeWikiRunError: If this knowledge base is not a code wiki, or is one with no + usable repository recorded — which would otherwise surface much later as a + task cloning an empty URL. + """ + spec = (knowledge_base.json or {}).get("spec", {}) + if spec.get("kbType") != KnowledgeBaseType.CODE_WIKI.value: + raise CodeWikiRunError("This knowledge base is not a code wiki") + + source = SourceRepository.from_spec(spec.get("source")) + if source is None or not source.source_url or not source.project_name: + raise CodeWikiRunError( + "This code wiki has no source repository recorded and cannot be generated" + ) + return source + + +def start_run( + db: Session, + *, + knowledge_base: Kind, + user: User, + head_commit: str = "", + changed_paths: Optional[Sequence[ChangedPath]] = None, + total_source_files: Optional[int] = None, +) -> StartedRun: + """Start a run for ``knowledge_base`` and hand its instructions to a task. + + Args: + db: Session. + knowledge_base: The code wiki to regenerate. + user: Identity the run is attributed to. + head_commit: Commit the repository is at now. When empty it is read from the + provider, and only if that also fails is the extent of the change taken + as unknown — which costs a full rebuild. + changed_paths: Diff since the published commit. ``None`` asks for it to be + read from the provider alongside the commit. + total_source_files: Repository size, used by the change-ratio threshold. + + Returns: + The started run, or a reason why none was needed. + + Raises: + CodeWikiRunError: If the wiki cannot be generated, or a run is already going. + GenerationInFlight: Propagated from the version store. + """ + source = source_of(knowledge_base) + team, task_user = _resolve_execution_context(db, user) + + previous_commit = published_commit(db, knowledge_base) + if not head_commit: + # Read as the user that will clone the repository, not the one who asked. On + # a schedule there is no asker, and a token that cannot read the repository + # would answer for a run that is about to fail at checkout anyway. + state = read_repository_state( + db, + user_id=task_user.id, + source=source, + since_commit=previous_commit, + ) + head_commit = state.head_commit + if changed_paths is None: + changed_paths = state.changed_paths + + started = start_generation( + db, + knowledge_base=knowledge_base, + user=user, + head_commit=head_commit, + changed_paths=changed_paths, + total_source_files=total_source_files, + team_id=team.id, + ) + if not started.started: + return StartedRun( + generation=None, reason=started.decision.reason, mode=started.decision.mode + ) + + generation = started.generation + full = RunMode(started.decision.mode) is RunMode.FULL + prompt = build_prompt( + WikiRunContext( + project_name=source.project_name, + generation_id=generation.id, + head_commit=head_commit, + language=_LANGUAGE_NAMES.get( + (wiki_settings.DEFAULT_LANGUAGE or "en").lower(), "English" + ), + previous_commit=previous_commit, + changed_paths=[change.path for change in changed_paths or ()], + existing_pages=[ + page.path for page in read_version_pages(db, generation.id) + ], + ), + full=full, + ) + + # Committed before the task exists, so that a task always has a version to report + # into. See the module docstring for why the reverse ordering is worse. + db.commit() + + task_id = _create_task( + db, + source=source, + team_id=team.id, + task_user=task_user, + prompt=prompt, + generation=generation, + ) + + generation.task_id = task_id + db.commit() + logger.info( + "[code_wiki] generation %s for kb %s running under task %s", + generation.id, + knowledge_base.id, + task_id, + ) + return StartedRun( + generation=generation, + reason=started.decision.reason, + mode=started.decision.mode, + task_id=task_id, + ) + + +def finish_run( + db: Session, + *, + generation: WikiGeneration, + succeeded: bool, + error_message: str = "", + head_commit: str = "", +) -> Optional[PublishResult]: + """Conclude a run the agent has reported on, and publish it if it succeeded. + + Args: + db: Session. + generation: The run being concluded. + succeeded: Whether the agent reported success. + error_message: Why it failed, when it did. + head_commit: Commit the agent actually documented. Recorded over whatever the + run started with, because the agent read the working tree and the trigger + only knew what it was told — and this value is what the next run's mode + decision compares against. + + Returns: + The publish outcome, or ``None`` when the run failed or was not publishable. + """ + knowledge_base = _knowledge_base_of(db, generation) + if knowledge_base is None: + raise CodeWikiRunError( + f"generation {generation.id} has no knowledge base to publish into" + ) + + user = db.get(User, generation.user_id) + if user is None: + raise CodeWikiRunError( + f"generation {generation.id} has no user to publish as " + f"(user {generation.user_id})" + ) + + if head_commit: + snapshot = dict(generation.source_snapshot or {}) + snapshot[SOURCE_COMMIT_KEY] = head_commit + generation.source_snapshot = snapshot + db.flush() + + return finish_generation( + db, + knowledge_base=knowledge_base, + generation=generation, + user=user, + effects=build_projection_side_effects( + db, knowledge_base=knowledge_base, user=user + ), + succeeded=succeeded, + error_message=error_message, + ) + + +def sweep_pending_index_cleanup(db: Session) -> dict: + """Finish the index deletions that publishes could not. + + Deleting a page removes its row inside the publish transaction and its chunks + outside one, because the vector store cannot join a database transaction. When + the second half fails the publish still succeeds — refusing a good version over + an external hiccup would be worse — and the owed deletions are parked on the + knowledge base for this sweep to finish. + + Without it they are never retried. The orphaned chunks outlive the page, and + retrieval goes on answering from it while citing a document id that no longer + resolves. Nothing else clears them: that would take deleting the same page a + second time, and it is already gone. + + Refs that keep failing stay parked and are tried again on the next sweep. Giving + up would make the orphan permanent, which is the state this exists to end. + """ + from app.services.knowledge.code_wiki_publisher import retry_pending_index_cleanup + + swept = 0 + drained = 0 + outstanding = 0 + for knowledge_base in _knowledge_bases_owing_cleanup(db): + owner = db.get(User, knowledge_base.user_id) + if owner is None: + logger.warning( + "[code_wiki] kb %s owes index cleanup but its owner %s is gone", + knowledge_base.id, + knowledge_base.user_id, + ) + continue + + swept += 1 + try: + still_pending = retry_pending_index_cleanup( + db, + knowledge_base=knowledge_base, + effects=build_projection_side_effects( + db, knowledge_base=knowledge_base, user=owner + ), + ) + except Exception as exc: + # One knowledge base's failure must not stop the others: they are + # independent, and the whole point is to keep draining. + db.rollback() + logger.error( + "[code_wiki] index cleanup sweep failed for kb %s: %s", + knowledge_base.id, + exc, + ) + continue + + outstanding += len(still_pending) + if not still_pending: + drained += 1 + + if swept: + logger.info( + "[code_wiki] index cleanup sweep: %s knowledge base(s), %s drained, " + "%s deletion(s) still owed", + swept, + drained, + outstanding, + ) + return {"swept": swept, "drained": drained, "outstanding": outstanding} + + +def _knowledge_bases_owing_cleanup(db: Session) -> list[Kind]: + """Knowledge bases with parked index deletions. + + The JSON filter narrows the scan to knowledge bases that have ever parked + anything; emptiness is then checked in Python, because a drained list is stored + as ``[]`` and comparing against that in SQL renders differently per database. + """ + from app.services.knowledge.code_wiki_projection import PENDING_INDEX_CLEANUP_KEY + + candidates = ( + db.query(Kind) + .filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, # noqa: E712 + func.json_extract(Kind.json, f"$.spec.{PENDING_INDEX_CLEANUP_KEY}").isnot( + None + ), + ) + .all() + ) + return [ + candidate + for candidate in candidates + if (candidate.json or {}).get("spec", {}).get(PENDING_INDEX_CLEANUP_KEY) + ] + + +def is_code_wiki_generation(db: Session, generation: WikiGeneration) -> bool: + """Whether this run belongs to a code wiki rather than the legacy wiki path.""" + return _knowledge_base_of(db, generation) is not None + + +def _knowledge_base_of(db: Session, generation: WikiGeneration) -> Optional[Kind]: + """The code wiki a run belongs to, or ``None`` for a legacy generation.""" + if not generation.kind_id: + return None + knowledge_base = db.get(Kind, generation.kind_id) + if knowledge_base is None or knowledge_base.kind != "KnowledgeBase": + return None + return knowledge_base + + +def _resolve_execution_context(db: Session, user: User) -> tuple[Kind, User]: + """Find the team that runs code wikis, and the user it runs as. + + Both come from configuration rather than from the request. A wiki is generated by + the system on the repository's behalf, so letting the caller choose the team would + let them choose the prompt and the tools the agent gets. + """ + from app.services.adapters.team_kinds import team_kinds_service + from app.services.user import user_service + + task_user = user + if wiki_settings.DEFAULT_USER_ID > 0: + configured = user_service.get_user_by_id(db, wiki_settings.DEFAULT_USER_ID) + if configured is None: + raise CodeWikiRunError( + f"Configured wiki user {wiki_settings.DEFAULT_USER_ID} does not exist. " + "Check WIKI_DEFAULT_USER_ID." + ) + task_user = configured + + team_name = wiki_settings.CODE_WIKI_TEAM_NAME + if not team_name: + raise CodeWikiRunError( + "WIKI_CODE_WIKI_TEAM_NAME is not configured, so there is no team to run " + "the wiki agent" + ) + + team = team_kinds_service.get_team_by_name_and_namespace( + db=db, + team_name=team_name, + team_namespace="default", + user_id=task_user.id, + ) + if not team: + raise CodeWikiRunError( + f"Code wiki team '{team_name}' was not found for user {task_user.id}. " + "Check WIKI_CODE_WIKI_TEAM_NAME and that the default resources are loaded." + ) + return team, task_user + + +def _create_task( + db: Session, + *, + source: SourceRepository, + team_id: int, + task_user: User, + prompt: str, + generation: WikiGeneration, +) -> int: + """Create the task that runs the agent, failing the run if it cannot be.""" + from app.services.adapters.task_kinds import task_kinds_service + + try: + task_id = task_kinds_service.create_task_id(db, task_user.id) + task_kinds_service.create_task_or_append( + db=db, + obj_in=TaskCreate( + title=f"Code wiki: {source.project_name}", + team_id=team_id, + git_url=source.source_url, + git_repo=source.project_name, + git_repo_id=0, + git_domain=source.source_domain, + # Empty means the repository's default branch, resolved at clone time, + # so a wiki always documents whatever that branch currently is. + branch_name="", + prompt=prompt, + type="online", + task_type="code", + auto_delete_executor="false", + source="code_wiki", + ), + user=task_user, + task_id=task_id, + ) + return task_id + except Exception as exc: + logger.error( + "[code_wiki] could not create a task for generation %s: %s", + generation.id, + exc, + ) + _fail_without_a_task(db, generation, exc) + raise CodeWikiRunError(f"Could not start the wiki task: {exc}") from exc + + +def _fail_without_a_task( + db: Session, generation: WikiGeneration, exc: Exception +) -> None: + """Mark a run failed when nothing will ever report on it. + + Left RUNNING it would block the wiki until the staleness sweep, which is hours of + a wiki refusing to regenerate over a configuration error visible right now. + """ + db.rollback() + live = db.get(WikiGeneration, generation.id) + if live is None: + return + live.status = WikiGenerationStatus.FAILED + live.completed_at = datetime.now(timezone.utc).replace(tzinfo=None) + ext = dict(live.ext or {}) + ext["errorMessage"] = f"task creation failed: {exc}" + live.ext = ext + db.commit() diff --git a/backend/app/services/knowledge/code_wiki_side_effects.py b/backend/app/services/knowledge/code_wiki_side_effects.py new file mode 100644 index 0000000000..b8e31cf30b --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_side_effects.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Wiring the projection's side effects to the real services. + +The projection takes these as injected callables so its ordering rules can be +asserted on directly. This module is the other half of that arrangement: the adapters +that talk to attachment storage, the vector store and the indexing queue. + +Keeping them here rather than inside the projection means the rules and the plumbing +fail separately — a signature that turns out to be wrong shows up as an adapter fault +rather than as an ordering test that no longer proves anything. +""" + +import logging + +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import KnowledgeDocument +from app.models.user import User +from app.services.knowledge.code_wiki_projection import ProjectionSideEffects + +logger = logging.getLogger(__name__) + + +def build_projection_side_effects( + db: Session, + *, + knowledge_base: Kind, + user: User, +) -> ProjectionSideEffects: + """Assemble the real side effects for projecting into ``knowledge_base``. + + Args: + db: Session the projection runs in. Attachment writes join it; the vector + store and the queue cannot, which is why their failures are retried + rather than rolled back. + knowledge_base: The code wiki being projected into. + user: Identity attachments and indexing run under. + """ + from app.services.context import context_service + + def write_attachment(*, filename: str, content: str) -> int: + attachment, _ = context_service.upload_attachment( + db=db, + user_id=user.id, + filename=filename, + binary_data=content.encode("utf-8"), + subtask_id=0, + ) + return attachment.id + + def delete_attachment(attachment_id: int) -> None: + context_service.delete_context( + db=db, + context_id=attachment_id, + user_id=user.id, + ) + + def delete_rag_document(document_id: int) -> None: + _delete_document_index( + db, knowledge_base=knowledge_base, user=user, document_id=document_id + ) + + def enqueue_reindex(document_id: int) -> None: + _enqueue_reindex( + db, knowledge_base=knowledge_base, user=user, document_id=document_id + ) + + return ProjectionSideEffects( + write_attachment=write_attachment, + delete_attachment=delete_attachment, + delete_rag_document=delete_rag_document, + enqueue_reindex=enqueue_reindex, + ) + + +def _delete_document_index( + db: Session, *, knowledge_base: Kind, user: User, document_id: int +) -> None: + """Remove a removed page's chunks from the vector store. + + Raises on failure so the caller can park the reference and retry: chunks left + behind outlive the page, and retrieval goes on answering from it while citing an + id that no longer resolves. + """ + from app.services.knowledge.index_runtime import get_kb_index_info_by_record + from app.services.rag.gateway_factory import get_delete_gateway + from app.services.rag.runtime_resolver import RagRuntimeResolver + + spec = (knowledge_base.json or {}).get("spec", {}) + if not spec.get("retrievalConfig"): + # Nothing was ever indexed, so there is nothing to remove. + return + + index_info = get_kb_index_info_by_record( + db=db, knowledge_base=knowledge_base, current_user_id=user.id + ) + delete_spec = RagRuntimeResolver().build_delete_runtime_spec( + db=db, + knowledge_base_id=knowledge_base.id, + # The index keys documents by their id rendered as a string, which is what + # KnowledgeService.delete_document also uses. + document_ref=str(document_id), + index_owner_user_id=index_info.index_owner_user_id, + ) + result = _run(get_delete_gateway().delete_document_index(delete_spec, db=db)) + status = (result or {}).get("status") + if status not in {"success", "deleted"}: + raise RuntimeError(f"index deletion returned status '{status}'") + + +def _enqueue_reindex( + db: Session, *, knowledge_base: Kind, user: User, document_id: int +) -> None: + """Queue a written page for indexing. + + A page stays invisible until its index succeeds — the state machine turns on + ``is_active`` and ``status`` together at that point — so this is what actually + publishes a page to readers. + """ + from app.services.knowledge.orchestrator import knowledge_orchestrator + + document = db.get(KnowledgeDocument, document_id) + if document is None: + return + knowledge_orchestrator._schedule_indexing_celery( + db=db, + knowledge_base=knowledge_base, + document=document, + user=user, + # Summaries are a knowledge base level concern and would be recomputed once + # per page here; the publish path refreshes them separately. + trigger_summary=False, + replace_active=True, + ) + + +def _run(awaitable): + """Run an awaitable from synchronous code.""" + from app.services.knowledge.knowledge_service import _run_async_in_new_loop + + return _run_async_in_new_loop(awaitable) diff --git a/backend/app/services/knowledge/code_wiki_source.py b/backend/app/services/knowledge/code_wiki_source.py new file mode 100644 index 0000000000..069f9210cc --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_source.py @@ -0,0 +1,261 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Source repository binding for code wikis, and the gate guarding it. + +A code wiki is derived from a repository, so its pages can expose whatever that +repository contains. Access is therefore split in two: + +- **Creating** a code wiki is gated here: the requester must be able to read the + repository. This stops someone from having a wiki built for a private repository + they cannot read themselves. +- **Reading** an existing code wiki is governed purely by knowledge-base permissions + (namespace roles, resource members, organization visibility). Re-checking the + repository on every read was slow and fragile, and it is not what decides who may + see a knowledge base. + +Leak protection therefore rests on this gate plus the namespace the creator picks, +which is the same trust model as pasting private code into any other knowledge base. +""" + +import logging +from dataclasses import dataclass +from typing import Any, Dict, Optional + +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from app.services.git_skill.utils import get_user_git_info, parse_repo_url + +logger = logging.getLogger(__name__) + +SUPPORTED_SOURCE_TYPES = ("github", "gitlab", "gitea") + + +class SourceAccessDenied(Exception): + """Raised when a user cannot read the repository they asked to document.""" + + +@dataclass(frozen=True) +class SourceRepository: + """The repository a code wiki is generated from. + + Every field except the type is derived from the URL rather than accepted + separately, so the repository this gate checks is necessarily the repository that + later gets cloned. Taking the domain and project name as independent inputs would + let a caller pass a repository they can read while storing a URL pointing + somewhere else entirely — the gate would approve one repository and the wiki would + be built from another. + """ + + source_type: str + # Credentials stripped: a URL may arrive with a token embedded, and this value is + # stored on the knowledge base where anyone who can read it would see them. + source_url: str + project_name: str + source_domain: str + + @classmethod + def from_url(cls, source_type: str, source_url: str) -> "SourceRepository": + """Build from a repository URL, deriving the domain and project name. + + Raises: + SourceAccessDenied: If the type is unsupported or the URL is unusable. + Both are refusals to bind a repository, so they surface the same way. + """ + if source_type not in SUPPORTED_SOURCE_TYPES: + raise SourceAccessDenied( + f"Unsupported repository type '{source_type}'. " + f"Supported types: {', '.join(SUPPORTED_SOURCE_TYPES)}" + ) + + try: + parsed = parse_repo_url(source_url) + except HTTPException as exc: + raise SourceAccessDenied( + f"Could not read a repository from '{source_url}': {exc.detail}" + ) from exc + except Exception as exc: + raise SourceAccessDenied( + f"Could not read a repository from '{source_url}'" + ) from exc + + if not parsed.domain or not parsed.owner or not parsed.repo: + raise SourceAccessDenied( + f"Could not read a host and project from '{source_url}'" + ) + + _assert_host_is_addressable(parsed.domain) + + repo = parsed.repo[:-4] if parsed.repo.endswith(".git") else parsed.repo + return cls( + source_type=source_type, + source_url=f"https://{parsed.domain}/{parsed.owner}/{repo}.git", + project_name=f"{parsed.owner}/{repo}", + source_domain=parsed.domain, + ) + + def to_spec(self) -> Dict[str, Any]: + """Render as the ``spec.source`` object stored on the knowledge base.""" + return { + "sourceType": self.source_type, + "sourceUrl": self.source_url, + "sourceDomain": self.source_domain, + "projectName": self.project_name, + } + + @classmethod + def from_spec( + cls, spec_source: Optional[Dict[str, Any]] + ) -> Optional["SourceRepository"]: + """Read back a stored source, or ``None`` when one was never recorded.""" + if not spec_source: + return None + return cls( + source_type=str(spec_source.get("sourceType", "")), + source_url=str(spec_source.get("sourceUrl", "")), + project_name=str(spec_source.get("projectName", "")), + source_domain=str(spec_source.get("sourceDomain", "")), + ) + + +def _assert_host_is_addressable(host: str) -> None: + """Refuse hosts that are never a Git server but are a useful SSRF target. + + Binding a repository makes the server fetch from whatever host the URL names, + carrying the caller's token, so the host is attacker-chosen input. The cloud + metadata endpoint sits on a link-local address and answers unauthenticated + requests with instance credentials — that is the one worth ruling out by name. + + Private ranges are deliberately still allowed: a self-hosted GitLab or Gitea on + an internal network is the normal deployment here, and blocking those would + break the product to close a much smaller hole. + + This bounds what a URL may *name*. It does not stop a public name that resolves + inward, which needs a check at connection time rather than here. + """ + import ipaddress + + if host in {"localhost", "localhost.localdomain"}: + raise SourceAccessDenied(f"'{host}' is not a reachable repository host") + + try: + address = ipaddress.ip_address(host) + except ValueError: + # A name rather than a literal; nothing more to check without resolving it. + return + + # ``is_unspecified`` is checked alongside the other two because 0.0.0.0 and :: + # are neither loopback nor link-local, and connecting to them reaches the local + # host anyway. Python already folds IPv4-mapped forms into these properties, so + # ::ffff:127.0.0.1 and ::ffff:169.254.169.254 are covered without special casing; + # the tests pin that, since it is a property of the standard library rather than + # of anything written here. + if address.is_loopback or address.is_link_local or address.is_unspecified: + raise SourceAccessDenied(f"'{host}' is not a reachable repository host") + + +def provider_for(source_type: str): + if source_type == "github": + from app.repository.github_provider import GitHubProvider + + return GitHubProvider() + if source_type == "gitlab": + from app.repository.gitlab_provider import GitLabProvider + + return GitLabProvider() + if source_type == "gitea": + from app.repository.gitea_provider import GiteaProvider + + return GiteaProvider() + return None + + +def _check_access(provider, source_type: str, token: str, source: SourceRepository): + """Ask the provider whether the token can read the repository. + + GitLab names its parameter ``project_id`` but accepts a full project path, which + is what is used here: a caller-supplied numeric id could point at a different + project than the URL, which is the mismatch this module exists to rule out. + """ + if source_type == "gitlab": + return provider.check_user_project_access( + token=token, + git_domain=source.source_domain, + project_id=source.project_name, + ) + return provider.check_user_project_access( + token=token, + git_domain=source.source_domain, + repo_name=source.project_name, + ) + + +def assert_user_can_read_source( + db: Session, user_id: int, source: SourceRepository +) -> Dict[str, Any]: + """Verify a user can read the repository, or raise ``SourceAccessDenied``. + + Returns the provider's access details so callers can log the granted level. + """ + if source.source_type not in SUPPORTED_SOURCE_TYPES: + raise SourceAccessDenied( + f"Unsupported repository type '{source.source_type}'. " + f"Supported types: {', '.join(SUPPORTED_SOURCE_TYPES)}" + ) + + git_info: Optional[Dict[str, Any]] = get_user_git_info( + user_id=user_id, domain=source.source_domain, db=db + ) + if not git_info or not git_info.get("token"): + raise SourceAccessDenied( + f"No credentials configured for {source.source_domain}. " + "Add a token for this Git domain before creating a code wiki." + ) + + # The credential's own type must agree with the declared one. Asking one provider + # about a repository hosted by another produces a meaningless answer, so require + # agreement rather than guessing which side is right. + configured_type = git_info.get("type") + if configured_type and configured_type != source.source_type: + raise SourceAccessDenied( + f"Credentials for {source.source_domain} are configured as " + f"'{configured_type}', but the repository was declared as " + f"'{source.source_type}'." + ) + + provider = provider_for(source.source_type) + if provider is None: + raise SourceAccessDenied(f"Unsupported repository type '{source.source_type}'") + + try: + result = _check_access(provider, source.source_type, git_info["token"], source) + except Exception as exc: + # Deny on error: an unreachable provider must not become an open door. + logger.warning( + "[code_wiki] repository access check failed for %s: %s", + source.project_name, + exc, + ) + # The provider's own error text is logged above but deliberately kept out of + # the message, which reaches the client as a 403 body. It is an external + # system's wording about our internal request, not an answer to the caller. + raise SourceAccessDenied( + f"Could not verify access to '{source.project_name}'. " + "Please try again later." + ) from exc + + if not result.get("has_access", False): + raise SourceAccessDenied( + f"You do not have read access to '{source.project_name}'. " + f"{result.get('error', '')}".strip() + ) + + logger.info( + "[code_wiki] user %s has %s access to %s", + user_id, + result.get("access_level_name", "read"), + source.project_name, + ) + return result diff --git a/backend/app/services/knowledge/code_wiki_version_store.py b/backend/app/services/knowledge/code_wiki_version_store.py new file mode 100644 index 0000000000..d2e19a9e66 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki_version_store.py @@ -0,0 +1,337 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""The version store behind a code wiki. + +A generation run writes here, not into the knowledge base. The knowledge base is a +projection of whichever version is published, so everything a run does — including +crashing halfway through — stays invisible until a complete version passes its publish +gate. + +That containment is the point: it puts the atomicity boundary around a deterministic +projection measured in seconds instead of around an LLM run measured in hours. + +Three invariants hold here: + +- **A version is a complete snapshot.** An incremental run is seeded from the + published version and revises it in place, so the projection can always treat a + version as the whole truth and compute orphans as a plain set difference. +- **Seeding is the server's job.** The agent is told what changed and what pages + exist, but is never responsible for carrying unchanged pages forward. +- **The published version is never collected.** Retention would otherwise eat the + rollback baseline exactly when it is needed most. +""" + +import logging +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Optional, Sequence + +from sqlalchemy import func, insert, literal, select +from sqlalchemy.orm import Session + +from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.services.knowledge.code_wiki_page_path import ( + collation_key, + normalize_page_path, +) + +logger = logging.getLogger(__name__) + +# Key under which a page's stable path lives in ``WikiContent.ext``. +PATH_EXT_KEY = "path" + +# How long a run may sit in flight before it is treated as abandoned. Generous, because +# a large repository legitimately takes a long time; the cost of being wrong is one +# duplicated run, whereas never expiring costs a wiki that can never be regenerated +# again — the lock below would refuse every later run forever. +STALE_RUN_AFTER_HOURS = 6.0 + +IN_FLIGHT_STATUSES = ( + WikiGenerationStatus.PENDING, + WikiGenerationStatus.RUNNING, +) + +# Retention defaults. Successful versions are the rollback material; failed ones are +# kept only long enough to be looked at. +DEFAULT_KEEP_SUCCESSFUL = 10 +DEFAULT_MAX_AGE_DAYS = 90 +DEFAULT_FAILED_RETENTION_DAYS = 7 + + +def _utcnow() -> datetime: + """Return a timezone-naive UTC timestamp, matching the wiki tables.""" + return datetime.now(timezone.utc).replace(tzinfo=None) + + +def _as_naive_utc(value: Optional[datetime]) -> datetime: + """Normalize a caller-supplied instant to the naive UTC the wiki tables store. + + An aware value is converted before its offset is dropped. Dropping it outright + would shift the instant by that offset, and both callers compare the result + against stored timestamps to decide what to reclaim or delete. + """ + if value is None: + return _utcnow() + if value.tzinfo is None: + return value + return value.astimezone(timezone.utc).replace(tzinfo=None) + + +@dataclass(frozen=True) +class SeedOutcome: + """What seeding did.""" + + copied_pages: int + skipped_reason: str = "" + + @property + def seeded(self) -> bool: + return not self.skipped_reason + + +def page_path_of(content: WikiContent) -> str: + """Return the stable path recorded on a version entry, or an empty string.""" + ext = content.ext or {} + return str(ext.get(PATH_EXT_KEY, "") or "") + + +def set_page_path(content: WikiContent, path: str) -> None: + """Record a normalized path on a version entry.""" + ext = dict(content.ext or {}) + ext[PATH_EXT_KEY] = path + content.ext = ext + + +def seed_from_published( + db: Session, + *, + target_generation_id: int, + published_generation_id: int, +) -> SeedOutcome: + """Copy the published version's pages into a new generation. + + Runs before the agent starts, so an incremental run revises a complete snapshot + rather than producing a partial one. A run that then fails leaves the seed as + unreferenced rows under a failed generation, which retention collects. + + Idempotent: a generation that already holds pages is left alone, so a retried + scheduling attempt cannot double the version. + """ + if published_generation_id <= 0: + return SeedOutcome(0, skipped_reason="no published version to seed from") + + if target_generation_id == published_generation_id: + return SeedOutcome(0, skipped_reason="target is the published version") + + existing = ( + db.query(func.count(WikiContent.id)) + .filter(WikiContent.generation_id == target_generation_id) + .scalar() + ) + if existing: + return SeedOutcome(0, skipped_reason="generation already holds pages") + + now = _utcnow() + # parent_id is deliberately not carried over: it refers to row ids inside the + # source generation, so copying it would point every seeded page at a row in a + # different version. Hierarchy comes from the page path, which is copied intact. + source = select( + literal(target_generation_id), + WikiContent.type, + WikiContent.title, + WikiContent.content, + literal(0), + WikiContent.ext, + literal(now), + literal(now), + ).where(WikiContent.generation_id == published_generation_id) + + db.execute( + insert(WikiContent).from_select( + [ + "generation_id", + "type", + "title", + "content", + "parent_id", + "ext", + "created_at", + "updated_at", + ], + source, + ) + ) + db.flush() + + copied = ( + db.query(func.count(WikiContent.id)) + .filter(WikiContent.generation_id == target_generation_id) + .scalar() + or 0 + ) + logger.info( + "[code_wiki] seeded generation %s with %s pages from %s", + target_generation_id, + copied, + published_generation_id, + ) + return SeedOutcome(copied) + + +def remove_page(db: Session, *, generation_id: int, path: str) -> bool: + """Drop a page from an in-flight version at the agent's request. + + Only the agent knows which page covered a module that no longer exists: phase one + records no provenance, so the server sees changed file paths but cannot infer the + page they belong to. The risk this creates is contained rather than avoided — the + removal lands in an unpublished version, the previous version still has the page, + and the publish gate checks how far the page count dropped. + + Returns: + Whether a page was removed. + """ + normalized = normalize_page_path(path) + wanted = collation_key(normalized) + + for content in ( + db.query(WikiContent).filter(WikiContent.generation_id == generation_id).all() + ): + if collation_key(page_path_of(content)) == wanted: + db.delete(content) + db.flush() + logger.info( + "[code_wiki] removed page '%s' from generation %s", + normalized, + generation_id, + ) + return True + return False + + +def reclaim_stale_generations( + db: Session, + *, + kind_id: int, + now: Optional[datetime] = None, + stale_after_hours: float = STALE_RUN_AFTER_HOURS, +) -> tuple[int, ...]: + """Fail in-flight generations whose worker is gone. + + Without this a single lost worker blocks the wiki permanently: scheduling takes + the generation row's lock and refuses to start while one is in flight, and a + crashed run never leaves that state on its own. + + Returns: + Ids of the generations that were failed. + """ + cutoff = _as_naive_utc(now) - timedelta(hours=stale_after_hours) + + stale: Sequence[WikiGeneration] = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == kind_id, + WikiGeneration.status.in_(IN_FLIGHT_STATUSES), + WikiGeneration.updated_at < cutoff, + ) + .all() + ) + if not stale: + return () + + for generation in stale: + generation.status = WikiGenerationStatus.FAILED + generation.completed_at = _as_naive_utc(now) + db.flush() + + reclaimed = tuple(generation.id for generation in stale) + logger.warning( + "[code_wiki] reclaimed %s abandoned generation(s) for kb %s: %s", + len(reclaimed), + kind_id, + reclaimed, + ) + return reclaimed + + +def apply_retention( + db: Session, + *, + kind_id: int, + published_generation_id: int, + keep_successful: int = DEFAULT_KEEP_SUCCESSFUL, + max_age_days: int = DEFAULT_MAX_AGE_DAYS, + failed_retention_days: int = DEFAULT_FAILED_RETENTION_DAYS, + now: Optional[datetime] = None, +) -> tuple[int, ...]: + """Collect versions that are no longer worth keeping. + + The published version is exempt unconditionally. Two ordinary situations would + otherwise collect it and leave the wiki with no version to roll back to: a run of + consecutive failures pushing it out of the newest ``keep_successful``, and a + repository quiet for long enough that every version ages out. + + Returns: + Ids of the generations that were deleted. + """ + reference = _as_naive_utc(now) + doomed: list[int] = [] + + successful = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == kind_id, + WikiGeneration.status == WikiGenerationStatus.COMPLETED, + ) + .order_by(WikiGeneration.created_at.desc(), WikiGeneration.id.desc()) + .all() + ) + age_cutoff = reference - timedelta(days=max_age_days) + for position, generation in enumerate(successful): + if generation.id == published_generation_id: + continue + over_count = position >= keep_successful + over_age = (generation.created_at or reference) < age_cutoff + if over_count or over_age: + doomed.append(generation.id) + + # Only terminal generations are collected. Excluding COMPLETED alone would sweep + # up PENDING and RUNNING rows once they are old enough, deleting the pages of a + # run still being written to. Reclamation normally fails an abandoned run long + # before that, but retention must not depend on it having done so. + failed_cutoff = reference - timedelta(days=failed_retention_days) + unsuccessful = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == kind_id, + WikiGeneration.status.notin_( + [WikiGenerationStatus.COMPLETED, *IN_FLIGHT_STATUSES] + ), + WikiGeneration.created_at < failed_cutoff, + ) + .all() + ) + for generation in unsuccessful: + if generation.id != published_generation_id: + doomed.append(generation.id) + + if not doomed: + return () + + # Contents are removed explicitly rather than through the foreign key, because + # SQLite does not enforce ON DELETE CASCADE unless asked to and the tests run there. + db.query(WikiContent).filter(WikiContent.generation_id.in_(doomed)).delete( + synchronize_session=False + ) + db.query(WikiGeneration).filter(WikiGeneration.id.in_(doomed)).delete( + synchronize_session=False + ) + db.flush() + + logger.info( + "[code_wiki] retention removed %s version(s) from kb %s", + len(doomed), + kind_id, + ) + return tuple(doomed) diff --git a/backend/app/services/knowledge/content_scope.py b/backend/app/services/knowledge/content_scope.py new file mode 100644 index 0000000000..538ead5a3c --- /dev/null +++ b/backend/app/services/knowledge/content_scope.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Query scopes for knowledge content, separated by ownership and target kind. + +A knowledge base can hold content that serves different purposes: + +- **Wiki pages** — documents a reader browses in the folder tree. +- **Code targets** (planned) — one hidden target per indexed source file. These are + retrieval artifacts, not pages: they carry no folder, never appear in the page + tree, and are only reachable through retrieval. + +Every read that means "the pages of this wiki" must exclude code targets, and every +write or cleanup driven by generation must be limited to agent-owned content. Leaving +that to each call site has already proven unreliable — the filter was missed twice +during design review, once for folder listing and once for the generated-content sweep, +where it would have deleted the whole code index. + +So the filters live here instead of at the call sites. Callers pick a scope by name +and cannot express "no filter" by accident; reaching code targets requires asking for +them explicitly. +""" + +from sqlalchemy.orm import Query + +from app.models.knowledge import ( + ContentOrigin, + DocumentSourceType, + KnowledgeDocument, + KnowledgeFolder, +) + +# Source type marking a document as an indexed source file rather than a wiki page. +CODE_TARGET_SOURCE_TYPE = DocumentSourceType.CODE.value + +# folder_id for targets that are deliberately outside the folder tree. Distinct from +# 0 (root level) so that listing a folder's children — including the root's — cannot +# return them even if a caller bypasses these scopes. +NO_FOLDER = -1 + + +def wiki_pages(query: Query) -> Query: + """Restrict a ``KnowledgeDocument`` query to browsable wiki pages. + + Excludes code targets. Use for the folder tree, document listing, page + navigation, and anything else a reader sees. + """ + return query.filter(KnowledgeDocument.source_type != CODE_TARGET_SOURCE_TYPE) + + +def generated_wiki_pages(query: Query) -> Query: + """Restrict a ``KnowledgeDocument`` query to agent-owned wiki pages. + + This is the scope the projection owns: the only documents it may create, update or + delete. It excludes both user-owned content — which is not regenerable, so a + mistaken delete is unrecoverable — and code targets, which no wiki version + produces and which a set difference would therefore treat as orphans. + """ + return wiki_pages(query).filter( + KnowledgeDocument.origin == ContentOrigin.GENERATED.value + ) + + +def generated_folders(query: Query) -> Query: + """Restrict a ``KnowledgeFolder`` query to agent-owned folders.""" + return query.filter(KnowledgeFolder.origin == ContentOrigin.GENERATED.value) + + +def code_targets(query: Query) -> Query: + """Restrict a ``KnowledgeDocument`` query to indexed source files. + + Separate from every reader-facing scope, so retrieval and index maintenance have + to name code targets deliberately. + """ + return query.filter(KnowledgeDocument.source_type == CODE_TARGET_SOURCE_TYPE) diff --git a/backend/app/services/knowledge/knowledge_service.py b/backend/app/services/knowledge/knowledge_service.py index 123e4accd0..66544ce21e 100644 --- a/backend/app/services/knowledge/knowledge_service.py +++ b/backend/app/services/knowledge/knowledge_service.py @@ -44,6 +44,7 @@ BatchOperationResult, KnowledgeBaseCreate, KnowledgeBaseResponse, + KnowledgeBaseType, KnowledgeBaseUpdate, KnowledgeBaseWithGroupInfo, KnowledgeDocumentCreate, @@ -59,6 +60,7 @@ get_user_groups, get_view_role_in_group, ) +from app.services.knowledge.content_scope import wiki_pages from app.services.knowledge.folder_policy import assert_document_can_be_placed_in_folder from app.services.knowledge.knowledge_access_policy import ( can_directly_access_knowledge_base as evaluate_direct_knowledge_base_access, @@ -292,11 +294,18 @@ def create_knowledge_base( "name": data.name, "description": data.description or "", "directAccessRequirement": data.direct_access_requirement, - "kbType": data.kb_type - or "notebook", # Default to 'notebook' if not provided + # A code wiki is fixed here; there is deliberately no code path that turns + # an existing knowledge base into one, or out of one. + "kbType": KnowledgeBaseType( + data.kb_type or KnowledgeBaseType.NOTEBOOK + ).value, "retrievalConfig": _to_json_dict(data.retrieval_config), "summaryEnabled": data.summary_enabled, } + # A code wiki records the repository it is generated from + if data.source: + spec_kwargs["source"] = data.source + # Add summaryModelRef if provided if data.summary_model_ref: spec_kwargs["summaryModelRef"] = data.summary_model_ref @@ -1040,7 +1049,15 @@ def update_knowledge_base_type( # Get current default view kb_json = kb.json spec = kb_json.get("spec", {}) - current_type = spec.get("kbType", "notebook") + + current_type = spec.get("kbType", KnowledgeBaseType.NOTEBOOK.value) + + # A code wiki is a repository binding, not a view preference, so the toggle does + # not apply. Rejecting it here also keeps this endpoint from being used to + # reinterpret a code wiki as an ordinary knowledge base, which would orphan its + # repository and version history. + if current_type == KnowledgeBaseType.CODE_WIKI.value: + raise ValueError("Cannot change the opening view of a code wiki") # If same type, return current kb if current_type == new_type: @@ -1472,8 +1489,12 @@ def list_documents( if not kb or not has_access: return [] - query = db.query(KnowledgeDocument).filter( - KnowledgeDocument.kind_id == knowledge_base_id, + # Scoped to browsable pages: code targets are retrieval artifacts and must + # never surface in a listing. See content_scope. + query = wiki_pages( + db.query(KnowledgeDocument).filter( + KnowledgeDocument.kind_id == knowledge_base_id, + ) ) if folder_id is not None: @@ -1502,8 +1523,11 @@ def list_documents_paginated( if not kb or not has_access: return [], 0 - query = db.query(KnowledgeDocument).filter( - KnowledgeDocument.kind_id == knowledge_base_id, + # Scoped to browsable pages; see list_documents. + query = wiki_pages( + db.query(KnowledgeDocument).filter( + KnowledgeDocument.kind_id == knowledge_base_id, + ) ) if folder_id is not None: diff --git a/backend/app/services/knowledge/mermaid_check.py b/backend/app/services/knowledge/mermaid_check.py new file mode 100644 index 0000000000..c7f74a8fea --- /dev/null +++ b/backend/app/services/knowledge/mermaid_check.py @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Structural checks on Mermaid diagrams in generated Markdown. + +Diagrams are rendered in the browser, so a malformed one spoils a figure rather than a +page. Withholding an otherwise good page over it would cost more than it saves, so +these findings are returned to the writing agent as warnings and never block +publication. + +The checks are structural on purpose: they catch the mistakes a model actually makes — +a misspelled diagram type, an unclosed fence, unbalanced brackets — without running +Mermaid itself, which is a JavaScript parser this service cannot host. What they +cannot do is certify that a diagram renders; they only report what is definitely +wrong. The warning contract is stable, so swapping in a real parser later would not +change the callers. +""" + +import re +from dataclasses import dataclass +from typing import List, Sequence + +FENCE_PATTERN = re.compile(r"^(\s*)(`{3,}|~{3,})\s*([A-Za-z0-9_+-]*)\s*$") + +# Diagram types Mermaid recognises. A block starting with anything else will not +# render, and a misspelling here is the most common failure in generated diagrams. +# Tracks the Mermaid the frontend actually resolves to (11.15.x via ^11.4.0). A type +# missing here is reported as a broken diagram and sent back to the agent to "fix", +# which wastes a round on a diagram that renders perfectly well — so this list has to +# be widened whenever the pinned version gains a declaration. +KNOWN_DIAGRAM_TYPES: frozenset[str] = frozenset( + { + "architecture-beta", + "block-beta", + "c4component", + "c4container", + "c4context", + "c4deployment", + "c4dynamic", + "classdiagram", + "classdiagram-v2", + "erdiagram", + "flowchart", + "flowchart-elk", + "gantt", + "gitgraph", + "graph", + "journey", + "kanban", + "mindmap", + "packet", + "packet-beta", + "pie", + "quadrantchart", + "radar", + "requirementdiagram", + "sankey-beta", + "sequencediagram", + "statediagram", + "statediagram-v2", + "timeline", + "treemap", + "xychart-beta", + "zenuml", + } +) + +BRACKET_PAIRS = {"(": ")", "[": "]", "{": "}"} + +# Diagram types where brackets delimit nodes, so an unbalanced one is a real mistake. +# The check is limited to these because elsewhere brackets are not delimiters at all: +# an ER diagram writes cardinality as ``||--o{``, which is deliberately "unbalanced", +# and the text-heavy types put arbitrary prose in labels. Reporting those would send +# the agent off to fix diagrams that are already correct. +BRACKET_CHECKED_TYPES: frozenset[str] = frozenset( + { + "block-beta", + "classdiagram", + "flowchart", + "graph", + "mindmap", + "statediagram", + "statediagram-v2", + } +) + + +@dataclass(frozen=True) +class MermaidWarning: + """One problem found in a diagram, addressed to the agent that wrote it.""" + + # 1-based line of the fence that opens the diagram, so the agent can find it. + line: int + message: str + + def __str__(self) -> str: + return f"line {self.line}: {self.message}" + + +def _diagram_type_of(body: Sequence[str]) -> str: + """First meaningful token of a diagram, lowercased; empty when there is none.""" + for line in body: + stripped = line.strip() + if not stripped or stripped.startswith("%%"): + continue + # "flowchart TD", "stateDiagram-v2", "graph LR" — the type is the first token. + return re.split(r"[\s:]", stripped, maxsplit=1)[0].lower() + return "" + + +def _unbalanced_bracket(body: Sequence[str]) -> str: + """Report the first unbalanced bracket, ignoring anything inside quotes.""" + stack: List[str] = [] + for line in body: + in_quote = False + for char in line: + if char == '"': + in_quote = not in_quote + continue + if in_quote: + continue + if char in BRACKET_PAIRS: + stack.append(char) + elif char in BRACKET_PAIRS.values(): + if not stack: + return f"unexpected closing '{char}'" + if BRACKET_PAIRS[stack.pop()] != char: + return f"mismatched closing '{char}'" + if in_quote: + return "unclosed quote" + if stack: + return f"unclosed '{stack[-1]}'" + return "" + + +def check_mermaid_blocks(markdown: str) -> List[MermaidWarning]: + """Find structural problems in the Mermaid diagrams of a Markdown document. + + Returns an empty list when nothing is definitely wrong. Nested fences inside other + fenced blocks are skipped, so a Mermaid example quoted inside a code sample is not + mistaken for a diagram. + """ + warnings: List[MermaidWarning] = [] + lines = markdown.splitlines() + + index = 0 + while index < len(lines): + match = FENCE_PATTERN.match(lines[index]) + if not match: + index += 1 + continue + + _, marker, info = match.groups() + opened_at = index + 1 + is_mermaid = info.lower() == "mermaid" + + body: List[str] = [] + index += 1 + closed = False + while index < len(lines): + closing = FENCE_PATTERN.match(lines[index]) + # A closing fence must use the same character and be at least as long as + # the opening one. Length matters because wrapping an example in a longer + # fence is how a diagram gets quoted rather than declared: without this, + # the inner ``` ends the outer ````, and the rest of the example is read + # as live markdown. + if ( + closing + and closing.group(2)[0] == marker[0] + and len(closing.group(2)) >= len(marker) + and not closing.group(3) + ): + closed = True + index += 1 + break + body.append(lines[index]) + index += 1 + + if not is_mermaid: + # A non-Mermaid fence is only consumed so its contents cannot be mistaken + # for a diagram; problems inside it are not ours to report. + continue + + if not closed: + warnings.append(MermaidWarning(opened_at, "diagram fence is never closed")) + continue + + diagram_type = _diagram_type_of(body) + if not diagram_type: + warnings.append(MermaidWarning(opened_at, "diagram is empty")) + continue + if diagram_type not in KNOWN_DIAGRAM_TYPES: + warnings.append( + MermaidWarning( + opened_at, + f"'{diagram_type}' is not a Mermaid diagram type, so this " + "diagram will not render", + ) + ) + continue + + if diagram_type in BRACKET_CHECKED_TYPES: + problem = _unbalanced_bracket(body) + if problem: + warnings.append(MermaidWarning(opened_at, problem)) + + return warnings + + +def describe_warnings(warnings: Sequence[MermaidWarning]) -> str: + """Render warnings as an instruction the writing agent can act on.""" + if not warnings: + return "" + listed = "\n".join(f"- {warning}" for warning in warnings) + return ( + "The Mermaid diagrams below will not render. Fix them and write the page " + f"again:\n{listed}" + ) diff --git a/backend/app/services/knowledge/orchestrator.py b/backend/app/services/knowledge/orchestrator.py index 77abcd3825..288eb708c6 100644 --- a/backend/app/services/knowledge/orchestrator.py +++ b/backend/app/services/knowledge/orchestrator.py @@ -36,11 +36,13 @@ KnowledgeBaseCreate, KnowledgeBaseListResponse, KnowledgeBaseResponse, + KnowledgeBaseType, KnowledgeDocumentCreate, KnowledgeDocumentListResponse, KnowledgeDocumentResponse, ResourceScope, ) +from app.services.knowledge.code_wiki_source import SourceRepository from app.services.knowledge.document_read_service import ( DOCUMENT_READ_ERROR_NOT_FOUND, document_read_service, @@ -1066,7 +1068,8 @@ def create_knowledge_base( description: Optional[str] = None, namespace: str = "default", direct_access_requirement: Literal["read", "edit"] = "read", - kb_type: str = "notebook", + kb_type: str = KnowledgeBaseType.NOTEBOOK.value, + source: Optional[SourceRepository] = None, summary_enabled: bool = False, rag_config_mode: Literal["auto", "disabled"] = "auto", # REST API scenario: pass complete config @@ -1191,7 +1194,8 @@ def create_knowledge_base( description=description, namespace=namespace, direct_access_requirement=direct_access_requirement, - kb_type=kb_type, + kb_type=KnowledgeBaseType(kb_type), + source=source.to_spec() if source else None, retrieval_config=resolved_retrieval_config, summary_enabled=summary_enabled, summary_model_ref=resolved_summary_model_ref, diff --git a/backend/app/services/wiki_service.py b/backend/app/services/wiki_service.py index 6db519d0aa..3904da6673 100644 --- a/backend/app/services/wiki_service.py +++ b/backend/app/services/wiki_service.py @@ -23,12 +23,25 @@ ) from app.schemas.task import TaskCreate from app.schemas.wiki import ( + WikiContentSummary, WikiContentWriteRequest, WikiGenerationCreate, WikiProjectCreate, ) from app.services.adapters.task_kinds import task_kinds_service from app.services.adapters.team_kinds import team_kinds_service +from app.services.knowledge.code_wiki_page_path import ( + InvalidPagePath, + assert_unique_within_version, + collation_key, + normalize_page_path, +) +from app.services.knowledge.code_wiki_runner import finish_run, is_code_wiki_generation +from app.services.knowledge.code_wiki_version_store import ( + page_path_of, + remove_page, + set_page_path, +) from app.services.user import user_service from shared.utils.url_util import domains_match @@ -562,10 +575,10 @@ def save_generation_contents( - Resilient writes regardless of current generation status so reruns can overwrite results """ has_sections = bool(payload.sections) - if not has_sections and not payload.summary: + if not has_sections and not payload.summary and not payload.removed_paths: raise HTTPException( status_code=400, - detail="No sections or summary provided", + detail="No sections, removals or summary provided", ) total_payload_size = ( @@ -595,34 +608,72 @@ def save_generation_contents( existing_contents: List[WikiContent] = [] if has_sections: + # Pages are identified by path when one is given. Normalising up front + # means a malformed path fails this write rather than the publish of the + # whole version, and keeps two spellings of one path from becoming two + # pages that the projection could not both honour. + normalized_paths: Dict[int, str] = {} + try: + for index, section in enumerate(payload.sections): + if section.path: + normalized_paths[index] = normalize_page_path(section.path) + assert_unique_within_version(normalized_paths.values()) + except InvalidPagePath as exc: + # The generation row is held with_for_update; release it before + # raising, as the other failure branches in this method do. + wiki_db.rollback() + raise HTTPException(status_code=400, detail=str(exc)) from exc + titles = [section.title for section in payload.sections] + # Everything in the generation is loaded, not just the titles being + # written: a page whose title changed has to be found by its path. existing_contents = ( wiki_db.query(WikiContent) - .filter( - WikiContent.generation_id == generation.id, - WikiContent.title.in_(titles), - ) + .filter(WikiContent.generation_id == generation.id) .with_for_update() .all() ) + existing_by_path: Dict[str, WikiContent] = {} + path_less: List[WikiContent] = [] + for content in existing_contents: + content_path = page_path_of(content) + if content_path: + existing_by_path[collation_key(content_path)] = content + else: + path_less.append(content) + + # The title-based indices deliberately exclude anything that already has a + # page path. Moving a page keeps its title, so a title can name several + # path-identified pages; a legacy write resolving through it would pick one + # of them by query order rather than by any rule, and overwrite it. existing_by_key: Dict[Tuple[str, str], WikiContent] = { - (content.type, content.title): content for content in existing_contents + (content.type, content.title): content for content in path_less } existing_by_title: Dict[str, WikiContent] = { - content.title: content for content in existing_contents + content.title: content + for content in path_less + if content.title in titles } - for section in payload.sections: - content_item = existing_by_key.get( - (section.type, section.title) - ) or existing_by_title.get(section.title) + for index, section in enumerate(payload.sections): + path = normalized_paths.get(index) + if path: + content_item = existing_by_path.get(collation_key(path)) + else: + # Legacy write path: no page identity was supplied, so fall back + # to matching on the title as this API originally did. + content_item = existing_by_key.get( + (section.type, section.title) + ) or existing_by_title.get(section.title) if content_item: content_item.type = section.type content_item.title = section.title content_item.content = section.content content_item.ext = section.ext or None + if path: + set_page_path(content_item, path) content_item.updated_at = now updated_sections += 1 else: @@ -638,6 +689,9 @@ def save_generation_contents( created_at=now, updated_at=now, ) + if path: + set_page_path(content_record, path) + existing_by_path[collation_key(path)] = content_record wiki_db.add(content_record) created_sections += 1 @@ -654,6 +708,10 @@ def save_generation_contents( status_code=400, detail="Failed to persist wiki contents" ) + # Applied after the writes, so that a payload both writing and removing a path + # ends with it removed regardless of the order the agent listed them in. + removed_paths = self._apply_removals(wiki_db, generation, payload.removed_paths) + summary = payload.summary previous_status = generation.status ext = generation.ext.copy() if isinstance(generation.ext, dict) else {} @@ -662,6 +720,8 @@ def save_generation_contents( content_meta["last_write_titles"] = titles content_meta["created_sections"] = created_sections content_meta["updated_sections"] = updated_sections + if removed_paths: + content_meta["removed_paths"] = removed_paths content_meta["status_before_write"] = ( previous_status.value if isinstance(previous_status, WikiGenerationStatus) @@ -685,6 +745,12 @@ def save_generation_contents( generation.ext = ext generation.updated_at = now + # A code wiki does not simply record its outcome: a successful version has to + # pass the publish gate and be projected into the knowledge base, and that runs + # after this write is committed rather than inside it. Deferring the status too + # keeps the two from disagreeing if the projection is refused. + finishes_a_code_wiki = False + if summary and summary.status: try: status_enum = WikiGenerationStatus(summary.status) @@ -698,8 +764,12 @@ def save_generation_contents( status_code=400, detail=f"Unsupported summary status: {summary.status}", ) from exc - generation.status = status_enum - if status_enum in { + finishes_a_code_wiki = is_code_wiki_generation(wiki_db, generation) + if finishes_a_code_wiki: + generation.status = WikiGenerationStatus.RUNNING + else: + generation.status = status_enum + if not finishes_a_code_wiki and status_enum in { WikiGenerationStatus.COMPLETED, WikiGenerationStatus.FAILED, WikiGenerationStatus.CANCELLED, @@ -748,6 +818,71 @@ def save_generation_contents( content_meta.get("status_after_write"), ) + if finishes_a_code_wiki: + self._finish_code_wiki(wiki_db, generation, summary) + + def _apply_removals( + self, + wiki_db: Session, + generation: WikiGeneration, + paths: List[str], + ) -> List[str]: + """Drop pages the agent declared gone, returning the ones that existed. + + A path that names no page is reported rather than refused: an agent listing a + page it already removed on a retry has nothing to correct, and failing the + whole write would lose the sections alongside it. + """ + removed: List[str] = [] + for raw_path in paths: + try: + normalized = normalize_page_path(raw_path) + except InvalidPagePath as exc: + wiki_db.rollback() + raise HTTPException(status_code=400, detail=str(exc)) from exc + if remove_page(wiki_db, generation_id=generation.id, path=normalized): + removed.append(normalized) + else: + logger.info( + "[wiki] removal of '%s' from generation %s matched no page", + normalized, + generation.id, + ) + return removed + + def _finish_code_wiki( + self, + wiki_db: Session, + generation: WikiGeneration, + summary: Optional[WikiContentSummary], + ) -> None: + """Conclude a code wiki run once its final write is safely committed. + + Failures here are reported to the agent as a 5xx rather than swallowed. The + version is committed either way, so a retried submission republishes it; a + silent failure would leave a run stuck RUNNING with content nobody projects. + """ + succeeded = summary is not None and summary.status == "COMPLETED" + try: + finish_run( + wiki_db, + generation=generation, + succeeded=succeeded, + error_message=(summary.error_message if summary else "") or "", + head_commit=(summary.head_commit if summary else "") or "", + ) + except Exception as exc: + wiki_db.rollback() + logger.error( + "[wiki] failed to conclude code wiki generation %s: %s", + generation.id, + exc, + ) + raise HTTPException( + status_code=500, + detail=f"Failed to publish the wiki version: {exc}", + ) from exc + def get_generations( self, db: Session, diff --git a/backend/app/tasks/knowledge_tasks.py b/backend/app/tasks/knowledge_tasks.py index ae8ad46c4f..4edd5dd1be 100644 --- a/backend/app/tasks/knowledge_tasks.py +++ b/backend/app/tasks/knowledge_tasks.py @@ -537,3 +537,19 @@ def scan_stale_index_tasks(): f"[StaleScanner] Scan complete, marked {marked_count} documents as FAILED" ) return {"marked_count": marked_count, "scanned_count": len(active_docs)} + + +@celery_app.task(name="app.tasks.knowledge_tasks.sweep_code_wiki_index_cleanup") +def sweep_code_wiki_index_cleanup(): + """Finish index deletions a code wiki publish could not complete. + + Publishing removes a deleted page's row inside a transaction and its chunks + outside one, so a vector store failure leaves the chunks behind. The publish is + allowed to succeed anyway and parks what it still owes; this is the sweep that + settles it. Without it, retrieval keeps answering from pages that no longer + exist and citing document ids that no longer resolve. + """ + from app.services.knowledge.code_wiki_runner import sweep_pending_index_cleanup + + with SessionLocal() as db: + return sweep_pending_index_cleanup(db) diff --git a/backend/init_data/02-public-resources.yaml b/backend/init_data/02-public-resources.yaml index eff96d9635..a4f6ed109c 100644 --- a/backend/init_data/02-public-resources.yaml +++ b/backend/init_data/02-public-resources.yaml @@ -328,6 +328,110 @@ status: --- apiVersion: agent.wecode.io/v1 kind: Ghost +metadata: + name: code-wiki-ghost + namespace: default +spec: + mcpServers: {} + skills: + - wiki_submit + systemPrompt: | + You write and maintain code wikis for a knowledge base. Each run is given a + repository, a generation id and a set of instructions; this prompt covers only how + to hand your work back, which is the same every time. + + ## Submitting + + Use the wiki_submit skill (wiki_submit.js). Submit each page as you finish it + rather than batching them at the end: a run that stops early then still leaves + behind the pages it completed. + + ```bash + node wiki_submit.js submit \ + --generation-id \ + --path architecture/backend \ + --title "Backend Architecture" \ + --file ./page.md + ``` + + ## Paths are identities + + `--path` is what identifies a page: `index`, `architecture/backend`, + `modules/indexing`. Lowercase, `/`-separated, no file extension, at most 4 folders + deep, and two paths may not differ only by case. + + Keep a page's path stable across runs. It is what lets an unchanged page keep its + place, its links and its search index. Changing it republishes the page as a + deletion plus an insertion, so reword titles freely and move paths rarely. + + Send a page's complete content every time. There is no patch format; what you send + replaces the page. + + ## Removing + + Only an incremental run removes pages by declaring them, because its version starts + as a copy of the published wiki: + + ```bash + node wiki_submit.js remove --generation-id --path modules/legacy-sync + ``` + + In a full rebuild the version starts empty, so a page you do not write is already + absent and declaring it would do nothing. + + ## Finishing + + Always finish the run, in success or failure. Nothing is published until you do, + and a run left unreported blocks the wiki until it is reclaimed hours later. + + ```bash + node wiki_submit.js complete --generation-id --head-commit "$(git rev-parse HEAD)" + node wiki_submit.js fail --generation-id --error-message "why it failed" + ``` + + Report the commit you actually documented. The next run compares against it to + decide what has changed, so a missing or wrong value costs a needless full rebuild + or, worse, skips changes nobody has written up. +status: + state: Available +--- +apiVersion: agent.wecode.io/v1 +kind: Bot +metadata: + name: code-wiki-bot + namespace: default +spec: + ghostRef: + name: code-wiki-ghost + namespace: default + shellRef: + name: ClaudeCode + namespace: default +status: + state: Available +--- +apiVersion: agent.wecode.io/v1 +kind: Team +metadata: + name: code-wiki-team + namespace: default +spec: + description: Generates and maintains a knowledge base code wiki from a repository. + members: + - role: leader + botRef: + name: code-wiki-bot + namespace: default + prompt: "" + collaborationModel: solo + bind_mode: [] + workflow: + mode: solo +status: + state: Available +--- +apiVersion: agent.wecode.io/v1 +kind: Ghost metadata: name: chat-ghost namespace: default diff --git a/backend/init_data/skills/wiki_submit/SKILL.md b/backend/init_data/skills/wiki_submit/SKILL.md index bc4ead8119..26258a0614 100644 --- a/backend/init_data/skills/wiki_submit/SKILL.md +++ b/backend/init_data/skills/wiki_submit/SKILL.md @@ -1,6 +1,6 @@ --- -description: "Submit wiki documentation sections to Wegent backend API. Simplifies the HTTP POST process for wiki content submission." -version: "1.1.0" +description: "Submit wiki documentation pages to Wegent backend API. Simplifies the HTTP POST process for wiki content submission." +version: "1.2.0" author: "Wegent Team" tags: ["wiki", "documentation", "api", "submission"] bindShells: ["ClaudeCode"] @@ -8,37 +8,68 @@ bindShells: ["ClaudeCode"] # Wiki Submit Skill -This skill provides a simple command-line tool to submit wiki documentation sections to the Wegent backend. +This skill provides a simple command-line tool to submit wiki documentation pages to the Wegent backend. -## Usage +## Page paths + +A page is identified by its `--path`: `index`, `architecture/backend`, `modules/indexing`. +Lowercase, `/`-separated, no file extension, at most 4 folders deep. Two paths may not +differ only by case — the projection matches them case-insensitively, so they would +collapse into one page. +**Keep a path stable across runs.** It is what lets an unchanged page keep its place, +its links and its search index. Changing it republishes the page as a deletion plus an +insertion, so reword titles freely and move paths rarely. -### Submit a single section from a markdown file +Send a page's **complete content** every time. There is no patch format; what you send +replaces the page. + +## Usage + +### Submit a page from a markdown file ```bash node wiki_submit.js submit \ --generation-id 123 \ - --type overview \ - --title "Project Overview" \ - --file /path/to/overview.md + --path architecture/backend \ + --title "Backend Architecture" \ + --file /path/to/page.md ``` -### Submit section content directly +### Submit page content directly + +Note the `$'...'` quoting: in a plain double-quoted string `\n` stays a backslash and +an `n`, and the page arrives as one long line. For anything beyond a few lines, write +the markdown to a file and use `--file`. ```bash node wiki_submit.js submit \ --generation-id 123 \ - --type architecture \ - --title "System Architecture" \ - --content "# Architecture\n\nYour markdown content here..." + --path index \ + --title "Overview" \ + --content $'# Overview\n\nYour markdown content here...' +``` + +### Remove pages that no longer have a subject + +Only meaningful in an incremental run, where your version starts as a copy of the +published wiki and not writing a page therefore does *not* remove it. + +```bash +node wiki_submit.js remove \ + --generation-id 123 \ + --path modules/legacy-sync \ + --path guides/old-setup ``` ### Complete the wiki generation +Report the commit you documented, so the next run knows what has already been covered. + ```bash node wiki_submit.js complete \ --generation-id 123 \ - --structure-order "overview: Project Overview" "architecture: System Architecture" "module: Core Modules" + --head-commit "$(git rev-parse HEAD)" ``` ### Mark generation as failed @@ -49,14 +80,12 @@ node wiki_submit.js fail \ --error-message "Failed to analyze repository structure" ``` -## Section Types +## Section types + +`--type` defaults to `chapter` and can be left out. The legacy wiki used it to group +pages; a code wiki organises them by path instead. -- `overview`: Project overview and objectives -- `architecture`: System architecture and design -- `module`: Module documentation -- `api`: API documentation -- `guide`: User guides and tutorials -- `deep`: In-depth technical analysis +Accepted values: `overview`, `architecture`, `module`, `api`, `guide`, `deep`, `chapter`. ## Authentication diff --git a/backend/init_data/skills/wiki_submit/wiki_submit.js b/backend/init_data/skills/wiki_submit/wiki_submit.js index 5b4c96492a..dc0e84c7a7 100644 --- a/backend/init_data/skills/wiki_submit/wiki_submit.js +++ b/backend/init_data/skills/wiki_submit/wiki_submit.js @@ -157,7 +157,7 @@ function makeRequest(url, options, body) { * @param {object|null} summary - Optional summary for completion * @returns {Promise} */ -async function submitSections(endpoint, token, generationId, sections, summary = null) { +async function submitSections(endpoint, token, generationId, sections, summary = null, removedPaths = []) { const headers = { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', @@ -172,6 +172,10 @@ async function submitSections(endpoint, token, generationId, sections, summary = payload.summary = summary } + if (removedPaths && removedPaths.length > 0) { + payload.removed_paths = removedPaths + } + return makeRequest(endpoint, { method: 'POST', headers }, JSON.stringify(payload)) } @@ -210,11 +214,17 @@ async function cmdSubmit(args) { } const section = { - type: args.type, + // Defaults to a plain chapter: a code wiki identifies pages by path, so the + // section type carries no meaning there and asking for one every time is noise. + type: args.type || 'chapter', title: args.title, content: content, } + if (args.path) { + section.path = args.path + } + if (args.ext) { try { section.ext = JSON.parse(args.ext) @@ -231,7 +241,40 @@ async function cmdSubmit(args) { return 1 } - console.log(`✅ Section '${args.title}' submitted successfully`) + console.log(`✅ Page '${args.path || args.title}' submitted successfully`) + return 0 +} + +/** + * Handle remove command: declare pages as gone. + * @param {object} args - Command arguments + * @returns {Promise} + */ +async function cmdRemove(args) { + const endpoint = getWikiEndpoint(args.endpoint) + const token = getAuthToken(args.token) + if (!token) { + console.error('Error: Authorization token is required. It can be obtained from TASK_INFO, WIKI_TOKEN env var, or --token argument.') + process.exit(1) + } + if (!args.generationId) { + console.error('Error: --generation-id is required.') + process.exit(1) + } + if (!args.paths.length) { + console.error('Error: --path is required for remove command') + return 1 + } + const generationId = parseInt(args.generationId, 10) + + const result = await submitSections(endpoint, token, generationId, [], null, args.paths) + + if (result.status === 'error') { + console.error(`❌ Error: ${result.message}`) + return 1 + } + + console.log(`✅ Removed ${args.paths.length} page(s): ${args.paths.join(', ')}`) return 0 } @@ -258,6 +301,9 @@ async function cmdComplete(args) { structure_order: args.structureOrder || [], } + if (args.headCommit) { + summary.head_commit = args.headCommit + } if (args.model) { summary.model = args.model } @@ -323,6 +369,8 @@ function parseArgs(argv) { generationId: null, type: null, title: null, + path: null, + paths: [], file: null, content: null, ext: null, @@ -330,6 +378,7 @@ function parseArgs(argv) { model: null, tokensUsed: null, errorMessage: null, + headCommit: null, } let i = 2 // Skip 'node' and script name @@ -359,6 +408,13 @@ function parseArgs(argv) { case '--title': args.title = argv[++i] break + case '--path': + args.path = argv[++i] + args.paths.push(args.path) + break + case '--head-commit': + args.headCommit = argv[++i] + break case '--file': case '-f': args.file = argv[++i] @@ -413,7 +469,8 @@ Wiki Submit Skill - Submit wiki documentation to Wegent backend Usage: node wiki_submit.js [options] Commands: - submit Submit a wiki section + submit Submit a wiki page + remove Declare wiki pages as gone complete Mark wiki generation as completed fail Mark wiki generation as failed @@ -428,13 +485,19 @@ environment variable when running inside an executor container. You don't need to specify it manually in most cases. Submit Options: - --type Section type (overview|architecture|module|api|guide|deep) - --title Section title - --file, -f Path to markdown file containing section content - --content, -c Section content (alternative to --file) + --path Stable page path, e.g. "architecture/backend". This is the + page's identity: keep it the same across runs. + --title Page title (required) + --type Section type, defaults to "chapter" + --file, -f Path to markdown file containing page content + --content, -c Page content (alternative to --file) --ext Extension data as JSON string +Remove Options: + --path Page path to remove. Repeat for several pages. + Complete Options: + --head-commit Commit that was documented, from \`git rev-parse HEAD\` --structure-order Ordered list of section identifiers --model Model name used for generation --tokens-used Number of tokens used @@ -443,8 +506,9 @@ Fail Options: --error-message, -m Error message describing the failure Examples: - node wiki_submit.js submit --generation-id 123 --type overview --title "Project Overview" --file ./overview.md - node wiki_submit.js complete --generation-id 123 --structure-order "overview: Project Overview" "architecture: System Architecture" + node wiki_submit.js submit --generation-id 123 --path architecture/backend --title "Backend Architecture" --file ./page.md + node wiki_submit.js remove --generation-id 123 --path modules/legacy-sync + node wiki_submit.js complete --generation-id 123 --head-commit $(git rev-parse HEAD) node wiki_submit.js fail --generation-id 123 --error-message "Failed to analyze repository" `) } @@ -463,8 +527,11 @@ async function main() { let exitCode switch (args.command) { case 'submit': - if (!args.type) { - console.error('Error: --type is required for submit command') + if (!args.path) { + // The path is the page's identity. Without one the write is accepted and the + // page is then skipped at publish time, so it would report success and + // silently produce nothing. + console.error('Error: --path is required for submit command') process.exit(1) } if (!args.title) { @@ -473,6 +540,9 @@ async function main() { } exitCode = await cmdSubmit(args) break + case 'remove': + exitCode = await cmdRemove(args) + break case 'complete': exitCode = await cmdComplete(args) break diff --git a/backend/tests/api/test_knowledge_code_wiki.py b/backend/tests/api/test_knowledge_code_wiki.py new file mode 100644 index 0000000000..3b763dd067 --- /dev/null +++ b/backend/tests/api/test_knowledge_code_wiki.py @@ -0,0 +1,241 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""API tests for creating a code wiki.""" + +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session, sessionmaker + +from app.core.security import create_access_token +from app.models.user import User +from app.schemas.knowledge import KnowledgeBaseType + +CREATE_URL = "/api/knowledge-bases/code-wikis" + +PAYLOAD = { + "name": "Wegent Wiki", + "namespace": "default", + "source_type": "github", + "source_url": "https://github.com/wecode-ai/Wegent.git", +} + + +@pytest.fixture +def auth_headers(test_user: User) -> dict[str, str]: + token = create_access_token(data={"sub": test_user.user_name}) + return {"Authorization": f"Bearer {token}"} + + +@pytest.fixture +def kind_services_use_test_db(test_db: Session, monkeypatch: pytest.MonkeyPatch): + """Point ``KindBaseService``'s own session at the test database. + + Creating a knowledge base resolves a default embedding model, which reaches + ``KindBaseService.list_resources``. That opens its own ``SessionLocal`` instead of + using the request's session, so the FastAPI dependency override does not reach it + and it connects to whatever database is configured. On a developer machine with + MySQL running the test then passes for the wrong reason; in CI there is no MySQL + and it fails. + """ + factory = sessionmaker( + autocommit=False, + autoflush=False, + bind=test_db.get_bind(), + expire_on_commit=False, + ) + monkeypatch.setattr("app.services.kind_base.SessionLocal", factory) + + +def test_creating_a_code_wiki_records_its_type_and_source( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + response = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + + assert response.status_code == 201, response.text + body = response.json() + assert body["kb_type"] == KnowledgeBaseType.CODE_WIKI.value + assert body["name"] == "Wegent Wiki" + + +def test_creation_is_refused_without_repository_access( + test_client: TestClient, auth_headers: dict[str, str] +): + """A wiki must not be built for a repository the requester cannot read.""" + from app.services.knowledge.code_wiki_source import SourceAccessDenied + + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + side_effect=SourceAccessDenied("You do not have read access to 'x/y'."), + ): + response = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + + assert response.status_code == 403 + assert "read access" in response.json()["detail"] + + +def test_creation_requires_authentication(test_client: TestClient): + response = test_client.post(CREATE_URL, json=PAYLOAD) + + assert response.status_code in (401, 403) + + +def test_unsupported_source_type_is_rejected_by_validation( + test_client: TestClient, auth_headers: dict[str, str] +): + payload = {**PAYLOAD, "source_type": "svn"} + + response = test_client.post(CREATE_URL, json=payload, headers=auth_headers) + + assert response.status_code == 422 + + +def test_the_general_endpoint_refuses_to_create_a_code_wiki( + test_client: TestClient, auth_headers: dict[str, str] +): + """Only the code wiki endpoint may create one, because only it checks the repo.""" + response = test_client.post( + "/api/knowledge-bases", + json={"name": "sneaky", "kb_type": "code_wiki"}, + headers=auth_headers, + ) + + assert response.status_code == 400 + assert "code-wikis" in response.json()["detail"] + + +def test_ordinary_knowledge_bases_are_still_created_as_notebooks( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + response = test_client.post( + "/api/knowledge-bases", json={"name": "plain notes"}, headers=auth_headers + ) + + assert response.status_code == 201, response.text + assert response.json()["kb_type"] == KnowledgeBaseType.NOTEBOOK.value + + +# --- triggering a run ------------------------------------------------------- + + +def _create_wiki(test_client: TestClient, auth_headers: dict[str, str]) -> int: + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + response = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + assert response.status_code == 201, response.text + return response.json()["id"] + + +def _run_url(knowledge_base_id: int) -> str: + return f"/api/knowledge-bases/{knowledge_base_id}/code-wiki/generations" + + +def test_a_run_can_be_triggered_without_waiting_for_a_schedule( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + kb_id = _create_wiki(test_client, auth_headers) + + with patch("app.api.endpoints.knowledge.start_run") as start: + start.return_value.started = True + start.return_value.mode = "full" + start.return_value.reason = "first run for this repository" + start.return_value.generation.id = 7 + start.return_value.task_id = 42 + + response = test_client.post( + _run_url(kb_id), json={"head_commit": "abc1234"}, headers=auth_headers + ) + + assert response.status_code == 202, response.text + body = response.json() + assert body["started"] is True + assert body["generation_id"] == 7 + assert body["task_id"] == 42 + + +def test_a_run_that_was_not_needed_is_a_success_not_a_failure( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """ "Nothing changed" is the answer the caller asked for, not an error.""" + kb_id = _create_wiki(test_client, auth_headers) + + with patch("app.api.endpoints.knowledge.start_run") as start: + start.return_value.started = False + start.return_value.mode = "skip" + start.return_value.reason = "repository unchanged since last run" + start.return_value.generation = None + start.return_value.task_id = 0 + + response = test_client.post(_run_url(kb_id), json={}, headers=auth_headers) + + assert response.status_code == 202, response.text + body = response.json() + assert body["started"] is False + assert body["mode"] == "skip" + assert body["generation_id"] == 0 + + +def test_a_second_run_while_one_is_live_is_a_conflict( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + from app.services.knowledge.code_wiki_generation import GenerationInFlight + + kb_id = _create_wiki(test_client, auth_headers) + + with patch( + "app.api.endpoints.knowledge.start_run", + side_effect=GenerationInFlight("generation 3 is already running"), + ): + response = test_client.post(_run_url(kb_id), json={}, headers=auth_headers) + + assert response.status_code == 409 + assert "already running" in response.json()["detail"] + + +def test_a_knowledge_base_that_is_not_a_code_wiki_cannot_be_generated( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + response = test_client.post( + "/api/knowledge-bases", json={"name": "plain notes"}, headers=auth_headers + ) + notebook_id = response.json()["id"] + + response = test_client.post(_run_url(notebook_id), json={}, headers=auth_headers) + + assert response.status_code == 400 + assert "not a code wiki" in response.json()["detail"] + + +def test_a_missing_knowledge_base_is_not_found( + test_client: TestClient, auth_headers: dict[str, str] +): + response = test_client.post(_run_url(999999), json={}, headers=auth_headers) + + assert response.status_code == 404 + + +def test_triggering_a_run_requires_authentication(test_client: TestClient): + response = test_client.post(_run_url(1), json={}) + + assert response.status_code in (401, 403) diff --git a/backend/tests/repository/test_repository_state_reads.py b/backend/tests/repository/test_repository_state_reads.py new file mode 100644 index 0000000000..505969cc41 --- /dev/null +++ b/backend/tests/repository/test_repository_state_reads.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the two repository reads a code wiki run depends on. + +Each platform words its compare API differently, and each has a way of telling you +less than you asked for: GitHub caps the file list, GitLab gives up on a large diff, +and an older self-hosted Gitea has no compare endpoint at all. All three have to +surface as ``None`` — the caller reads that as "the extent of the change is unknown" +and rebuilds — because an incomplete diff mistaken for a complete one picks an +incremental run for a change that reshaped the repository. +""" + +from unittest.mock import Mock, patch + +from app.repository.gitea_provider import GiteaProvider +from app.repository.github_provider import GITHUB_COMPARE_FILE_LIMIT, GitHubProvider +from app.repository.gitlab_provider import GitLabProvider + +DOMAIN_ARGS = dict(token="t0ken", repo_name="wecode-ai/Wegent") + + +def _response(payload: dict, status_code: int = 200) -> Mock: + response = Mock() + response.status_code = status_code + response.json.return_value = payload + response.raise_for_status.return_value = None + return response + + +# --- GitHub ----------------------------------------------------------------- + + +def test_github_reads_the_default_branch_and_its_commit(): + provider = GitHubProvider() + responses = [ + _response({"default_branch": "develop"}), + _response({"commit": {"sha": "abc123"}}), + ] + + with patch("app.repository.github_provider.requests.get", side_effect=responses): + result = provider.get_default_branch_head( + git_domain="github.com", **DOMAIN_ARGS + ) + + assert result == {"branch": "develop", "commit": "abc123"} + + +def test_github_maps_compare_statuses_to_name_status_letters(): + """The run-mode rules are written against git's letters, not GitHub's words.""" + provider = GitHubProvider() + payload = { + "files": [ + {"filename": "a.py", "status": "added"}, + {"filename": "b.py", "status": "modified"}, + {"filename": "c.py", "status": "removed"}, + {"filename": "d.py", "status": "renamed"}, + ] + } + + with patch( + "app.repository.github_provider.requests.get", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="github.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed == [ + {"path": "a.py", "status": "A"}, + {"path": "b.py", "status": "M"}, + {"path": "c.py", "status": "D"}, + {"path": "d.py", "status": "R"}, + ] + + +def test_github_reports_a_truncated_compare_as_unknown(): + """GitHub caps the list at 300 and does not say so; a capped list read as + complete would understate a change big enough to need a rebuild.""" + provider = GitHubProvider() + payload = { + "files": [ + {"filename": f"file{index}.py", "status": "modified"} + for index in range(GITHUB_COMPARE_FILE_LIMIT) + ] + } + + with patch( + "app.repository.github_provider.requests.get", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="github.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed is None + + +def test_github_reports_a_compare_below_the_cap_normally(): + provider = GitHubProvider() + payload = { + "files": [ + {"filename": f"file{index}.py", "status": "modified"} + for index in range(GITHUB_COMPARE_FILE_LIMIT - 1) + ] + } + + with patch( + "app.repository.github_provider.requests.get", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="github.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed is not None + assert len(changed) == GITHUB_COMPARE_FILE_LIMIT - 1 + + +# --- GitLab ----------------------------------------------------------------- + + +def test_gitlab_reads_the_default_branch_and_its_commit(): + provider = GitLabProvider() + responses = [ + _response({"default_branch": "main"}), + _response({"commit": {"id": "def456"}}), + ] + + with patch.object( + GitLabProvider, "_make_request_with_auth_retry", side_effect=responses + ): + result = provider.get_default_branch_head( + git_domain="gitlab.com", **DOMAIN_ARGS + ) + + assert result == {"branch": "main", "commit": "def456"} + + +def test_gitlab_derives_status_from_the_diff_flags(): + provider = GitLabProvider() + payload = { + "diffs": [ + {"new_path": "a.py", "new_file": True}, + {"new_path": "b.py"}, + {"old_path": "c.py", "deleted_file": True}, + {"new_path": "d.py", "renamed_file": True}, + ] + } + + with patch.object( + GitLabProvider, "_make_request_with_auth_retry", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="gitlab.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed == [ + {"path": "a.py", "status": "A"}, + {"path": "b.py", "status": "M"}, + {"path": "c.py", "status": "D"}, + {"path": "d.py", "status": "R"}, + ] + + +def test_gitlab_reports_a_timed_out_compare_as_unknown(): + """GitLab answers 200 with a partial diff and a flag, so the flag is the only + thing separating "small change" from "gave up".""" + provider = GitLabProvider() + payload = { + "compare_timeout": True, + "diffs": [{"new_path": "a.py"}], + } + + with patch.object( + GitLabProvider, "_make_request_with_auth_retry", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="gitlab.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed is None + + +# --- Gitea ------------------------------------------------------------------ + + +def test_gitea_reads_the_default_branch_and_its_commit(): + provider = GiteaProvider() + responses = [ + _response({"default_branch": "master"}), + _response({"commit": {"id": "789abc"}}), + ] + + with patch("app.repository.gitea_provider.requests.get", side_effect=responses): + result = provider.get_default_branch_head(git_domain="gitea.com", **DOMAIN_ARGS) + + assert result == {"branch": "master", "commit": "789abc"} + + +def test_gitea_maps_compare_statuses_to_name_status_letters(): + provider = GiteaProvider() + payload = { + "files": [ + {"filename": "a.py", "status": "added"}, + {"filename": "b.py", "status": "changed"}, + {"filename": "c.py", "status": "deleted"}, + ] + } + + with patch( + "app.repository.gitea_provider.requests.get", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="gitea.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed == [ + {"path": "a.py", "status": "A"}, + {"path": "b.py", "status": "M"}, + {"path": "c.py", "status": "D"}, + ] + + +def test_an_older_gitea_without_a_compare_endpoint_reports_unknown(): + """Self-hosted instances lag; a 404 here must degrade, not raise.""" + provider = GiteaProvider() + + with patch( + "app.repository.gitea_provider.requests.get", + return_value=_response({}, status_code=404), + ): + changed = provider.get_changed_files( + git_domain="gitea.example.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed is None diff --git a/backend/tests/services/knowledge/test_code_wiki_cleanup_sweep.py b/backend/tests/services/knowledge/test_code_wiki_cleanup_sweep.py new file mode 100644 index 0000000000..86939e30f5 --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_cleanup_sweep.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for draining the index deletions a publish could not finish. + +A page's row is deleted inside the publish transaction and its chunks outside one, +because the vector store cannot join a database transaction. The publish is allowed +to succeed when the second half fails — refusing a good version over an external +hiccup would be worse — and parks what it still owes. + +Nothing else settles that debt. It cannot self-heal by deleting the page again, +because the page is already gone. So these tests are about the sweep finding the +debt, clearing what it can, keeping what it cannot, and not letting one broken +knowledge base stop the others. +""" + +from dataclasses import dataclass, field + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.user import User +from app.services.knowledge.code_wiki_projection import ( + PENDING_INDEX_CLEANUP_KEY, + ProjectionSideEffects, +) +from app.services.knowledge.code_wiki_runner import sweep_pending_index_cleanup + + +@dataclass +class FakeIndex: + """Stands in for the vector store.""" + + deleted: list[int] = field(default_factory=list) + refuses: set[int] = field(default_factory=set) + explodes_for_kb: set[int] = field(default_factory=set) + + def effects_for(self, kind_id: int) -> ProjectionSideEffects: + def delete_rag_document(document_id: int) -> None: + if kind_id in self.explodes_for_kb: + raise RuntimeError("vector store unreachable") + if document_id in self.refuses: + raise RuntimeError(f"index deletion refused for {document_id}") + self.deleted.append(document_id) + + return ProjectionSideEffects( + write_attachment=lambda **_: 0, + delete_attachment=lambda _: None, + delete_rag_document=delete_rag_document, + enqueue_reindex=lambda _: None, + ) + + +@pytest.fixture +def index(monkeypatch) -> FakeIndex: + from app.services.knowledge import code_wiki_runner + + fake = FakeIndex() + monkeypatch.setattr( + code_wiki_runner, + "build_projection_side_effects", + lambda db, *, knowledge_base, user: fake.effects_for(knowledge_base.id), + ) + return fake + + +def _knowledge_base( + test_db: Session, user: User, name: str, pending: list[str] | None = None +) -> Kind: + spec = {"name": name, "kbType": "code_wiki"} + if pending is not None: + spec[PENDING_INDEX_CLEANUP_KEY] = pending + kind = Kind( + kind="KnowledgeBase", + name=name, + namespace="default", + user_id=user.id, + json={"spec": spec}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def _pending(test_db: Session, knowledge_base: Kind) -> list[str]: + test_db.refresh(knowledge_base) + return (knowledge_base.json or {}).get("spec", {}).get( + PENDING_INDEX_CLEANUP_KEY + ) or [] + + +def test_owed_deletions_are_finished( + test_db: Session, test_user: User, index: FakeIndex +): + kb = _knowledge_base(test_db, test_user, "owes", pending=["11", "12"]) + + result = sweep_pending_index_cleanup(test_db) + + assert sorted(index.deleted) == [11, 12] + assert _pending(test_db, kb) == [] + assert result["drained"] == 1 + + +def test_a_deletion_that_still_fails_stays_owed( + test_db: Session, test_user: User, index: FakeIndex +): + """Giving up would make the orphaned chunks permanent, which is the state this + sweep exists to end.""" + index.refuses = {12} + kb = _knowledge_base(test_db, test_user, "partly", pending=["11", "12"]) + + result = sweep_pending_index_cleanup(test_db) + + assert index.deleted == [11] + assert _pending(test_db, kb) == ["12"] + assert result["outstanding"] == 1 + assert result["drained"] == 0 + + +def test_a_failing_ref_is_tried_again_on_the_next_sweep( + test_db: Session, test_user: User, index: FakeIndex +): + index.refuses = {12} + kb = _knowledge_base(test_db, test_user, "recovers", pending=["12"]) + sweep_pending_index_cleanup(test_db) + + index.refuses = set() + sweep_pending_index_cleanup(test_db) + + assert index.deleted == [12] + assert _pending(test_db, kb) == [] + + +def test_knowledge_bases_owing_nothing_are_not_touched( + test_db: Session, test_user: User, index: FakeIndex +): + _knowledge_base(test_db, test_user, "never-parked") + _knowledge_base(test_db, test_user, "already-drained", pending=[]) + + result = sweep_pending_index_cleanup(test_db) + + assert result == {"swept": 0, "drained": 0, "outstanding": 0} + assert index.deleted == [] + + +def test_one_broken_knowledge_base_does_not_stop_the_others( + test_db: Session, test_user: User, index: FakeIndex +): + """They are independent, and the whole point is to keep draining.""" + broken = _knowledge_base(test_db, test_user, "broken", pending=["21"]) + index.explodes_for_kb = {broken.id} + healthy = _knowledge_base(test_db, test_user, "healthy", pending=["22"]) + + sweep_pending_index_cleanup(test_db) + + assert index.deleted == [22] + assert _pending(test_db, healthy) == [] + + +def test_a_knowledge_base_whose_owner_is_gone_is_skipped_not_fatal( + test_db: Session, test_user: User, index: FakeIndex +): + orphaned = _knowledge_base(test_db, test_user, "orphaned", pending=["31"]) + orphaned.user_id = 999999 + healthy = _knowledge_base(test_db, test_user, "healthy", pending=["32"]) + test_db.flush() + + result = sweep_pending_index_cleanup(test_db) + + assert index.deleted == [32] + assert result["swept"] == 1 + assert _pending(test_db, healthy) == [] + + +def test_a_ref_that_can_never_be_deleted_is_dropped_not_retried_forever( + test_db: Session, test_user: User, index: FakeIndex +): + """It is not a document id, so no sweep can ever settle it. Kept, it would fail + on every future sweep and the list would never drain.""" + kb = _knowledge_base(test_db, test_user, "unusable", pending=["not-an-id", "41"]) + + sweep_pending_index_cleanup(test_db) + + assert index.deleted == [41] + assert _pending(test_db, kb) == [] + + +def test_a_ref_parked_twice_is_only_recorded_once( + test_db: Session, test_user: User, index: FakeIndex +): + """The retry writes refs back as strings, so parking an int afterwards would + miss the membership test and accumulate a duplicate.""" + from app.services.knowledge.code_wiki_publisher import _park_unfinished_cleanup + + kb = _knowledge_base(test_db, test_user, "dupes", pending=["51"]) + + _park_unfinished_cleanup(test_db, kb, [51, "51"]) + + assert _pending(test_db, kb) == ["51"] diff --git a/backend/tests/services/knowledge/test_code_wiki_content_write.py b/backend/tests/services/knowledge/test_code_wiki_content_write.py new file mode 100644 index 0000000000..81a96ed20b --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_content_write.py @@ -0,0 +1,289 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for writing pages into a version by path. + +Matching on path rather than title is what keeps a page's document id — and with it +its RAG index entry and any stored citation — stable when the agent rewords a heading. +""" + +from datetime import datetime + +import pytest +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from app.models.wiki import ( + WikiContent, + WikiGeneration, + WikiGenerationStatus, + WikiGenerationType, +) +from app.schemas.wiki import WikiContentSection, WikiContentWriteRequest +from app.services.knowledge.code_wiki_version_store import page_path_of +from app.services.wiki_service import WikiService + +KIND_ID = 91 + + +@pytest.fixture +def generation(test_db: Session) -> WikiGeneration: + record = WikiGeneration( + project_id=1, + kind_id=KIND_ID, + user_id=1, + task_id=0, + team_id=1, + generation_type=WikiGenerationType.INCREMENTAL, + source_snapshot={}, + status=WikiGenerationStatus.RUNNING, + # The column defaults to a string literal, which SQLite refuses. + completed_at=datetime(1970, 1, 1), + ) + test_db.add(record) + test_db.flush() + return record + + +def _write(db: Session, generation: WikiGeneration, *sections: WikiContentSection): + WikiService().save_generation_contents( + db, + WikiContentWriteRequest(generation_id=generation.id, sections=list(sections)), + ) + + +def _section(path: str | None, title: str, content: str = "body") -> WikiContentSection: + return WikiContentSection(type="chapter", title=title, content=content, path=path) + + +def _pages(db: Session, generation_id: int) -> list[WikiContent]: + return ( + db.query(WikiContent).filter(WikiContent.generation_id == generation_id).all() + ) + + +def test_a_page_is_written_with_its_path(test_db: Session, generation: WikiGeneration): + _write(test_db, generation, _section("architecture/backend", "Backend")) + + (page,) = _pages(test_db, generation.id) + assert page_path_of(page) == "architecture/backend" + assert page.title == "Backend" + + +def test_rewording_a_title_revises_the_same_page( + test_db: Session, generation: WikiGeneration +): + """The point of path identity: no delete-and-recreate, so the id survives.""" + _write(test_db, generation, _section("architecture/backend", "Backend", "v1")) + (original,) = _pages(test_db, generation.id) + + _write( + test_db, + generation, + _section("architecture/backend", "The Backend Service", "v2"), + ) + + pages = _pages(test_db, generation.id) + assert len(pages) == 1 + assert pages[0].id == original.id + assert pages[0].title == "The Backend Service" + assert pages[0].content == "v2" + + +def test_moving_a_page_to_a_new_path_creates_a_new_page( + test_db: Session, generation: WikiGeneration +): + _write(test_db, generation, _section("architecture/backend", "Backend")) + + _write(test_db, generation, _section("services/backend", "Backend")) + + assert {page_path_of(page) for page in _pages(test_db, generation.id)} == { + "architecture/backend", + "services/backend", + } + + +def test_the_path_survives_a_rewrite_that_supplies_no_ext( + test_db: Session, generation: WikiGeneration +): + """``ext`` is replaced wholesale on update, which previously dropped the path.""" + _write(test_db, generation, _section("architecture/backend", "Backend", "v1")) + + _write(test_db, generation, _section("architecture/backend", "Backend", "v2")) + + (page,) = _pages(test_db, generation.id) + assert page_path_of(page) == "architecture/backend" + + +def test_a_path_is_normalized_before_it_is_stored( + test_db: Session, generation: WikiGeneration +): + _write(test_db, generation, _section(" architecture//backend.md ", "Backend")) + + (page,) = _pages(test_db, generation.id) + assert page_path_of(page) == "architecture/backend" + + +def test_a_malformed_path_fails_the_write_not_the_publish( + test_db: Session, generation: WikiGeneration +): + with pytest.raises(HTTPException) as exc: + _write(test_db, generation, _section("../escape", "Escape")) + + assert exc.value.status_code == 400 + + +def test_two_paths_colliding_by_case_are_refused( + test_db: Session, generation: WikiGeneration +): + """The knowledge tables collate case-insensitively; both cannot be honoured.""" + with pytest.raises(HTTPException) as exc: + _write( + test_db, + generation, + _section("architecture/backend", "One"), + _section("Architecture/Backend", "Two"), + ) + + assert exc.value.status_code == 400 + assert "case" in exc.value.detail + + +def test_a_rejected_write_leaves_the_version_unchanged( + test_db: Session, generation: WikiGeneration +): + _write(test_db, generation, _section("index", "Index")) + + with pytest.raises(HTTPException): + _write(test_db, generation, _section("bad\\path", "Bad")) + + assert {page_path_of(page) for page in _pages(test_db, generation.id)} == {"index"} + + +def test_writes_without_a_path_still_match_on_title( + test_db: Session, generation: WikiGeneration +): + """The legacy write path predates page identity and must keep working.""" + _write(test_db, generation, _section(None, "Overview", "v1")) + + _write(test_db, generation, _section(None, "Overview", "v2")) + + (page,) = _pages(test_db, generation.id) + assert page.content == "v2" + + +def test_a_legacy_write_never_resolves_to_a_path_identified_page( + test_db: Session, generation: WikiGeneration +): + """Moving a page keeps its title, so a title can name several pages. + + If the title fallback could reach them, which one a path-less write overwrote + would depend on query order rather than on any rule. + """ + _write(test_db, generation, _section("architecture/backend", "Backend", "kept")) + _write(test_db, generation, _section("services/backend", "Backend", "kept too")) + + _write(test_db, generation, _section(None, "Backend", "legacy")) + + by_path = { + page_path_of(page): page.content for page in _pages(test_db, generation.id) + } + assert by_path["architecture/backend"] == "kept" + assert by_path["services/backend"] == "kept too" + # The legacy write created its own path-less entry instead of hijacking one. + assert by_path[""] == "legacy" + + +# --- removals --------------------------------------------------------------- + + +def _remove(db: Session, generation: WikiGeneration, *paths: str): + WikiService().save_generation_contents( + db, + WikiContentWriteRequest( + generation_id=generation.id, sections=[], removed_paths=list(paths) + ), + ) + + +def test_a_page_the_agent_declares_gone_is_dropped_from_the_version( + test_db: Session, generation: WikiGeneration +): + """An incremental version is a copy of the published one, so not writing a page + cannot mean removing it. Declaring it is the only channel there is.""" + _write(test_db, generation, _section("index", "Index"), _section("legacy", "Old")) + + _remove(test_db, generation, "legacy") + + assert {page_path_of(page) for page in _pages(test_db, generation.id)} == {"index"} + + +def test_a_removal_is_matched_the_same_way_a_write_is( + test_db: Session, generation: WikiGeneration +): + """A path spelled differently must still name the same page, or a removal + silently does nothing and the page outlives its subject.""" + _write(test_db, generation, _section("modules/sync", "Sync")) + + _remove(test_db, generation, " Modules//Sync.md ") + + assert _pages(test_db, generation.id) == [] + + +def test_removing_a_page_that_is_already_gone_is_not_an_error( + test_db: Session, generation: WikiGeneration +): + """A retried submission would otherwise lose the sections sent alongside it.""" + _write(test_db, generation, _section("index", "Index")) + + WikiService().save_generation_contents( + test_db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[_section("guide", "Guide")], + removed_paths=["never-existed"], + ), + ) + + assert {page_path_of(page) for page in _pages(test_db, generation.id)} == { + "index", + "guide", + } + + +def test_a_malformed_removal_path_is_refused( + test_db: Session, generation: WikiGeneration +): + with pytest.raises(HTTPException) as exc: + _remove(test_db, generation, "../escape") + + assert exc.value.status_code == 400 + + +def test_a_page_written_and_removed_in_one_payload_ends_up_removed( + test_db: Session, generation: WikiGeneration +): + """Order within a payload must not decide the outcome.""" + WikiService().save_generation_contents( + test_db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[_section("doomed", "Doomed")], + removed_paths=["doomed"], + ), + ) + + assert _pages(test_db, generation.id) == [] + + +def test_a_payload_with_nothing_in_it_is_still_refused( + test_db: Session, generation: WikiGeneration +): + with pytest.raises(HTTPException) as exc: + WikiService().save_generation_contents( + test_db, + WikiContentWriteRequest(generation_id=generation.id, sections=[]), + ) + + assert exc.value.status_code == 400 diff --git a/backend/tests/services/knowledge/test_code_wiki_generation.py b/backend/tests/services/knowledge/test_code_wiki_generation.py new file mode 100644 index 0000000000..fe0ee1f318 --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_generation.py @@ -0,0 +1,349 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for starting and finishing a code wiki run. + +This is where the separate decisions meet, so these tests are about the combinations: +a run that must not start, a version that must be seeded before the agent sees it, and +a failure that must leave the published wiki exactly as it was. +""" + +from dataclasses import dataclass, field +from datetime import datetime, timedelta + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import KnowledgeDocument +from app.models.user import User +from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.services.knowledge.code_wiki_generation import ( + GenerationInFlight, + finish_generation, + published_commit, + start_generation, +) +from app.services.knowledge.code_wiki_projection import ProjectionSideEffects +from app.services.knowledge.code_wiki_publisher import published_generation_id +from app.services.knowledge.code_wiki_run_mode import ChangedPath, RunMode +from app.services.knowledge.code_wiki_version_store import ( + STALE_RUN_AFTER_HOURS, + set_page_path, +) + +HEAD = "aaaaaaa" +NEXT_HEAD = "bbbbbbb" +NOW = datetime(2026, 7, 31, 12, 0, 0) + + +@dataclass +class FakeEffects: + written: list[str] = field(default_factory=list) + next_id: int = 7000 + + def build(self) -> ProjectionSideEffects: + return ProjectionSideEffects( + write_attachment=self._write, + delete_attachment=lambda _: None, + delete_rag_document=lambda _: None, + enqueue_reindex=lambda _: None, + ) + + def _write(self, *, filename: str, content: str) -> int: + self.next_id += 1 + self.written.append(filename) + return self.next_id + + +@pytest.fixture +def effects() -> FakeEffects: + return FakeEffects() + + +@pytest.fixture +def knowledge_base(test_db: Session, test_user: User) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-generation", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def _write_page(test_db: Session, generation: WikiGeneration, path: str, body: str): + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path, + content=body, + parent_id=0, + ) + set_page_path(entry, path) + test_db.add(entry) + test_db.flush() + + +def _publish_a_first_wiki(test_db, knowledge_base, test_user, effects, *, pages=3): + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + for index in range(pages): + _write_page(test_db, started.generation, f"page-{index}", "body") + finish_generation( + test_db, + knowledge_base=knowledge_base, + generation=started.generation, + user=test_user, + effects=effects.build(), + succeeded=True, + now=NOW, + ) + return started.generation + + +def test_a_first_run_rebuilds_everything( + test_db: Session, knowledge_base: Kind, test_user: User +): + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + + assert started.started + assert RunMode(started.decision.mode) is RunMode.FULL + assert started.seeded_pages == 0 + + +def test_an_unchanged_repository_starts_nothing( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + _publish_a_first_wiki(test_db, knowledge_base, test_user, effects) + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + + assert not started.started + assert RunMode(started.decision.mode) is RunMode.SKIP + + +def test_an_incremental_run_is_seeded_before_the_agent_sees_it( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + """An unseeded incremental version would be projected as a mass deletion.""" + _publish_a_first_wiki(test_db, knowledge_base, test_user, effects, pages=3) + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + now=NOW, + ) + + assert RunMode(started.decision.mode) is RunMode.INCREMENTAL + assert started.seeded_pages == 3 + + +def test_a_full_run_starts_from_an_empty_version( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + _publish_a_first_wiki(test_db, knowledge_base, test_user, effects, pages=3) + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=None, # unknown diff forces a rebuild + now=NOW, + ) + + assert RunMode(started.decision.mode) is RunMode.FULL + assert started.seeded_pages == 0 + + +def test_a_second_run_is_refused_while_one_is_live( + test_db: Session, knowledge_base: Kind, test_user: User +): + start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + + with pytest.raises(GenerationInFlight): + start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + now=NOW, + ) + + +def test_an_abandoned_run_does_not_block_the_wiki_forever( + test_db: Session, knowledge_base: Kind, test_user: User +): + """A crashed worker would otherwise make the wiki unregenerable.""" + stuck = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ).generation + + # Aged explicitly: the row's updated_at comes from the database default, so + # leaving it alone would measure staleness against the wall clock and make this + # test pass or fail depending on the day it runs. + stuck.updated_at = NOW + test_db.flush() + + later = NOW + timedelta(hours=STALE_RUN_AFTER_HOURS + 1) + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + now=later, + ) + + assert started.started + test_db.refresh(stuck) + assert stuck.status == WikiGenerationStatus.FAILED + + +def test_a_successful_run_publishes_its_version( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + _write_page(test_db, started.generation, "index", "overview") + + result = finish_generation( + test_db, + knowledge_base=knowledge_base, + generation=started.generation, + user=test_user, + effects=effects.build(), + succeeded=True, + now=NOW, + ) + + assert result is not None and result.published + assert published_generation_id(knowledge_base) == started.generation.id + assert published_commit(test_db, knowledge_base) == HEAD + + +def test_a_failed_run_leaves_the_published_wiki_untouched( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + first = _publish_a_first_wiki(test_db, knowledge_base, test_user, effects, pages=3) + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + now=NOW, + ) + result = finish_generation( + test_db, + knowledge_base=knowledge_base, + generation=started.generation, + user=test_user, + effects=effects.build(), + succeeded=False, + error_message="model timed out", + now=NOW, + ) + + assert result is None + assert published_generation_id(knowledge_base) == first.id + assert started.generation.status == WikiGenerationStatus.FAILED + assert ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .count() + == 3 + ) + + +def test_a_failure_leaves_the_work_to_be_redone_not_skipped( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + """The published commit must not advance, or the next run skips the changes.""" + _publish_a_first_wiki(test_db, knowledge_base, test_user, effects, pages=3) + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + now=NOW, + ) + finish_generation( + test_db, + knowledge_base=knowledge_base, + generation=started.generation, + user=test_user, + effects=effects.build(), + succeeded=False, + now=NOW, + ) + + assert published_commit(test_db, knowledge_base) == HEAD + + retry = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + now=NOW, + ) + + assert retry.started + + +def test_the_run_mode_reason_is_kept_for_troubleshooting( + test_db: Session, knowledge_base: Kind, test_user: User +): + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + + assert started.generation.ext["runModeReason"] diff --git a/backend/tests/services/knowledge/test_code_wiki_page_path.py b/backend/tests/services/knowledge/test_code_wiki_page_path.py new file mode 100644 index 0000000000..e364f7e93f --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_page_path.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for wiki page path identity. + +The path is what keeps a page's document id — and therefore its RAG index entry and +any stored citation — stable across regenerations, so these tests are mostly about +refusing input that would silently resolve to a different page. +""" + +import pytest + +from app.services.knowledge.code_wiki_page_path import ( + MAX_DIRECTORY_DEPTH, + MAX_PATH_LENGTH, + MAX_SEGMENT_LENGTH, + InvalidPagePath, + assert_unique_within_version, + collation_key, + normalize_page_path, + split_page_path, +) + + +def test_a_plain_path_is_kept_as_is(): + assert normalize_page_path("architecture/backend") == "architecture/backend" + + +def test_surrounding_whitespace_and_repeated_separators_are_meaningless(): + assert normalize_page_path(" architecture//backend/ ") == "architecture/backend" + + +def test_a_trailing_markdown_suffix_is_accepted_and_dropped(): + """A model writing markdown will include it; the projection appends it itself.""" + assert normalize_page_path("architecture/backend.md") == "architecture/backend" + assert normalize_page_path("architecture/backend.MD") == "architecture/backend" + + +def test_a_suffix_inside_the_name_is_left_alone(): + assert normalize_page_path("notes/readme.md.draft") == "notes/readme.md.draft" + + +def test_an_empty_path_is_rejected(): + with pytest.raises(InvalidPagePath): + normalize_page_path(" ") + + +def test_an_absolute_path_is_rejected_rather_than_stripped(): + """Stripping the slash would quietly turn one page into another.""" + with pytest.raises(InvalidPagePath, match="relative"): + normalize_page_path("/architecture/backend") + + +def test_relative_segments_are_rejected(): + with pytest.raises(InvalidPagePath, match="relative segments"): + normalize_page_path("architecture/../../etc/passwd") + + +@pytest.mark.parametrize("raw", ["a/...md", "a/..md", "...md", "..md"]) +def test_a_suffix_strip_cannot_reintroduce_a_relative_segment(raw: str): + """Stripping ".md" from "...md" yields "..", which must not survive. + + Validation therefore runs after the suffix is removed. Checking first and + stripping afterwards let a leaf pass the relative-segment check and then become + the exact value that check exists to reject. + """ + with pytest.raises(InvalidPagePath, match="relative segments"): + normalize_page_path(raw) + + +def test_a_backslash_separator_is_rejected_as_ambiguous(): + with pytest.raises(InvalidPagePath, match="separator"): + normalize_page_path("architecture\\backend") + + +def test_control_characters_are_rejected(): + with pytest.raises(InvalidPagePath, match="control characters"): + normalize_page_path("architecture/back\x00end") + + +def test_reserved_characters_are_rejected(): + with pytest.raises(InvalidPagePath, match="reserved character"): + normalize_page_path('architecture/back"end') + + +def test_an_over_long_segment_is_rejected(): + with pytest.raises(InvalidPagePath, match="segment exceeds"): + normalize_page_path("x" * (MAX_SEGMENT_LENGTH + 1)) + + +def test_an_over_long_path_is_rejected(): + # Stay within the segment and depth limits so the length check is what fires. + segments = ["y" * 120] * 4 + with pytest.raises(InvalidPagePath, match="exceeds"): + normalize_page_path("/".join(segments) + "/" + "z" * 120) + + +def test_a_path_at_the_folder_depth_limit_is_accepted(): + path = "/".join(["d"] * MAX_DIRECTORY_DEPTH) + "/page" + + assert normalize_page_path(path) == path + + +def test_a_path_deeper_than_the_folder_tree_allows_is_rejected(): + """Rejected on write, so one bad path cannot fail an entire version at publish.""" + path = "/".join(["d"] * (MAX_DIRECTORY_DEPTH + 1)) + "/page" + + with pytest.raises(InvalidPagePath, match="folder tree allows"): + normalize_page_path(path) + + +def test_a_root_level_page_has_no_folders(): + assert split_page_path("index") == ((), "index") + + +def test_splitting_separates_folders_from_the_document_name(): + assert split_page_path("a/b/page") == (("a", "b"), "page") + + +def test_paths_differing_only_by_case_are_the_same_page(): + """The knowledge tables collate case-insensitively, so the database agrees.""" + assert collation_key("Architecture/Backend") == collation_key( + "architecture/backend" + ) + + +def test_a_version_may_not_contain_two_paths_that_collide_by_case(): + with pytest.raises(InvalidPagePath, match="differ only by case"): + assert_unique_within_version(["architecture/backend", "Architecture/Backend"]) + + +def test_a_version_may_not_repeat_a_path(): + with pytest.raises(InvalidPagePath, match="more than once"): + assert_unique_within_version(["architecture/backend", "architecture/backend"]) + + +def test_distinct_paths_pass(): + assert_unique_within_version(["index", "architecture/backend", "modules/api"]) + + +def test_the_path_limit_leaves_room_for_a_realistic_layout(): + """Guards against a limit tightened to the point of rejecting ordinary pages.""" + path = "architecture/backend/services/knowledge/document-indexing-pipeline" + + assert len(path) < MAX_PATH_LENGTH + assert normalize_page_path(path) == path diff --git a/backend/tests/services/knowledge/test_code_wiki_projection.py b/backend/tests/services/knowledge/test_code_wiki_projection.py new file mode 100644 index 0000000000..a8dc0fe58a --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_projection.py @@ -0,0 +1,389 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for applying a projection plan. + +Attachment bytes live outside the transaction, so the order in which they are written +and deleted decides whether a failed publish leaves litter or destroys content. These +tests record the sequence of side effects and assert on it directly. +""" + +from dataclasses import dataclass, field + +import pytest +from sqlalchemy.orm import Session + +from app.models.knowledge import ContentOrigin, KnowledgeDocument, KnowledgeFolder +from app.services.knowledge.code_wiki_projection import ( + ProjectionSideEffects, + apply_projection_plan, + finish_projection, +) +from app.services.knowledge.code_wiki_projection_plan import ( + CONTENT_HASH_KEY, + PAGE_PATH_KEY, + PageSource, + ProjectedPage, + compute_projection_plan, + content_fingerprint, +) + +KIND_ID = 501 +USER_ID = 9 + + +@dataclass +class RecordingEffects: + """Records every side effect in order, so ordering can be asserted on.""" + + calls: list[tuple[str, object]] = field(default_factory=list) + next_attachment_id: int = 1000 + failing_rag_refs: set[int] = field(default_factory=set) + + def as_side_effects(self) -> ProjectionSideEffects: + return ProjectionSideEffects( + write_attachment=self._write, + delete_attachment=self._delete_attachment, + delete_rag_document=self._delete_rag, + enqueue_reindex=self._enqueue, + ) + + def _write(self, *, filename: str, content: str) -> int: + self.next_attachment_id += 1 + self.calls.append(("write_attachment", filename)) + return self.next_attachment_id + + def _delete_attachment(self, attachment_id: int) -> None: + self.calls.append(("delete_attachment", attachment_id)) + + def _delete_rag(self, doc_ref: int) -> None: + if doc_ref in self.failing_rag_refs: + raise RuntimeError("vector store unavailable") + self.calls.append(("delete_rag", doc_ref)) + + def _enqueue(self, document_id: int) -> None: + self.calls.append(("enqueue_reindex", document_id)) + + def names(self) -> list[str]: + return [name for name, _ in self.calls] + + +@pytest.fixture +def effects() -> RecordingEffects: + return RecordingEffects() + + +def _source(path: str, content: str = "body") -> PageSource: + return PageSource(path=path, title=path.rsplit("/", 1)[-1], content=content) + + +def _existing_document( + db: Session, path: str, content: str, *, attachment_id: int = 1 +) -> KnowledgeDocument: + folders = path.split("/")[:-1] + parent_id = 0 + for segment in folders: + folder = KnowledgeFolder( + kind_id=KIND_ID, + parent_id=parent_id, + name=segment, + origin=ContentOrigin.GENERATED.value, + ) + db.add(folder) + db.flush() + parent_id = folder.id + + document = KnowledgeDocument( + kind_id=KIND_ID, + attachment_id=attachment_id, + name=path.rsplit("/", 1)[-1], + file_extension="md", + file_size=len(content), + user_id=USER_ID, + folder_id=parent_id, + origin=ContentOrigin.GENERATED.value, + source_config={ + PAGE_PATH_KEY: path, + CONTENT_HASH_KEY: content_fingerprint(content), + }, + ) + db.add(document) + db.flush() + return document + + +def _projected(document: KnowledgeDocument) -> ProjectedPage: + config = document.source_config or {} + return ProjectedPage( + document_id=document.id, + path=config[PAGE_PATH_KEY], + content_hash=config[CONTENT_HASH_KEY], + ) + + +def _apply(db: Session, plan, effects: RecordingEffects): + return apply_projection_plan( + db, + kind_id=KIND_ID, + user_id=USER_ID, + plan=plan, + effects=effects.as_side_effects(), + ) + + +# --- ordering -------------------------------------------------------------- + + +def test_attachments_are_written_before_any_row_changes( + test_db: Session, effects: RecordingEffects +): + """Writing first costs an orphaned object on failure; the reverse costs data.""" + plan = compute_projection_plan([_source("index")], []) + + _apply(test_db, plan, effects) + + assert effects.names() == ["write_attachment"] + + +def test_no_attachment_is_deleted_before_the_transaction_commits( + test_db: Session, effects: RecordingEffects +): + """A rollback after deleting bytes cannot bring them back.""" + existing = _existing_document(test_db, "index", "old", attachment_id=77) + plan = compute_projection_plan([_source("index", "new")], [_projected(existing)]) + + _apply(test_db, plan, effects) + + assert "delete_attachment" not in effects.names() + + +def test_cleanup_and_reindex_only_run_after_the_commit( + test_db: Session, effects: RecordingEffects +): + existing = _existing_document(test_db, "index", "old", attachment_id=77) + plan = compute_projection_plan([_source("index", "new")], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + before_commit = list(effects.names()) + finish_projection( + outcome, superseded_attachment_ids=[77], effects=effects.as_side_effects() + ) + + assert before_commit == ["write_attachment"] + assert effects.names()[1:] == ["delete_attachment", "enqueue_reindex"] + + +# --- what each case does --------------------------------------------------- + + +def test_an_added_page_becomes_an_inactive_document_in_its_folder( + test_db: Session, effects: RecordingEffects +): + plan = compute_projection_plan([_source("architecture/backend")], []) + + outcome = _apply(test_db, plan, effects) + + document = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]) + assert document.name == "backend" + assert document.origin == ContentOrigin.GENERATED.value + # Left off until indexing succeeds, exactly as any other document is. + assert document.is_active is False + folder = test_db.get(KnowledgeFolder, document.folder_id) + assert folder.name == "architecture" + assert folder.origin == ContentOrigin.GENERATED.value + + +def test_an_updated_page_keeps_its_document_id( + test_db: Session, effects: RecordingEffects +): + """The reason path identity exists: the RAG index is keyed on this id.""" + existing = _existing_document(test_db, "index", "old", attachment_id=77) + original_id = existing.id + plan = compute_projection_plan([_source("index", "new")], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + + assert outcome.updated_document_ids == (original_id,) + + +def test_an_updated_page_repoints_instead_of_overwriting( + test_db: Session, effects: RecordingEffects +): + """Overwriting happens before the commit, so a rollback would lose both versions.""" + existing = _existing_document(test_db, "index", "old", attachment_id=77) + plan = compute_projection_plan([_source("index", "new")], [_projected(existing)]) + + _apply(test_db, plan, effects) + + assert existing.attachment_id != 77 + + +def test_a_page_whose_row_vanished_is_added_back_rather_than_lost( + test_db: Session, effects: RecordingEffects +): + """The plan counted it as an update, so skipping it would leave the published + version short a page and strand the attachment already written for it.""" + existing = _existing_document(test_db, "index", "old", attachment_id=77) + plan = compute_projection_plan([_source("index", "new")], [_projected(existing)]) + test_db.delete(existing) + test_db.flush() + + outcome = _apply(test_db, plan, effects) + + assert outcome.updated_document_ids == () + assert len(outcome.created_document_ids) == 1 + restored = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]) + assert restored.name == "index" + + +def test_a_removed_page_records_its_doc_ref_before_the_row_is_gone( + test_db: Session, effects: RecordingEffects +): + """After the commit there is nothing left to derive the RAG key from.""" + existing = _existing_document(test_db, "modules/legacy", "body") + doomed_id = existing.id + plan = compute_projection_plan([], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + + assert outcome.deleted_document_ids == (doomed_id,) + assert outcome.unfinished_index_cleanup == (str(doomed_id),) + assert test_db.get(KnowledgeDocument, doomed_id) is None + + +def test_an_unchanged_page_causes_no_side_effects_at_all( + test_db: Session, effects: RecordingEffects +): + existing = _existing_document(test_db, "index", "same") + plan = compute_projection_plan([_source("index", "same")], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + + assert effects.calls == [] + assert outcome.created_document_ids == () + assert outcome.updated_document_ids == () + + +def test_the_fingerprint_is_stamped_so_the_next_run_can_skip( + test_db: Session, effects: RecordingEffects +): + plan = compute_projection_plan([_source("index", "v1")], []) + + outcome = _apply(test_db, plan, effects) + + document = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]) + assert document.source_config[CONTENT_HASH_KEY] == content_fingerprint("v1") + assert document.source_config[PAGE_PATH_KEY] == "index" + + +# --- folders --------------------------------------------------------------- + + +def test_a_folder_emptied_by_deletion_is_removed( + test_db: Session, effects: RecordingEffects +): + existing = _existing_document(test_db, "modules/legacy", "body") + plan = compute_projection_plan([], [_projected(existing)]) + + _apply(test_db, plan, effects) + + assert ( + test_db.query(KnowledgeFolder) + .filter(KnowledgeFolder.kind_id == KIND_ID) + .count() + == 0 + ) + + +def test_an_empty_user_folder_is_left_alone( + test_db: Session, effects: RecordingEffects +): + """Tidying it away would be the projection reaching outside what it owns.""" + user_folder = KnowledgeFolder( + kind_id=KIND_ID, + parent_id=0, + name="my notes", + origin=ContentOrigin.USER.value, + ) + test_db.add(user_folder) + test_db.flush() + + _apply(test_db, compute_projection_plan([], []), effects) + + assert test_db.get(KnowledgeFolder, user_folder.id) is not None + + +def test_folders_are_reused_rather_than_duplicated( + test_db: Session, effects: RecordingEffects +): + plan = compute_projection_plan( + [_source("architecture/backend"), _source("architecture/frontend")], [] + ) + + _apply(test_db, plan, effects) + + assert ( + test_db.query(KnowledgeFolder) + .filter( + KnowledgeFolder.kind_id == KIND_ID, KnowledgeFolder.name == "architecture" + ) + .count() + == 1 + ) + + +# --- retriable cleanup ----------------------------------------------------- + + +def test_a_failed_rag_deletion_is_reported_for_retry_not_raised( + test_db: Session, effects: RecordingEffects +): + """The pages are already correct; failing the publish would regenerate them all.""" + existing = _existing_document(test_db, "modules/legacy", "body") + doomed_id = existing.id + effects.failing_rag_refs.add(doomed_id) + plan = compute_projection_plan([], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + unfinished = finish_projection( + outcome, superseded_attachment_ids=[], effects=effects.as_side_effects() + ) + + assert unfinished == (str(doomed_id),) + + +def test_successful_cleanup_leaves_nothing_to_retry( + test_db: Session, effects: RecordingEffects +): + existing = _existing_document(test_db, "modules/legacy", "body") + plan = compute_projection_plan([], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + unfinished = finish_projection( + outcome, superseded_attachment_ids=[], effects=effects.as_side_effects() + ) + + assert unfinished == () + + +def test_user_content_is_never_touched(test_db: Session, effects: RecordingEffects): + """It is not regenerable, so a mistaken delete here is unrecoverable.""" + user_document = KnowledgeDocument( + kind_id=KIND_ID, + attachment_id=5, + name="my note", + file_extension="md", + file_size=4, + user_id=USER_ID, + folder_id=0, + origin=ContentOrigin.USER.value, + ) + test_db.add(user_document) + test_db.flush() + + # The caller scopes ``existing`` to generated pages, so user content simply is + # not in the comparison — this asserts the projection does not find it anyway. + _apply(test_db, compute_projection_plan([], []), effects) + + assert test_db.get(KnowledgeDocument, user_document.id) is not None diff --git a/backend/tests/services/knowledge/test_code_wiki_projection_plan.py b/backend/tests/services/knowledge/test_code_wiki_projection_plan.py new file mode 100644 index 0000000000..b8774dbfb1 --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_projection_plan.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the projection plan. + +The plan decides what publishing a version does to the knowledge base, including what +it deletes, so these tests are mostly about it not reaching further than it should. +""" + +from app.services.knowledge.code_wiki_projection_plan import ( + PageSource, + ProjectedPage, + compute_projection_plan, + content_fingerprint, +) + + +def _source(path: str, content: str = "body") -> PageSource: + return PageSource(path=path, title=path.rsplit("/", 1)[-1], content=content) + + +def _projected(document_id: int, path: str, content: str = "body") -> ProjectedPage: + return ProjectedPage( + document_id=document_id, + path=path, + content_hash=content_fingerprint(content), + ) + + +def test_a_page_only_the_version_has_is_added(): + plan = compute_projection_plan([_source("index")], []) + + assert [page.path for page in plan.adds] == ["index"] + assert plan.updates == () + assert plan.deletes == () + + +def test_a_page_whose_content_changed_is_updated(): + plan = compute_projection_plan( + [_source("index", "new")], [_projected(1, "index", "old")] + ) + + assert [update.existing.document_id for update in plan.updates] == [1] + assert plan.adds == () + + +def test_an_unchanged_page_is_skipped_entirely(): + """Where an incremental run's savings come from: no write, no reindex, no row.""" + plan = compute_projection_plan( + [_source("index", "same")], [_projected(1, "index", "same")] + ) + + assert plan.skips == ("index",) + assert plan.adds == () + assert plan.updates == () + assert plan.deletes == () + assert plan.is_empty + + +def test_a_page_only_the_knowledge_base_has_is_deleted(): + plan = compute_projection_plan([], [_projected(7, "modules/legacy")]) + + assert [page.document_id for page in plan.deletes] == [7] + + +def test_a_moved_page_is_an_add_and_a_delete(): + """Identity is the path, so a move is not an in-place rename.""" + plan = compute_projection_plan( + [_source("services/backend")], [_projected(3, "architecture/backend")] + ) + + assert [page.path for page in plan.adds] == ["services/backend"] + assert [page.document_id for page in plan.deletes] == [3] + + +def test_matching_ignores_case_because_the_database_does(): + plan = compute_projection_plan( + [_source("Architecture/Backend", "same")], + [_projected(1, "architecture/backend", "same")], + ) + + assert plan.skips == ("Architecture/Backend",) + assert plan.deletes == () + + +def test_a_rewritten_title_alone_does_not_touch_the_page(): + """Titles live in the version; the projection compares content, not headings.""" + existing = _projected(1, "index", "unchanged body") + renamed = PageSource( + path="index", title="A Better Heading", content="unchanged body" + ) + + plan = compute_projection_plan([renamed], [existing]) + + assert plan.skips == ("index",) + + +def test_an_empty_version_removes_everything_it_owns(): + """Deliberate: the publish gate, not the plan, decides whether that is acceptable.""" + plan = compute_projection_plan( + [], [_projected(1, "a"), _projected(2, "b"), _projected(3, "c")] + ) + + assert len(plan.deletes) == 3 + + +def test_a_mixed_version_is_reported_in_full(): + plan = compute_projection_plan( + [ + _source("index", "changed"), + _source("architecture/backend", "same"), + _source("modules/new"), + ], + [ + _projected(1, "index", "original"), + _projected(2, "architecture/backend", "same"), + _projected(3, "modules/gone"), + ], + ) + + assert [page.path for page in plan.adds] == ["modules/new"] + assert [update.existing.document_id for update in plan.updates] == [1] + assert plan.skips == ("architecture/backend",) + assert [page.document_id for page in plan.deletes] == [3] + assert plan.touched_pages == 3 + assert plan.describe() == "1 added, 1 updated, 1 removed, 1 unchanged" + + +def test_an_identical_snapshot_produces_no_work(): + pages = [_projected(1, "index", "x"), _projected(2, "a/b", "y")] + sources = [_source("index", "x"), _source("a/b", "y")] + + assert compute_projection_plan(sources, pages).is_empty + + +def test_the_fingerprint_distinguishes_content(): + assert content_fingerprint("a") != content_fingerprint("b") + assert content_fingerprint("a") == content_fingerprint("a") diff --git a/backend/tests/services/knowledge/test_code_wiki_prompts.py b/backend/tests/services/knowledge/test_code_wiki_prompts.py new file mode 100644 index 0000000000..88db3ece5a --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_prompts.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the wiki generation instructions. + +Prompt quality cannot be asserted, but its contract can: the projection depends on +pages arriving with stable paths and complete content, and the two modes differ in +whether an unwritten page means "unchanged" or "gone". Getting either wrong is a data +loss, so those are pinned here. +""" + +from app.services.knowledge.code_wiki_prompts import ( + REQUIRED_COVERAGE, + WikiRunContext, + build_diagram_correction, + build_full_prompt, + build_incremental_prompt, + build_prompt, +) + + +def _context(**overrides) -> WikiRunContext: + defaults = dict(project_name="wecode-ai/Wegent", generation_id=42) + defaults.update(overrides) + return WikiRunContext(**defaults) + + +# --- the write contract ---------------------------------------------------- + + +def test_both_modes_require_a_stable_path(): + """The path is the page's identity; a shifting one republishes the page.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "path" in prompt + assert "stable" in prompt + + +def test_both_modes_ask_for_complete_content(): + """There is no patch format, so a partial page would replace a whole one.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "complete content" in prompt + + +def test_a_full_rebuild_says_an_unwritten_page_is_absent(): + prompt = build_full_prompt(_context()) + + assert "begins empty" in prompt + assert "every page the wiki should contain" in prompt + + +def test_an_incremental_run_says_an_unwritten_page_is_kept(): + """The opposite of a full run, and confusing them deletes or resurrects pages.""" + prompt = build_incremental_prompt(_context()) + + assert "copy of the published" in prompt + assert "does **not** remove it" in prompt + + +def test_only_the_incremental_mode_offers_explicit_removal(): + """A full run removes a page by not writing it; declaring it would be redundant.""" + assert "declare its removal" in build_incremental_prompt(_context()) + assert "declare its removal" not in build_full_prompt(_context()) + + +def test_the_path_limits_match_what_the_validator_enforces(): + """A prompt promising more than the validator accepts fails at write time.""" + prompt = build_full_prompt(_context()) + + assert "4 folders deep" in prompt + assert "differ only by case" in prompt + + +# --- content discipline ---------------------------------------------------- + + +def test_every_mode_forbids_inventing_evidence(): + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "Do not invent" in prompt + + +def test_every_mode_bounds_how_much_is_read(): + """Reading everything is how a run exhausts its budget before writing.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "Do not read every file" in prompt + + +def test_every_mode_asks_history_to_explain_why(): + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "why" in prompt + + +def test_every_mode_carries_the_relevance_test(): + """The one rule aimed at the generic, padded output the old wiki produced.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "would this change what someone does here" in prompt + + +def test_a_full_rebuild_plans_pages_and_links_before_writing(): + prompt = build_full_prompt(_context()) + + assert "Before writing any page" in prompt + assert "relationship" in prompt + + +def test_a_full_rebuild_states_the_expected_coverage(): + prompt = build_full_prompt(_context()) + + for item in REQUIRED_COVERAGE: + assert item in prompt + + +# --- run context ----------------------------------------------------------- + + +def test_the_incremental_prompt_lists_what_changed(): + prompt = build_incremental_prompt( + _context( + previous_commit="aaaa", + head_commit="bbbb", + changed_paths=["backend/app/main.py", "frontend/src/app.tsx"], + ) + ) + + assert "aaaa" in prompt and "bbbb" in prompt + assert "backend/app/main.py" in prompt + + +def test_the_incremental_prompt_lists_the_pages_that_exist(): + """Without it the agent cannot tell an update from a new page.""" + prompt = build_incremental_prompt(_context(existing_pages=["index", "arch/api"])) + + assert "arch/api" in prompt + + +def test_an_empty_change_list_is_stated_rather_than_left_blank(): + prompt = build_incremental_prompt(_context()) + + assert "no file list was available" in prompt + + +def test_an_empty_wiki_is_stated_rather_than_left_blank(): + prompt = build_incremental_prompt(_context()) + + assert "currently empty" in prompt + + +def test_the_generation_id_travels_with_the_run(): + assert "42" in build_full_prompt(_context()) + + +def test_the_mode_selects_the_prompt(): + assert build_prompt(_context(), full=True) == build_full_prompt(_context()) + assert build_prompt(_context(), full=False) == build_incremental_prompt(_context()) + + +# --- diagram feedback ------------------------------------------------------ + + +def test_diagram_problems_come_back_as_a_correction(): + correction = build_diagram_correction(["index: line 4 is not a diagram type"]) + + assert correction is not None + assert "index: line 4" in correction + assert "paths unchanged" in correction + + +def test_no_diagram_problems_means_no_follow_up(): + assert build_diagram_correction([]) is None diff --git a/backend/tests/services/knowledge/test_code_wiki_publish_end_to_end.py b/backend/tests/services/knowledge/test_code_wiki_publish_end_to_end.py new file mode 100644 index 0000000000..c0b6c104c9 --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_publish_end_to_end.py @@ -0,0 +1,322 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Publishing a version through the real attachment and indexing services. + +Every other test for this pipeline injects fakes, which proves the logic but not the +assumptions it rests on. This one uses the real attachment store and asserts on what +a reader would actually get, so that a wrong signature or a document that never +becomes visible fails here rather than in production. +""" + +from datetime import datetime +from unittest.mock import patch + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import ( + ContentOrigin, + DocumentIndexStatus, + DocumentStatus, + KnowledgeDocument, +) +from app.models.subtask_context import SubtaskContext +from app.models.user import User +from app.models.wiki import ( + WikiContent, + WikiGeneration, + WikiGenerationStatus, + WikiGenerationType, +) +from app.services.knowledge.code_wiki_projection_plan import PAGE_PATH_KEY +from app.services.knowledge.code_wiki_publisher import ( + publish_generation, + published_generation_id, +) +from app.services.knowledge.code_wiki_side_effects import ( + build_projection_side_effects, +) +from app.services.knowledge.code_wiki_version_store import set_page_path +from app.services.knowledge.index_state_machine import ( + mark_document_index_succeeded, +) + + +@pytest.fixture +def knowledge_base(test_db: Session, test_user: User) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-wiki-e2e", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +@pytest.fixture +def generation(test_db: Session, knowledge_base: Kind, test_user: User): + def build() -> WikiGeneration: + record = WikiGeneration( + project_id=1, + kind_id=knowledge_base.id, + user_id=test_user.id, + task_id=0, + team_id=1, + generation_type=WikiGenerationType.FULL, + source_snapshot={}, + status=WikiGenerationStatus.COMPLETED, + completed_at=datetime(1970, 1, 1), + ) + test_db.add(record) + test_db.flush() + return record + + return build + + +def _page(test_db: Session, generation: WikiGeneration, path: str, content: str): + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path.rsplit("/", 1)[-1], + content=content, + parent_id=0, + ) + set_page_path(entry, path) + test_db.add(entry) + test_db.flush() + + +def _publish(test_db, knowledge_base, generation, test_user, enqueued): + """Publish with real attachment storage; only the queue is intercepted.""" + effects = build_projection_side_effects( + test_db, knowledge_base=knowledge_base, user=test_user + ) + with patch( + "app.services.knowledge.code_wiki_side_effects._enqueue_reindex", + side_effect=lambda db, **kw: enqueued.append(kw["document_id"]), + ): + return publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=test_user.id, + effects=effects, + ) + + +def test_a_published_page_has_real_content_behind_it( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + """The document must point at an attachment that actually holds the page.""" + version = generation() + _page(test_db, version, "architecture/backend", "# Backend\n\nHow it works.") + enqueued: list[int] = [] + + result = _publish(test_db, knowledge_base, version, test_user, enqueued) + + assert result.published + document = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + ) + attachment = test_db.get(SubtaskContext, document.attachment_id) + assert attachment is not None + assert attachment.extracted_text.strip().startswith("# Backend") + assert document.file_size == len("# Backend\n\nHow it works.".encode("utf-8")) + + +def test_a_published_page_is_queued_for_indexing( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + version = generation() + _page(test_db, version, "index", "overview") + enqueued: list[int] = [] + + _publish(test_db, knowledge_base, version, test_user, enqueued) + + document = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + ) + assert enqueued == [document.id] + + +def test_a_page_stays_invisible_until_its_index_succeeds( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + """The assumption the whole publish path rests on. + + Pages are created switched off and the indexing state machine turns them on. If + that did not hold, a published wiki would be silently unreadable while every unit + test still passed. + """ + from app.services.knowledge.index_state_machine import ( + prepare_document_index_enqueue, + ) + + version = generation() + _page(test_db, version, "index", "overview") + enqueued: list[int] = [] + + _publish(test_db, knowledge_base, version, test_user, enqueued) + + document = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + ) + assert document.is_active is False + assert document.status == DocumentStatus.DISABLED + + # The real path, not a shortcut: the state machine only accepts a success for a + # document it queued, so jumping straight to it proves nothing. + decision = prepare_document_index_enqueue(test_db, document_id=document.id) + assert decision.should_enqueue + mark_document_index_succeeded( + test_db, document_id=document.id, generation=decision.generation + ) + test_db.refresh(document) + + assert document.is_active is True + assert document.status == DocumentStatus.ENABLED + + +def test_indexing_is_not_blocked_by_the_page_being_switched_off( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + """Enqueueing looks at the index status, not at whether the page is visible.""" + version = generation() + _page(test_db, version, "index", "overview") + enqueued: list[int] = [] + _publish(test_db, knowledge_base, version, test_user, enqueued) + + document = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + ) + + from app.services.knowledge.index_state_machine import ( + prepare_document_index_enqueue, + ) + + decision = prepare_document_index_enqueue(test_db, document_id=document.id) + + assert decision.should_enqueue is True + + +def test_a_second_publish_replaces_the_content_and_keeps_the_document( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + first = generation() + _page(test_db, first, "index", "first draft") + enqueued: list[int] = [] + _publish(test_db, knowledge_base, first, test_user, enqueued) + document_id = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + .id + ) + original_attachment = test_db.get(KnowledgeDocument, document_id).attachment_id + + second = generation() + _page(test_db, second, "index", "second draft") + _publish(test_db, knowledge_base, second, test_user, enqueued) + + document = test_db.get(KnowledgeDocument, document_id) + assert document is not None, "the document id must survive a rewrite" + assert document.attachment_id != original_attachment + attachment = test_db.get(SubtaskContext, document.attachment_id) + assert "second draft" in attachment.extracted_text + assert published_generation_id(knowledge_base) == second.id + + +def test_an_unchanged_page_writes_no_new_attachment( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + first = generation() + _page(test_db, first, "index", "identical") + enqueued: list[int] = [] + _publish(test_db, knowledge_base, first, test_user, enqueued) + attachment_count = test_db.query(SubtaskContext).count() + + second = generation() + _page(test_db, second, "index", "identical") + _publish(test_db, knowledge_base, second, test_user, enqueued) + + assert test_db.query(SubtaskContext).count() == attachment_count + + +def test_a_removed_page_takes_its_attachment_with_it( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + first = generation() + _page(test_db, first, "index", "kept") + _page(test_db, first, "doomed", "removed later") + enqueued: list[int] = [] + _publish(test_db, knowledge_base, first, test_user, enqueued) + doomed = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.name == "doomed") + .one() + ) + doomed_attachment = doomed.attachment_id + + second = generation() + _page(test_db, second, "index", "kept") + result = _publish(test_db, knowledge_base, second, test_user, enqueued) + + # Stated rather than assumed: removing one of two pages sits exactly on the + # gate's threshold, and everything below only means something if it published. + assert result.published + assert test_db.get(KnowledgeDocument, doomed.id) is None + assert test_db.get(SubtaskContext, doomed_attachment) is None + + +def test_the_folder_tree_is_built_from_the_page_paths( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + version = generation() + _page(test_db, version, "index", "root page") + _page(test_db, version, "architecture/backend", "nested page") + enqueued: list[int] = [] + + _publish(test_db, knowledge_base, version, test_user, enqueued) + + documents = { + (document.source_config or {}).get(PAGE_PATH_KEY): document + for document in test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .all() + } + assert documents["index"].folder_id == 0 + assert documents["architecture/backend"].folder_id != 0 + assert documents["architecture/backend"].origin == ContentOrigin.GENERATED.value + + +def test_a_fresh_page_starts_unindexed( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + version = generation() + _page(test_db, version, "index", "overview") + enqueued: list[int] = [] + + _publish(test_db, knowledge_base, version, test_user, enqueued) + + document = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + ) + assert document.index_status == DocumentIndexStatus.NOT_INDEXED diff --git a/backend/tests/services/knowledge/test_code_wiki_publish_gate.py b/backend/tests/services/knowledge/test_code_wiki_publish_gate.py new file mode 100644 index 0000000000..e9f73effc5 --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_publish_gate.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the publish gate. + +The gate is what makes agent-declared deletion safe to allow: it lets a version remove +pages while refusing one that removes most of them. + +Removal is measured over paths, not counts, so the cases below pair a version with the +set of paths currently published rather than with a number. +""" + +from app.services.knowledge.code_wiki_projection_plan import PageSource +from app.services.knowledge.code_wiki_publish_gate import ( + PublishPolicy, + evaluate_publish_gate, +) + + +def _pages(count: int, content: str = "flowchart TD\n A --> B") -> list[PageSource]: + return [ + PageSource(path=f"page-{index}", title=f"Page {index}", content=content) + for index in range(count) + ] + + +def _published(count: int) -> list[str]: + """The paths a previously published version left in the knowledge base.""" + return [f"page-{index}" for index in range(count)] + + +def test_a_version_of_similar_size_passes(): + verdict = evaluate_publish_gate(_pages(10), published_paths=_published(10)) + + assert verdict.passed + + +def test_a_first_publish_has_nothing_to_compare_against(): + verdict = evaluate_publish_gate(_pages(3), published_paths=_published(0)) + + assert verdict.passed + + +def test_a_version_that_produced_nothing_is_rejected(): + """An empty repository still yields an overview page; zero means a failed run.""" + verdict = evaluate_publish_gate([], published_paths=_published(0)) + + assert not verdict.passed + assert "nothing usable" in verdict.reason + + +def test_dropping_most_of_the_wiki_is_rejected(): + verdict = evaluate_publish_gate(_pages(2), published_paths=_published(10)) + + assert not verdict.passed + assert "80%" in verdict.reason + + +def test_a_moderate_removal_is_allowed(): + """Deleting pages is legitimate; this is what the gate deliberately permits.""" + verdict = evaluate_publish_gate(_pages(7), published_paths=_published(10)) + + assert verdict.passed + + +def test_the_removal_limit_is_configurable(): + strict = PublishPolicy(max_removed_share=0.1) + + verdict = evaluate_publish_gate( + _pages(8), published_paths=_published(10), policy=strict + ) + + assert not verdict.passed + + +def test_growth_is_never_treated_as_removal(): + verdict = evaluate_publish_gate(_pages(50), published_paths=_published(2)) + + assert verdict.passed + + +def test_a_broken_diagram_is_reported_but_does_not_block(): + """A diagram that will not render is a local display fault, not a bad version.""" + pages = [ + PageSource(path="index", title="Index", content="```mermaid\nflowchat TD\n```") + ] + + verdict = evaluate_publish_gate(pages, published_paths=["index"]) + + assert verdict.passed + assert len(verdict.warnings) == 1 + assert "index:" in verdict.warnings[0] + + +def test_diagrams_can_be_made_blocking_by_policy(): + pages = [ + PageSource(path="index", title="Index", content="```mermaid\nflowchat TD\n```") + ] + + verdict = evaluate_publish_gate( + pages, published_paths=["index"], policy=PublishPolicy(block_on_mermaid=True) + ) + + assert not verdict.passed + + +def test_a_rejection_still_carries_its_warnings(): + """The verdict is stored to explain the rejection, so it must be complete.""" + pages = [ + PageSource(path="index", title="Index", content="```mermaid\nflowchat TD\n```") + ] + + verdict = evaluate_publish_gate(pages, published_paths=_published(10)) + + assert not verdict.passed + assert verdict.warnings + + +def test_the_verdict_renders_for_storage(): + verdict = evaluate_publish_gate(_pages(1), published_paths=_published(10)) + + stored = verdict.to_ext("2026-07-31T00:00:00") + + assert stored["result"] == "rejected" + assert stored["reason"] + assert stored["checkedAt"] == "2026-07-31T00:00:00" + + +def test_a_passing_verdict_renders_as_passed(): + stored = evaluate_publish_gate(_pages(5), published_paths=_published(5)).to_ext( + "now" + ) + + assert stored["result"] == "passed" + + +# --- removal is about paths, not counts ------------------------------------- + + +def test_replacing_every_path_is_a_mass_deletion_even_at_the_same_size(): + """The case counting cannot see. Ten pages become ten pages, and every one of + them is a new document: the old ids are deleted, and every stored citation and + index entry pointing at them breaks.""" + renamed = [ + PageSource(path=f"guide/page-{index}", title=f"Page {index}", content="body") + for index in range(10) + ] + + verdict = evaluate_publish_gate(renamed, published_paths=_published(10)) + + assert not verdict.passed + assert "100%" in verdict.reason + + +def test_renaming_a_few_paths_is_still_allowed(): + """Moving a page is legitimate; the gate only refuses moving most of them.""" + moved = [ + PageSource(path="guide/page-0", title="Page 0", content="body"), + *_pages(10)[1:], + ] + + verdict = evaluate_publish_gate(moved, published_paths=_published(10)) + + assert verdict.passed + + +def test_a_path_kept_under_a_different_case_is_not_a_removal(): + """The knowledge tables collate case-insensitively, so the two are one page.""" + recased = [ + PageSource(path=f"Page-{index}", title=f"Page {index}", content="body") + for index in range(10) + ] + + verdict = evaluate_publish_gate(recased, published_paths=_published(10)) + + assert verdict.passed diff --git a/backend/tests/services/knowledge/test_code_wiki_publisher.py b/backend/tests/services/knowledge/test_code_wiki_publisher.py new file mode 100644 index 0000000000..b2f9fafbaa --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_publisher.py @@ -0,0 +1,402 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for publishing a code wiki version. + +Publishing is the only thing that moves the published pointer, and a rollback is the +same operation aimed at an older version, so these tests cover both through one path. +""" + +from dataclasses import dataclass, field +from datetime import datetime + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import ContentOrigin, KnowledgeDocument +from app.models.wiki import ( + WikiContent, + WikiGeneration, + WikiGenerationStatus, + WikiGenerationType, +) +from app.services.knowledge.code_wiki_projection import ( + PENDING_INDEX_CLEANUP_KEY, + ProjectionSideEffects, +) +from app.services.knowledge.code_wiki_publish_gate import ( + PUBLISH_GATE_EXT_KEY, + PublishPolicy, +) +from app.services.knowledge.code_wiki_publisher import ( + PUBLISHED_GENERATION_KEY, + publish_generation, + published_generation_id, + retry_pending_index_cleanup, +) +from app.services.knowledge.code_wiki_version_store import set_page_path + +USER_ID = 11 + + +@dataclass +class FakeEffects: + written: list[str] = field(default_factory=list) + deleted_attachments: list[int] = field(default_factory=list) + deleted_rag: list[int] = field(default_factory=list) + reindexed: list[int] = field(default_factory=list) + failing_rag: set[int] = field(default_factory=set) + next_id: int = 5000 + + def build(self) -> ProjectionSideEffects: + return ProjectionSideEffects( + write_attachment=self._write, + delete_attachment=self.deleted_attachments.append, + delete_rag_document=self._delete_rag, + enqueue_reindex=self.reindexed.append, + ) + + def _write(self, *, filename: str, content: str) -> int: + self.next_id += 1 + self.written.append(filename) + return self.next_id + + def _delete_rag(self, doc_ref: int) -> None: + if doc_ref in self.failing_rag: + raise RuntimeError("vector store down") + self.deleted_rag.append(doc_ref) + + +@pytest.fixture +def effects() -> FakeEffects: + return FakeEffects() + + +@pytest.fixture +def knowledge_base(test_db: Session) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-code-wiki", + namespace="default", + user_id=USER_ID, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def _generation( + test_db: Session, + kind_id: int, + *, + status: WikiGenerationStatus = WikiGenerationStatus.COMPLETED, +) -> WikiGeneration: + generation = WikiGeneration( + project_id=1, + kind_id=kind_id, + user_id=USER_ID, + task_id=0, + team_id=1, + generation_type=WikiGenerationType.FULL, + source_snapshot={}, + status=status, + completed_at=datetime(1970, 1, 1), + ) + test_db.add(generation) + test_db.flush() + return generation + + +def _page(test_db: Session, generation: WikiGeneration, path: str, content: str): + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path.rsplit("/", 1)[-1], + content=content, + parent_id=0, + ) + set_page_path(entry, path) + test_db.add(entry) + test_db.flush() + return entry + + +def _live_paths(test_db: Session, kind_id: int) -> set[str]: + from app.services.knowledge.code_wiki_projection_plan import PAGE_PATH_KEY + + return { + (document.source_config or {}).get(PAGE_PATH_KEY) + for document in test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == kind_id) + .all() + } + + +def test_a_first_publish_creates_the_pages_and_moves_the_pointer( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + generation = _generation(test_db, knowledge_base.id) + _page(test_db, generation, "index", "overview") + _page(test_db, generation, "architecture/backend", "details") + + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert result.published + assert _live_paths(test_db, knowledge_base.id) == {"index", "architecture/backend"} + assert published_generation_id(knowledge_base) == generation.id + + +def test_a_rejected_version_leaves_the_pointer_and_pages_alone( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """The published wiki must survive a run that produced almost nothing.""" + first = _generation(test_db, knowledge_base.id) + for index in range(10): + _page(test_db, first, f"page-{index}", "body") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=first, + user_id=USER_ID, + effects=effects.build(), + ) + + second = _generation(test_db, knowledge_base.id) + _page(test_db, second, "page-0", "body") + + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=second, + user_id=USER_ID, + effects=effects.build(), + ) + + assert not result.published + assert published_generation_id(knowledge_base) == first.id + assert len(_live_paths(test_db, knowledge_base.id)) == 10 + + +def test_a_rejection_is_recorded_on_the_generation( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """Explains why a finished version is not live; the pointer still decides.""" + generation = _generation(test_db, knowledge_base.id) + + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + recorded = (generation.ext or {})[PUBLISH_GATE_EXT_KEY] + assert recorded["result"] == "rejected" + assert recorded["reason"] + + +def test_an_unfinished_generation_is_not_published( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + generation = _generation( + test_db, knowledge_base.id, status=WikiGenerationStatus.RUNNING + ) + _page(test_db, generation, "index", "body") + + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert not result.published + assert published_generation_id(knowledge_base) == 0 + + +def test_publishing_again_with_unchanged_content_does_nothing( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + generation = _generation(test_db, knowledge_base.id) + _page(test_db, generation, "index", "same") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + repeat = FakeEffects() + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=repeat.build(), + ) + + assert result.published + assert repeat.written == [] + assert repeat.reindexed == [] + + +def test_rolling_back_is_publishing_an_older_version( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """No separate machinery: the same call, aimed at the version you want back.""" + first = _generation(test_db, knowledge_base.id) + _page(test_db, first, "index", "original") + _page(test_db, first, "keep-me", "body") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=first, + user_id=USER_ID, + effects=effects.build(), + ) + + second = _generation(test_db, knowledge_base.id) + _page(test_db, second, "index", "rewritten") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=second, + user_id=USER_ID, + effects=effects.build(), + policy=PublishPolicy(max_removed_share=1.0), + ) + assert _live_paths(test_db, knowledge_base.id) == {"index"} + + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=first, + user_id=USER_ID, + effects=effects.build(), + require_completed=False, + ) + + assert result.published + assert _live_paths(test_db, knowledge_base.id) == {"index", "keep-me"} + assert published_generation_id(knowledge_base) == first.id + + +def test_user_content_survives_a_publish( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """It is not regenerable, so the projection must never see it.""" + note = KnowledgeDocument( + kind_id=knowledge_base.id, + attachment_id=1, + name="my note", + file_extension="md", + file_size=4, + user_id=USER_ID, + folder_id=0, + origin=ContentOrigin.USER.value, + ) + test_db.add(note) + test_db.flush() + + generation = _generation(test_db, knowledge_base.id) + _page(test_db, generation, "index", "body") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert test_db.get(KnowledgeDocument, note.id) is not None + + +def test_a_failed_index_deletion_is_parked_for_retry( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """A row gone while its chunks remain leaves retrieval citing a dead page.""" + first = _generation(test_db, knowledge_base.id) + _page(test_db, first, "index", "body") + _page(test_db, first, "doomed", "body") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=first, + user_id=USER_ID, + effects=effects.build(), + ) + doomed_id = next( + document.id + for document in test_db.query(KnowledgeDocument).all() + if document.name == "doomed" + ) + + second = _generation(test_db, knowledge_base.id) + _page(test_db, second, "index", "body") + broken = FakeEffects(failing_rag={doomed_id}) + + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=second, + user_id=USER_ID, + effects=broken.build(), + ) + + # Stated rather than assumed: this version removes exactly half the published + # pages, which sits on the gate's threshold. Without this the test would still + # fail if the gate ever tightened, but as a confusing KeyError below. + assert result.published + parked = (knowledge_base.json or {})["spec"][PENDING_INDEX_CLEANUP_KEY] + assert parked == [str(doomed_id)] + + +def test_parked_cleanup_is_cleared_once_it_succeeds( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + knowledge_base.json = { + "spec": { + "name": "wiki", + "kbType": "code_wiki", + PENDING_INDEX_CLEANUP_KEY: ["42"], + } + } + test_db.flush() + + outstanding = retry_pending_index_cleanup( + test_db, knowledge_base=knowledge_base, effects=effects.build() + ) + + assert outstanding == () + assert knowledge_base.json["spec"][PENDING_INDEX_CLEANUP_KEY] == [] + assert effects.deleted_rag == [42] + + +def test_cleanup_that_still_fails_stays_parked(test_db: Session, knowledge_base: Kind): + knowledge_base.json = {"spec": {"name": "wiki", PENDING_INDEX_CLEANUP_KEY: ["42"]}} + test_db.flush() + broken = FakeEffects(failing_rag={42}) + + outstanding = retry_pending_index_cleanup( + test_db, knowledge_base=knowledge_base, effects=broken.build() + ) + + assert outstanding == ("42",) + + +def test_the_published_pointer_starts_at_nothing(knowledge_base: Kind): + assert published_generation_id(knowledge_base) == 0 + assert PUBLISHED_GENERATION_KEY not in knowledge_base.json["spec"] diff --git a/backend/tests/services/knowledge/test_code_wiki_repo_state.py b/backend/tests/services/knowledge/test_code_wiki_repo_state.py new file mode 100644 index 0000000000..8e3cd52797 --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_repo_state.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for reading what a repository is at, before a run is decided. + +The value here is entirely in the failure modes. A correct read saves a full rebuild; +a *wrong* read is worse than no read at all, because a partial diff mistaken for a +complete one picks an incremental run for a change that reshaped the repository. + +So every case that cannot be answered confidently has to come back as "unknown" — +which the run-mode rules turn into a full rebuild — and never as "nothing changed". +The one exception is pinned too: a repository already at the documented commit really +has an empty diff, and reporting that as unknown would rebuild a wiki that is current. +""" + +from unittest.mock import patch + +import pytest +from sqlalchemy.orm import Session + +from app.services.knowledge.code_wiki_repo_state import ( + RepositoryState, + read_repository_state, +) +from app.services.knowledge.code_wiki_source import ( + SUPPORTED_SOURCE_TYPES, + SourceRepository, + provider_for, +) + +SOURCE = SourceRepository( + source_type="github", + source_url="https://github.com/wecode-ai/Wegent.git", + project_name="wecode-ai/Wegent", + source_domain="github.com", +) + +HEAD = "bbbbbbb" +PUBLISHED = "aaaaaaa" + + +class FakeProvider: + """A provider that answers exactly what a test tells it to.""" + + def __init__(self, head=None, files=None, head_raises=False, files_raise=False): + self._head = head if head is not None else {"branch": "main", "commit": HEAD} + self._files = files + self._head_raises = head_raises + self._files_raise = files_raise + self.diff_calls = [] + + def get_default_branch_head(self, *, token, git_domain, repo_name): + if self._head_raises: + raise RuntimeError("provider is down") + return self._head + + def get_changed_files(self, *, token, git_domain, repo_name, base, head): + self.diff_calls.append((base, head)) + if self._files_raise: + raise RuntimeError("compare failed") + return self._files + + +def _with(provider, token="t0ken"): + """Patch the provider lookup and credentials this module reaches for.""" + return ( + patch( + "app.services.knowledge.code_wiki_repo_state.provider_for", + return_value=provider, + ), + patch( + "app.services.knowledge.code_wiki_repo_state.get_user_git_info", + return_value=({"token": token} if token else None), + ), + ) + + +def _read(test_db: Session, provider, *, token="t0ken", since=PUBLISHED): + provider_patch, git_patch = _with(provider, token) + with provider_patch, git_patch: + return read_repository_state( + test_db, user_id=1, source=SOURCE, since_commit=since + ) + + +# --- the happy path --------------------------------------------------------- + + +def test_the_default_branch_head_is_read(test_db: Session): + state = _read(test_db, FakeProvider(), since="") + + assert state.head_commit == HEAD + assert state.branch == "main" + + +def test_the_diff_is_read_against_the_published_commit(test_db: Session): + provider = FakeProvider( + files=[ + {"path": "backend/app/main.py", "status": "M"}, + {"path": "backend/app/new.py", "status": "A"}, + ] + ) + + state = _read(test_db, provider) + + assert provider.diff_calls == [(PUBLISHED, HEAD)] + assert [change.path for change in state.changed_paths] == [ + "backend/app/main.py", + "backend/app/new.py", + ] + assert state.changed_paths[1].is_structural_move + + +def test_a_first_run_asks_for_no_diff(test_db: Session): + """There is nothing to compare against, and asking would only cost a call.""" + provider = FakeProvider() + + state = _read(test_db, provider, since="") + + assert provider.diff_calls == [] + assert state.changed_paths is None + + +def test_a_repository_already_at_the_documented_commit_has_an_empty_diff( + test_db: Session, +): + """Empty, not unknown. Unknown would rebuild a wiki that is already current.""" + provider = FakeProvider(head={"branch": "main", "commit": PUBLISHED}) + + state = _read(test_db, provider) + + assert state.head_commit == PUBLISHED + assert state.changed_paths == () + assert provider.diff_calls == [] + + +# --- everything that cannot be answered ------------------------------------- + + +def test_an_unreachable_provider_leaves_the_state_unknown(test_db: Session): + state = _read(test_db, FakeProvider(head_raises=True)) + + assert state == RepositoryState() + + +def test_a_missing_credential_leaves_the_state_unknown(test_db: Session): + state = _read(test_db, FakeProvider(), token="") + + assert state == RepositoryState() + + +def test_an_unsupported_platform_leaves_the_state_unknown(test_db: Session): + state = _read(test_db, None) + + assert state == RepositoryState() + + +def test_a_head_that_cannot_be_read_stops_before_the_diff(test_db: Session): + provider = FakeProvider(head={"branch": "main", "commit": ""}) + + state = _read(test_db, provider) + + assert state.head_commit == "" + assert provider.diff_calls == [] + + +def test_a_failed_diff_is_unknown_rather_than_empty(test_db: Session): + """Reported empty, the run would be skipped and the changes never documented.""" + state = _read(test_db, FakeProvider(files_raise=True)) + + assert state.head_commit == HEAD + assert state.changed_paths is None + + +def test_a_diff_the_provider_would_not_complete_is_unknown(test_db: Session): + """The provider says ``None`` when it truncated or gave up; that must survive.""" + state = _read(test_db, FakeProvider(files=None)) + + assert state.head_commit == HEAD + assert state.changed_paths is None + + +def test_a_genuinely_empty_diff_is_kept_as_empty(test_db: Session): + """The commit moved but nothing changed in it — that is a skip, not a rebuild.""" + state = _read(test_db, FakeProvider(files=[])) + + assert state.changed_paths == () + + +# --- the platforms that must be able to answer ------------------------------ + + +@pytest.mark.parametrize("source_type", SUPPORTED_SOURCE_TYPES) +def test_every_supported_platform_can_report_repository_state(source_type: str): + """A platform a code wiki accepts but cannot read state for would silently + rebuild from scratch on every run.""" + provider = provider_for(source_type) + + assert provider is not None + assert hasattr(provider, "get_default_branch_head") + assert hasattr(provider, "get_changed_files") diff --git a/backend/tests/services/knowledge/test_code_wiki_run_mode.py b/backend/tests/services/knowledge/test_code_wiki_run_mode.py new file mode 100644 index 0000000000..14c5c65590 --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_run_mode.py @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for code wiki run mode selection. + +The mode decides whether a run's reported page set may be used to delete pages, so +these tests guard against the failure that would delete a whole wiki. +""" + +from app.services.knowledge.code_wiki_run_mode import ( + ChangedPath, + RunMode, + RunModeDecision, + RunModePolicy, + decide_run_mode, +) + +HEAD = "aaaaaaa" +PREVIOUS = "bbbbbbb" + + +def _edits(count: int) -> list[ChangedPath]: + return [ChangedPath(f"src/module_{i}.py", "M") for i in range(count)] + + +def test_first_run_rebuilds_everything(): + decision = decide_run_mode(head_commit=HEAD, last_commit=None) + + assert decision.mode is RunMode.FULL + assert decision.seeds_from_published is False + + +def test_unchanged_repository_is_skipped(): + decision = decide_run_mode(head_commit=HEAD, last_commit=HEAD) + + assert decision.mode is RunMode.SKIP + assert decision.seeds_from_published is False + + +def test_small_change_stays_incremental(): + decision = decide_run_mode( + head_commit=HEAD, last_commit=PREVIOUS, changed_paths=_edits(3) + ) + + assert decision.mode is RunMode.INCREMENTAL + + +def test_an_incremental_run_must_be_seeded(): + """An incremental run only revises the pages its diff affects. + + Without a seed the version would hold just those pages, and the projection — + which compares complete snapshots — would read every untouched page as an orphan + and delete it. That is the whole wiki minus a handful. + """ + decision = decide_run_mode( + head_commit=HEAD, last_commit=PREVIOUS, changed_paths=_edits(1) + ) + + assert decision.mode is RunMode.INCREMENTAL + assert decision.seeds_from_published is True + + +def test_a_full_rebuild_starts_from_an_empty_version(): + assert ( + decide_run_mode(head_commit=HEAD, last_commit=None).seeds_from_published + is False + ) + + +def test_unknown_diff_rebuilds_rather_than_guessing(): + decision = decide_run_mode( + head_commit=HEAD, last_commit=PREVIOUS, changed_paths=None + ) + + assert decision.mode is RunMode.FULL + assert "unknown" in decision.reason + + +def test_commit_moved_but_nothing_documented_changed_is_skipped(): + decision = decide_run_mode(head_commit=HEAD, last_commit=PREVIOUS, changed_paths=[]) + + assert decision.mode is RunMode.SKIP + + +def test_dependency_manifest_change_forces_a_rebuild(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=[ChangedPath("backend/pyproject.toml", "M")], + ) + + assert decision.mode is RunMode.FULL + assert "manifest" in decision.reason + + +def test_new_top_level_module_forces_a_rebuild(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=[ChangedPath("gateway/main.go", "A")], + previous_top_level_dirs=frozenset({"backend", "frontend"}), + current_top_level_dirs=frozenset({"backend", "frontend", "gateway"}), + ) + + assert decision.mode is RunMode.FULL + assert "gateway" in decision.reason + + +def test_removed_top_level_module_forces_a_rebuild(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=[ChangedPath("legacy/app.py", "D")], + previous_top_level_dirs=frozenset({"backend", "legacy"}), + current_top_level_dirs=frozenset({"backend"}), + ) + + assert decision.mode is RunMode.FULL + assert "legacy" in decision.reason + + +def test_unknown_top_level_dirs_skip_that_check(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(2), + previous_top_level_dirs=None, + current_top_level_dirs=frozenset({"backend"}), + ) + + assert decision.mode is RunMode.INCREMENTAL + + +def test_too_many_changed_files_forces_a_rebuild(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(51), + policy=RunModePolicy(max_changed_files=50), + ) + + assert decision.mode is RunMode.FULL + assert "over the limit" in decision.reason + + +def test_large_share_of_the_repository_forces_a_rebuild(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(30), + total_source_files=100, + policy=RunModePolicy(max_changed_files=50, max_changed_ratio=0.25), + ) + + assert decision.mode is RunMode.FULL + assert "of files changed" in decision.reason + + +def test_accumulated_incremental_runs_force_a_periodic_rebuild(): + """Restructuring orphans are invisible to incremental runs, so full runs recur.""" + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(2), + incrementals_since_full=10, + policy=RunModePolicy(max_incrementals_since_full=10), + ) + + assert decision.mode is RunMode.FULL + assert "incremental runs since" in decision.reason + + +def test_age_since_last_full_run_forces_a_periodic_rebuild(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(2), + days_since_full=31, + policy=RunModePolicy(max_days_since_full=30), + ) + + assert decision.mode is RunMode.FULL + assert "days since" in decision.reason + + +def test_skip_takes_precedence_over_a_due_periodic_rebuild(): + """There is nothing to rebuild when the repository has not moved.""" + decision = decide_run_mode( + head_commit=HEAD, + last_commit=HEAD, + incrementals_since_full=99, + days_since_full=999, + ) + + assert decision.mode is RunMode.SKIP + + +def test_a_removed_module_is_caught_even_when_the_diff_looks_empty(): + """The diff may be filtered to documented file types while the tree is not. + + Removing a module of protos or assets can then show up only in the directory sets, + and skipping first would orphan its pages with nothing left to catch it. + """ + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=[], + previous_top_level_dirs=frozenset({"backend", "protos"}), + current_top_level_dirs=frozenset({"backend"}), + ) + + assert decision.mode is RunMode.FULL + assert "protos" in decision.reason + + +def test_many_added_or_removed_files_force_a_rebuild_sooner_than_edits(): + """Files appearing and disappearing reshape the wiki more than edits do.""" + moves = [ChangedPath(f"src/new_{i}.py", "A") for i in range(16)] + + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=moves, + policy=RunModePolicy(max_changed_files=50, max_structural_moves=15), + ) + + assert decision.mode is RunMode.FULL + assert "added, removed or renamed" in decision.reason + + +def test_the_same_number_of_plain_edits_stays_incremental(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(16), + policy=RunModePolicy(max_changed_files=50, max_structural_moves=15), + ) + + assert decision.mode is RunMode.INCREMENTAL + + +def test_a_mode_stored_as_a_plain_string_still_seeds(): + """Modes survive task payloads as strings; identity comparison would fail. + + Getting this wrong leaves an incremental version unseeded, which the projection + then treats as a complete snapshot containing only the revised pages. + """ + assert RunModeDecision("incremental", "restored").seeds_from_published is True + assert RunModeDecision("full", "restored").seeds_from_published is False + + +def test_changed_path_recognises_structural_moves(): + assert ChangedPath("a.py", "A").is_structural_move is True + assert ChangedPath("a.py", "D").is_structural_move is True + assert ChangedPath("a.py", "R100").is_structural_move is True + assert ChangedPath("a.py", "M").is_structural_move is False diff --git a/backend/tests/services/knowledge/test_code_wiki_runner.py b/backend/tests/services/knowledge/test_code_wiki_runner.py new file mode 100644 index 0000000000..b5a1e80c1f --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_runner.py @@ -0,0 +1,524 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for driving a run from a knowledge base to a task and back. + +The pieces either side of the runner are covered on their own, so what is asserted +here is the wiring: that the agent gets instructions matching the mode the version +store chose, that a run which cannot get a task does not sit RUNNING and block the +wiki, and that the commit the agent reports is the one the next run compares against. +""" + +from dataclasses import dataclass, field +from datetime import datetime + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.user import User +from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.schemas.task import TaskCreate +from app.services.knowledge.code_wiki_generation import published_commit +from app.services.knowledge.code_wiki_publisher import published_generation_id +from app.services.knowledge.code_wiki_run_mode import ChangedPath +from app.services.knowledge.code_wiki_runner import ( + CodeWikiRunError, + finish_run, + is_code_wiki_generation, + source_of, + start_run, +) +from app.services.knowledge.code_wiki_version_store import set_page_path + +HEAD = "aaaaaaa" +NEXT_HEAD = "bbbbbbb" + +SOURCE = { + "sourceType": "github", + "sourceUrl": "https://github.com/wecode-ai/Wegent.git", + "sourceDomain": "github.com", + "projectName": "wecode-ai/Wegent", +} + + +@dataclass +class FakeTasks: + """Stands in for the task service, capturing what the agent would be sent.""" + + created: list[TaskCreate] = field(default_factory=list) + next_id: int = 500 + fails: bool = False + + def create_task_id(self, db, user_id: int) -> int: + self.next_id += 1 + return self.next_id + + def create_task_or_append(self, *, db, obj_in: TaskCreate, user, task_id): + if self.fails: + raise RuntimeError("no executor available") + self.created.append(obj_in) + return {"id": task_id} + + @property + def prompt(self) -> str: + return self.created[-1].prompt + + +@pytest.fixture +def tasks(monkeypatch, test_db: Session, test_user: User) -> FakeTasks: + """Replace team lookup and task creation, which reach outside this unit.""" + from app.services.adapters import task_kinds, team_kinds + + team = Kind( + kind="Team", + name="code-wiki-team", + namespace="default", + user_id=test_user.id, + json={"spec": {"description": "code wiki"}}, + is_active=True, + ) + test_db.add(team) + test_db.flush() + + fake = FakeTasks() + monkeypatch.setattr( + team_kinds.team_kinds_service, + "get_team_by_name_and_namespace", + lambda **kwargs: team, + ) + monkeypatch.setattr( + task_kinds.task_kinds_service, "create_task_id", fake.create_task_id + ) + monkeypatch.setattr( + task_kinds.task_kinds_service, + "create_task_or_append", + fake.create_task_or_append, + ) + return fake + + +@pytest.fixture +def knowledge_base(test_db: Session, test_user: User) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-runner", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "wiki", "kbType": "code_wiki", "source": SOURCE}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +@dataclass +class FakeEffects: + written: list[str] = field(default_factory=list) + next_id: int = 8000 + + def _write(self, *, filename: str, content: str) -> int: + self.next_id += 1 + self.written.append(filename) + return self.next_id + + +@pytest.fixture +def no_side_effects(monkeypatch) -> FakeEffects: + """Publish without touching attachment storage, the index or the queue.""" + from app.services.knowledge import code_wiki_runner + from app.services.knowledge.code_wiki_projection import ProjectionSideEffects + + fake = FakeEffects() + monkeypatch.setattr( + code_wiki_runner, + "build_projection_side_effects", + lambda db, *, knowledge_base, user: ProjectionSideEffects( + write_attachment=fake._write, + delete_attachment=lambda _: None, + delete_rag_document=lambda _: None, + enqueue_reindex=lambda _: None, + ), + ) + return fake + + +def _write_page(test_db: Session, generation: WikiGeneration, path: str): + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path, + content="body", + parent_id=0, + ) + set_page_path(entry, path) + test_db.add(entry) + test_db.flush() + + +# --- what the wiki is bound to --------------------------------------------- + + +def test_a_knowledge_base_that_is_not_a_code_wiki_cannot_be_generated( + test_db: Session, test_user: User +): + notebook = Kind( + kind="KnowledgeBase", + name="kb-notebook", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "notes", "kbType": "notebook"}}, + is_active=True, + ) + test_db.add(notebook) + test_db.flush() + + with pytest.raises(CodeWikiRunError, match="not a code wiki"): + source_of(notebook) + + +def test_a_code_wiki_with_no_repository_is_refused_before_a_task_exists( + test_db: Session, test_user: User +): + """Otherwise it surfaces much later as a task cloning an empty URL.""" + unbound = Kind( + kind="KnowledgeBase", + name="kb-unbound", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(unbound) + test_db.flush() + + with pytest.raises(CodeWikiRunError, match="no source repository"): + source_of(unbound) + + +# --- starting a run --------------------------------------------------------- + + +def test_a_first_run_sends_the_agent_a_task_for_the_bound_repository( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + assert started.started + created = tasks.created[0] + assert created.git_url == SOURCE["sourceUrl"] + assert created.git_repo == SOURCE["projectName"] + assert created.git_domain == SOURCE["sourceDomain"] + assert created.source == "code_wiki" + + +def test_the_task_clones_the_default_branch( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + """A pinned branch would document whatever it was at the day the wiki was made.""" + start_run(test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD) + + assert tasks.created[0].branch_name == "" + + +def test_the_run_is_reachable_from_the_task_it_created( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + assert started.generation.task_id == started.task_id + assert started.task_id > 0 + + +def test_a_first_run_gets_the_full_rebuild_instructions( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + start_run(test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD) + + assert "begins empty" in tasks.prompt + assert "copy of the published" not in tasks.prompt + + +def test_the_prompt_carries_the_generation_the_agent_must_write_into( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + assert str(started.generation.id) in tasks.prompt + + +def test_an_unchanged_repository_starts_no_task_at_all( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + assert not started.started + assert started.mode == "skip" + assert len(tasks.created) == 1 + + +def test_an_incremental_run_tells_the_agent_which_pages_already_exist( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + """Without them the agent cannot tell an update from a new page.""" + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + + start_run( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + ) + + assert "copy of the published" in tasks.prompt + assert "architecture/backend" in tasks.prompt + + +def test_an_incremental_run_tells_the_agent_what_changed( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + + start_run( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + ) + + assert "src/one.py" in tasks.prompt + assert HEAD in tasks.prompt and NEXT_HEAD in tasks.prompt + + +def test_a_run_whose_task_cannot_be_created_does_not_block_the_wiki( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + """Left RUNNING it would refuse to regenerate until the six-hour sweep.""" + tasks.fails = True + + with pytest.raises(CodeWikiRunError, match="Could not start the wiki task"): + start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + generation = test_db.query(WikiGeneration).one() + assert generation.status == WikiGenerationStatus.FAILED + assert "task creation failed" in generation.ext["errorMessage"] + + tasks.fails = False + retry = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + assert retry.started + + +# --- finishing a run -------------------------------------------------------- + + +def _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) -> WikiGeneration: + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + _write_page(test_db, started.generation, "index") + _write_page(test_db, started.generation, "architecture/backend") + finish_run(test_db, generation=started.generation, succeeded=True, head_commit=HEAD) + return started.generation + + +def test_a_successful_run_publishes_into_the_knowledge_base( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + generation = _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + + assert published_generation_id(knowledge_base) == generation.id + + +def test_the_commit_the_agent_reports_is_what_the_next_run_compares_against( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + """The trigger only knew what it was told; the agent read the working tree.""" + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit="" + ) + _write_page(test_db, started.generation, "index") + + finish_run( + test_db, generation=started.generation, succeeded=True, head_commit=NEXT_HEAD + ) + + assert published_commit(test_db, knowledge_base) == NEXT_HEAD + + +def test_a_failed_run_publishes_nothing( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + first = _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + + started = start_run( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + ) + result = finish_run( + test_db, + generation=started.generation, + succeeded=False, + error_message="model timed out", + ) + + assert result is None + assert published_generation_id(knowledge_base) == first.id + + +def test_a_run_belonging_to_no_knowledge_base_is_not_a_code_wiki_run( + test_db: Session, test_user: User +): + """The legacy wiki writes through the same API and must keep its own behaviour.""" + legacy = WikiGeneration( + project_id=1, + kind_id=0, + user_id=test_user.id, + task_id=0, + team_id=1, + generation_type="full", + source_snapshot={}, + status=WikiGenerationStatus.RUNNING, + completed_at=datetime(1970, 1, 1), + ) + test_db.add(legacy) + test_db.flush() + + assert not is_code_wiki_generation(test_db, legacy) + + +# --- resolving the repository's state when the caller did not supply it ------ + + +def _repository_at(monkeypatch, head: str, changed=None): + """Answer as the provider would, without reaching one.""" + from app.services.knowledge import code_wiki_runner + from app.services.knowledge.code_wiki_repo_state import RepositoryState + + monkeypatch.setattr( + code_wiki_runner, + "read_repository_state", + lambda db, *, user_id, source, since_commit: RepositoryState( + head_commit=head, branch="main", changed_paths=changed + ), + ) + + +def test_an_unchanged_repository_is_recognised_without_being_told( + monkeypatch, + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + """The whole point of reading HEAD: a schedule over a quiet repository must + cost one comparison, not a full pass through the model.""" + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + _repository_at(monkeypatch, HEAD, changed=()) + + started = start_run(test_db, knowledge_base=knowledge_base, user=test_user) + + assert not started.started + assert started.mode == "skip" + assert len(tasks.created) == 1 + + +def test_a_changed_repository_is_updated_incrementally_without_being_told( + monkeypatch, + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + _repository_at(monkeypatch, NEXT_HEAD, changed=(ChangedPath("src/one.py", "M"),)) + + started = start_run(test_db, knowledge_base=knowledge_base, user=test_user) + + assert started.mode == "incremental" + assert "src/one.py" in tasks.prompt + + +def test_a_repository_that_cannot_be_read_falls_back_to_a_rebuild( + monkeypatch, + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + """Expensive, and the only safe answer: an unknown diff might be anything.""" + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + _repository_at(monkeypatch, "", changed=None) + + started = start_run(test_db, knowledge_base=knowledge_base, user=test_user) + + assert started.mode == "full" + + +def test_a_supplied_commit_is_not_second_guessed( + monkeypatch, + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, +): + """A caller that knows the commit — a webhook, a test — must be believed.""" + from app.services.knowledge import code_wiki_runner + + def refuse(*args, **kwargs): + raise AssertionError("the provider must not be consulted") + + monkeypatch.setattr(code_wiki_runner, "read_repository_state", refuse) + + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + assert started.started diff --git a/backend/tests/services/knowledge/test_code_wiki_source.py b/backend/tests/services/knowledge/test_code_wiki_source.py new file mode 100644 index 0000000000..142cf5f037 --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_source.py @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the repository access gate guarding code wiki creation.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from app.services.knowledge.code_wiki_source import ( + SourceAccessDenied, + SourceRepository, + assert_user_can_read_source, +) + +GITHUB_SOURCE = SourceRepository.from_url( + "github", "https://github.com/wecode-ai/Wegent.git" +) +GITLAB_SOURCE = SourceRepository.from_url( + "gitlab", "https://gitlab.example.com/team/app.git" +) + + +def _granted(): + return {"has_access": True, "access_level_name": "Reporter"} + + +def test_access_granted_returns_provider_details(): + provider = MagicMock() + provider.check_user_project_access.return_value = _granted() + + with ( + patch( + "app.services.knowledge.code_wiki_source.get_user_git_info", + return_value={"type": "github", "token": "t0ken"}, + ), + patch( + "app.services.knowledge.code_wiki_source.provider_for", + return_value=provider, + ), + ): + result = assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + assert result["has_access"] is True + provider.check_user_project_access.assert_called_once_with( + token="t0ken", git_domain="github.com", repo_name="wecode-ai/Wegent" + ) + + +def test_gitlab_is_identified_by_project_id(): + provider = MagicMock() + provider.check_user_project_access.return_value = _granted() + + with ( + patch( + "app.services.knowledge.code_wiki_source.get_user_git_info", + return_value={"type": "gitlab", "token": "t0ken"}, + ), + patch( + "app.services.knowledge.code_wiki_source.provider_for", + return_value=provider, + ), + ): + assert_user_can_read_source(MagicMock(), 1, GITLAB_SOURCE) + + provider.check_user_project_access.assert_called_once_with( + token="t0ken", git_domain="gitlab.example.com", project_id="team/app" + ) + + +def test_missing_credentials_are_denied(): + with patch( + "app.services.knowledge.code_wiki_source.get_user_git_info", + return_value=None, + ): + with pytest.raises(SourceAccessDenied, match="No credentials configured"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_no_repository_access_is_denied(): + provider = MagicMock() + provider.check_user_project_access.return_value = { + "has_access": False, + "error": "Not a member", + } + + with ( + patch( + "app.services.knowledge.code_wiki_source.get_user_git_info", + return_value={"type": "github", "token": "t0ken"}, + ), + patch( + "app.services.knowledge.code_wiki_source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="do not have read access"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_provider_error_denies_rather_than_allows(): + """An unreachable provider must not become an open door.""" + provider = MagicMock() + provider.check_user_project_access.side_effect = RuntimeError("gateway timeout") + + with ( + patch( + "app.services.knowledge.code_wiki_source.get_user_git_info", + return_value={"type": "github", "token": "t0ken"}, + ), + patch( + "app.services.knowledge.code_wiki_source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="Could not verify access"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_unsupported_repository_type_is_refused_at_construction(): + with pytest.raises(SourceAccessDenied, match="Unsupported repository type"): + SourceRepository.from_url("svn", "svn://example.com/app") + + +def test_the_checked_repository_is_the_one_that_will_be_cloned(): + """Host and project come from the URL, so they cannot disagree with it. + + Accepting them separately would let a caller pass a repository they can read while + storing a URL pointing at a private one — the gate would approve one repository and + the wiki would be built from another. + """ + source = SourceRepository.from_url( + "gitlab", "https://gitlab.internal.corp/secret/payroll.git" + ) + + assert source.source_domain == "gitlab.internal.corp" + assert source.project_name == "secret/payroll" + assert source.source_url == "https://gitlab.internal.corp/secret/payroll.git" + + +def test_nested_group_paths_are_preserved(): + source = SourceRepository.from_url( + "gitlab", "https://gitlab.example.com/weibo_rd/common/wecode/wegent.git" + ) + + assert source.project_name == "weibo_rd/common/wecode/wegent" + + +def test_credentials_embedded_in_a_url_are_not_stored(): + """The stored URL is visible to everyone who can read the knowledge base.""" + source = SourceRepository.from_url( + "github", "https://ghp_secrettoken@github.com/owner/repo.git" + ) + + assert "ghp_secrettoken" not in source.source_url + assert source.source_url == "https://github.com/owner/repo.git" + + +def test_an_unusable_url_is_refused(): + with pytest.raises(SourceAccessDenied, match="Could not read a repository"): + SourceRepository.from_url("github", "not-a-url") + + +def test_credentials_for_another_platform_are_refused(): + """Asking one provider about another's repository gives a meaningless answer.""" + with patch( + "app.services.knowledge.code_wiki_source.get_user_git_info", + return_value={"type": "gitlab", "token": "t0ken"}, + ): + with pytest.raises(SourceAccessDenied, match="configured as 'gitlab'"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_source_survives_a_round_trip_through_the_spec(): + assert SourceRepository.from_spec(GITLAB_SOURCE.to_spec()) == GITLAB_SOURCE + + +def test_a_knowledge_base_without_a_source_reads_as_none(): + assert SourceRepository.from_spec(None) is None + assert SourceRepository.from_spec({}) is None + + +# --- what a repository URL may name ----------------------------------------- + + +@pytest.mark.parametrize( + "url", + [ + "https://169.254.169.254/owner/repo.git", + "https://[fe80::1]/owner/repo.git", + "https://127.0.0.1/owner/repo.git", + "https://localhost/owner/repo.git", + # Written the long way round. The first two rely on the standard library + # folding IPv4-mapped forms into is_loopback/is_link_local, which is worth + # pinning because it is not this module's behaviour to keep. The last two + # are neither loopback nor link-local, and still reach the local host. + "https://[::ffff:127.0.0.1]/owner/repo.git", + "https://[::ffff:169.254.169.254]/owner/repo.git", + "https://0.0.0.0/owner/repo.git", + "https://[::]/owner/repo.git", + ], +) +def test_a_host_that_is_never_a_git_server_is_refused(url: str): + """Binding a repository makes the server fetch from the host the URL names, + carrying the caller's token. The metadata endpoint is link-local and answers + unauthenticated requests with instance credentials.""" + with pytest.raises(SourceAccessDenied, match="not a reachable repository host"): + SourceRepository.from_url("gitlab", url) + + +@pytest.mark.parametrize( + "url", + [ + "https://gitlab.internal.example.com/owner/repo.git", + "https://10.0.0.7/owner/repo.git", + "https://192.168.1.10/owner/repo.git", + ], +) +def test_a_self_hosted_host_on_an_internal_network_is_still_allowed(url: str): + """The normal deployment here. Blocking private ranges would break the product + to close a much smaller hole than the one above.""" + source = SourceRepository.from_url("gitlab", url) + + assert source.project_name == "owner/repo" + + +@pytest.mark.parametrize( + "git_info", + [ + None, + {}, + {"type": "github"}, + {"type": "github", "token": ""}, + ], + ids=["no-record", "empty-record", "no-token-key", "blank-token"], +) +def test_a_credential_record_without_a_usable_token_denies_access(git_info): + """Every shape of "configured but unusable" has to deny. A record that exists + but carries no token would otherwise reach the provider with ``None``.""" + with patch( + "app.services.knowledge.code_wiki_source.get_user_git_info", + return_value=git_info, + ): + with pytest.raises(SourceAccessDenied, match="No credentials configured"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) diff --git a/backend/tests/services/knowledge/test_code_wiki_submit_to_publish.py b/backend/tests/services/knowledge/test_code_wiki_submit_to_publish.py new file mode 100644 index 0000000000..e48a5bf978 --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_submit_to_publish.py @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the agent's final submission reaching the knowledge base. + +The agent has one channel back to the server, and the last thing it sends through it +is a summary saying the run finished. For a code wiki that has to do more than record +a status: the version is offered to the publish gate and projected into the knowledge +base. These tests cover that handoff, including the case it must not affect — the +legacy wiki, which writes through the same endpoint and only records a status. +""" + +from dataclasses import dataclass, field +from datetime import datetime + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import KnowledgeDocument +from app.models.user import User +from app.models.wiki import ( + WikiContent, + WikiGeneration, + WikiGenerationStatus, + WikiGenerationType, +) +from app.schemas.wiki import ( + WikiContentSection, + WikiContentSummary, + WikiContentWriteRequest, +) +from app.services.knowledge.code_wiki_generation import published_commit +from app.services.knowledge.code_wiki_publish_gate import PUBLISH_GATE_EXT_KEY +from app.services.knowledge.code_wiki_publisher import published_generation_id +from app.services.knowledge.code_wiki_version_store import set_page_path +from app.services.wiki_service import WikiService + +HEAD = "aaaaaaa" + + +@dataclass +class FakeEffects: + written: list[str] = field(default_factory=list) + next_id: int = 9000 + + def _write(self, *, filename: str, content: str) -> int: + self.next_id += 1 + self.written.append(filename) + return self.next_id + + +@pytest.fixture +def no_side_effects(monkeypatch) -> FakeEffects: + from app.services.knowledge import code_wiki_runner + from app.services.knowledge.code_wiki_projection import ProjectionSideEffects + + fake = FakeEffects() + monkeypatch.setattr( + code_wiki_runner, + "build_projection_side_effects", + lambda db, *, knowledge_base, user: ProjectionSideEffects( + write_attachment=fake._write, + delete_attachment=lambda _: None, + delete_rag_document=lambda _: None, + enqueue_reindex=lambda _: None, + ), + ) + return fake + + +@pytest.fixture +def knowledge_base(test_db: Session, test_user: User) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-submit", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def _generation(test_db: Session, user: User, kind_id: int) -> WikiGeneration: + record = WikiGeneration( + project_id=0, + kind_id=kind_id, + user_id=user.id, + task_id=1, + team_id=1, + generation_type=WikiGenerationType.FULL, + source_snapshot={}, + status=WikiGenerationStatus.RUNNING, + completed_at=datetime(1970, 1, 1), + ) + test_db.add(record) + test_db.flush() + return record + + +def _seed_page(test_db: Session, generation: WikiGeneration, path: str): + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path, + content="body", + parent_id=0, + ) + set_page_path(entry, path) + test_db.add(entry) + test_db.flush() + + +def _submit(db: Session, generation: WikiGeneration, **summary_fields): + WikiService().save_generation_contents( + db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[], + summary=WikiContentSummary(**summary_fields), + ), + ) + + +def _documents(db: Session, kind_id: int) -> list[KnowledgeDocument]: + return ( + db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == kind_id).all() + ) + + +def test_completing_a_code_wiki_run_publishes_it_into_the_knowledge_base( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + generation = _generation(test_db, test_user, knowledge_base.id) + _seed_page(test_db, generation, "index") + + _submit(test_db, generation, status="COMPLETED", head_commit=HEAD) + + assert published_generation_id(knowledge_base) == generation.id + assert [doc.name for doc in _documents(test_db, knowledge_base.id)] == ["index"] + + +def test_a_published_run_ends_completed_rather_than_running( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + generation = _generation(test_db, test_user, knowledge_base.id) + _seed_page(test_db, generation, "index") + + _submit(test_db, generation, status="COMPLETED", head_commit=HEAD) + + test_db.refresh(generation) + assert generation.status == WikiGenerationStatus.COMPLETED + + +def test_a_version_the_gate_refuses_does_not_replace_the_published_one( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + """This is why the agent's word is not enough on its own: it says the run + succeeded, and the gate still decides whether the result may go live.""" + first = _generation(test_db, test_user, knowledge_base.id) + for path in ("index", "one", "two", "three"): + _seed_page(test_db, first, path) + _submit(test_db, first, status="COMPLETED", head_commit=HEAD) + + collapsed = _generation(test_db, test_user, knowledge_base.id) + _seed_page(test_db, collapsed, "index") + + _submit(test_db, collapsed, status="COMPLETED", head_commit="bbbbbbb") + + test_db.refresh(collapsed) + assert published_generation_id(knowledge_base) == first.id + assert collapsed.ext[PUBLISH_GATE_EXT_KEY]["result"] == "rejected" + assert {doc.name for doc in _documents(test_db, knowledge_base.id)} == { + "index", + "one", + "two", + "three", + } + + +def test_the_reported_commit_reaches_the_published_version( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + generation = _generation(test_db, test_user, knowledge_base.id) + _seed_page(test_db, generation, "index") + + _submit(test_db, generation, status="COMPLETED", head_commit=HEAD) + + assert published_commit(test_db, knowledge_base) == HEAD + + +def test_a_failed_submission_publishes_nothing( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + generation = _generation(test_db, test_user, knowledge_base.id) + _seed_page(test_db, generation, "index") + + _submit(test_db, generation, status="FAILED", error_message="ran out of budget") + + test_db.refresh(generation) + assert generation.status == WikiGenerationStatus.FAILED + assert published_generation_id(knowledge_base) == 0 + assert _documents(test_db, knowledge_base.id) == [] + + +def test_pages_sent_with_the_final_summary_are_published_too( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + """The agent may finish in one call, so the write must be committed first.""" + generation = _generation(test_db, test_user, knowledge_base.id) + + WikiService().save_generation_contents( + test_db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[ + WikiContentSection( + type="chapter", + title="Backend Architecture", + content="body", + path="architecture/backend", + ) + ], + summary=WikiContentSummary(status="COMPLETED", head_commit=HEAD), + ), + ) + + # Named for the path's leaf, not the title: the path is the identity, and a + # reworded heading must not move the document. + assert [doc.name for doc in _documents(test_db, knowledge_base.id)] == ["backend"] + + +def test_a_legacy_generation_only_records_its_status( + test_db: Session, test_user: User, no_side_effects: FakeEffects +): + """Nothing to publish into, so the old behaviour has to survive untouched.""" + generation = _generation(test_db, test_user, kind_id=0) + _seed_page(test_db, generation, "index") + + _submit(test_db, generation, status="COMPLETED") + + test_db.refresh(generation) + assert generation.status == WikiGenerationStatus.COMPLETED + assert no_side_effects.written == [] diff --git a/backend/tests/services/knowledge/test_code_wiki_version_store.py b/backend/tests/services/knowledge/test_code_wiki_version_store.py new file mode 100644 index 0000000000..6547d365a5 --- /dev/null +++ b/backend/tests/services/knowledge/test_code_wiki_version_store.py @@ -0,0 +1,448 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the code wiki version store. + +The store exists so that a failed or half-finished run stays invisible, so most of +these tests are about what survives a run going wrong. +""" + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy.orm import Session + +from app.models.wiki import ( + WikiContent, + WikiGeneration, + WikiGenerationStatus, + WikiGenerationType, +) +from app.services.knowledge.code_wiki_page_path import InvalidPagePath +from app.services.knowledge.code_wiki_version_store import ( + STALE_RUN_AFTER_HOURS, + apply_retention, + page_path_of, + reclaim_stale_generations, + remove_page, + seed_from_published, + set_page_path, +) + +KIND_ID = 77 +NOW = datetime(2026, 7, 31, 12, 0, 0) + + +def _generation( + db: Session, + *, + status: WikiGenerationStatus = WikiGenerationStatus.COMPLETED, + created_at: datetime = NOW, + updated_at: datetime = NOW, + kind_id: int = KIND_ID, +) -> WikiGeneration: + generation = WikiGeneration( + project_id=1, + kind_id=kind_id, + user_id=1, + task_id=0, + team_id=1, + generation_type=WikiGenerationType.FULL, + source_snapshot={}, + status=status, + created_at=created_at, + updated_at=updated_at, + completed_at=created_at, + ) + db.add(generation) + db.flush() + return generation + + +def _page( + db: Session, generation: WikiGeneration, path: str, content: str = "body" +) -> WikiContent: + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path.rsplit("/", 1)[-1], + content=content, + parent_id=0, + ) + set_page_path(entry, path) + db.add(entry) + db.flush() + return entry + + +def _paths(db: Session, generation_id: int) -> set[str]: + return { + page_path_of(entry) + for entry in db.query(WikiContent) + .filter(WikiContent.generation_id == generation_id) + .all() + } + + +# --- seeding --------------------------------------------------------------- + + +def test_seeding_copies_the_published_version_verbatim(test_db: Session): + published = _generation(test_db) + _page(test_db, published, "index", "overview") + _page(test_db, published, "architecture/backend", "backend notes") + target = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + outcome = seed_from_published( + test_db, + target_generation_id=target.id, + published_generation_id=published.id, + ) + + assert outcome.seeded + assert outcome.copied_pages == 2 + assert _paths(test_db, target.id) == {"index", "architecture/backend"} + + +def test_seeded_content_is_byte_identical(test_db: Session): + """The projection skips pages whose hash is unchanged, which relies on this.""" + published = _generation(test_db) + _page(test_db, published, "index", "exact bytes") + target = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + seed_from_published( + test_db, + target_generation_id=target.id, + published_generation_id=published.id, + ) + + copied = ( + test_db.query(WikiContent).filter(WikiContent.generation_id == target.id).one() + ) + assert copied.content == "exact bytes" + + +def test_seeding_does_not_carry_over_parent_ids(test_db: Session): + """They name rows in the source version; copying them would dangle.""" + published = _generation(test_db) + parent = _page(test_db, published, "architecture") + child = _page(test_db, published, "architecture/backend") + child.parent_id = parent.id + test_db.flush() + target = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + seed_from_published( + test_db, + target_generation_id=target.id, + published_generation_id=published.id, + ) + + copied = ( + test_db.query(WikiContent).filter(WikiContent.generation_id == target.id).all() + ) + assert {entry.parent_id for entry in copied} == {0} + + +def test_seeding_twice_does_not_double_the_version(test_db: Session): + """A retried scheduling attempt must not duplicate every page.""" + published = _generation(test_db) + _page(test_db, published, "index") + target = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + seed_from_published( + test_db, + target_generation_id=target.id, + published_generation_id=published.id, + ) + second = seed_from_published( + test_db, + target_generation_id=target.id, + published_generation_id=published.id, + ) + + assert not second.seeded + assert len(_paths(test_db, target.id)) == 1 + + +def test_a_first_run_has_nothing_to_seed_from(test_db: Session): + target = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + outcome = seed_from_published( + test_db, target_generation_id=target.id, published_generation_id=0 + ) + + assert not outcome.seeded + assert _paths(test_db, target.id) == set() + + +# --- agent-declared deletion ---------------------------------------------- + + +def test_an_agent_can_drop_a_page_from_the_in_flight_version(test_db: Session): + generation = _generation(test_db, status=WikiGenerationStatus.RUNNING) + _page(test_db, generation, "index") + _page(test_db, generation, "modules/legacy") + + assert remove_page(test_db, generation_id=generation.id, path="modules/legacy") + + assert _paths(test_db, generation.id) == {"index"} + + +def test_removing_a_page_leaves_the_published_version_untouched(test_db: Session): + """Deletion is recoverable precisely because it lands in an unpublished version.""" + published = _generation(test_db) + _page(test_db, published, "modules/legacy") + draft = _generation(test_db, status=WikiGenerationStatus.RUNNING) + _page(test_db, draft, "modules/legacy") + + remove_page(test_db, generation_id=draft.id, path="modules/legacy") + + assert _paths(test_db, published.id) == {"modules/legacy"} + + +def test_removing_matches_the_page_regardless_of_case(test_db: Session): + generation = _generation(test_db, status=WikiGenerationStatus.RUNNING) + _page(test_db, generation, "Modules/Legacy") + + assert remove_page(test_db, generation_id=generation.id, path="modules/legacy") + + +def test_removing_an_unknown_page_reports_that_nothing_happened(test_db: Session): + generation = _generation(test_db, status=WikiGenerationStatus.RUNNING) + _page(test_db, generation, "index") + + assert not remove_page(test_db, generation_id=generation.id, path="nope") + + +def test_removing_rejects_a_malformed_path(test_db: Session): + generation = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + with pytest.raises(InvalidPagePath): + remove_page(test_db, generation_id=generation.id, path="../escape") + + +# --- stale run reclamation ------------------------------------------------- + + +def test_an_abandoned_run_is_failed_so_the_wiki_is_not_blocked(test_db: Session): + """A crashed worker would otherwise hold the wiki forever.""" + stuck = _generation( + test_db, + status=WikiGenerationStatus.RUNNING, + updated_at=NOW - timedelta(hours=STALE_RUN_AFTER_HOURS + 1), + ) + + reclaimed = reclaim_stale_generations(test_db, kind_id=KIND_ID, now=NOW) + + assert reclaimed == (stuck.id,) + assert stuck.status == WikiGenerationStatus.FAILED + + +def test_a_slow_but_live_run_is_left_alone(test_db: Session): + _generation( + test_db, + status=WikiGenerationStatus.RUNNING, + updated_at=NOW - timedelta(hours=STALE_RUN_AFTER_HOURS - 1), + ) + + assert reclaim_stale_generations(test_db, kind_id=KIND_ID, now=NOW) == () + + +def test_reclamation_ignores_finished_runs(test_db: Session): + _generation( + test_db, + status=WikiGenerationStatus.COMPLETED, + updated_at=NOW - timedelta(days=30), + ) + + assert reclaim_stale_generations(test_db, kind_id=KIND_ID, now=NOW) == () + + +def test_reclamation_is_scoped_to_one_knowledge_base(test_db: Session): + _generation( + test_db, + kind_id=KIND_ID + 1, + status=WikiGenerationStatus.RUNNING, + updated_at=NOW - timedelta(days=5), + ) + + assert reclaim_stale_generations(test_db, kind_id=KIND_ID, now=NOW) == () + + +# --- retention ------------------------------------------------------------- + + +def test_retention_keeps_the_newest_successful_versions(test_db: Session): + generations = [ + _generation(test_db, created_at=NOW - timedelta(days=index)) + for index in range(5) + ] + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=generations[0].id, + keep_successful=3, + now=NOW, + ) + + assert set(removed) == {generations[3].id, generations[4].id} + + +def test_retention_deletes_the_pages_of_a_collected_version(test_db: Session): + old = _generation(test_db, created_at=NOW - timedelta(days=10)) + _page(test_db, old, "index") + current = _generation(test_db, created_at=NOW) + + apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=current.id, + keep_successful=1, + now=NOW, + ) + + assert _paths(test_db, old.id) == set() + + +def test_the_published_version_survives_newer_versions_that_never_published( + test_db: Session, +): + """A rejected publish gate leaves a completed version that was never published. + + Enough of those push the published one out of the newest ``keep_successful``, + which would collect the only version there is to roll back to. Note that runs + which *fail* cannot cause this — they never enter the successful list at all. + """ + published = _generation(test_db, created_at=NOW - timedelta(days=9)) + for index in range(5): + _generation(test_db, created_at=NOW - timedelta(days=index)) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=published.id, + keep_successful=3, + now=NOW, + ) + + assert published.id not in removed + assert test_db.get(WikiGeneration, published.id) is not None + + +def test_the_published_version_survives_a_repository_that_never_changes( + test_db: Session, +): + """With no new commits there are no new versions, so every version ages out.""" + published = _generation(test_db, created_at=NOW - timedelta(days=400)) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=published.id, + max_age_days=90, + now=NOW, + ) + + assert removed == () + assert test_db.get(WikiGeneration, published.id) is not None + + +def test_retention_drops_versions_past_the_age_limit(test_db: Session): + stale = _generation(test_db, created_at=NOW - timedelta(days=200)) + current = _generation(test_db, created_at=NOW) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=current.id, + max_age_days=90, + now=NOW, + ) + + assert removed == (stale.id,) + + +def test_failed_versions_are_kept_briefly_for_investigation(test_db: Session): + recent = _generation( + test_db, + status=WikiGenerationStatus.FAILED, + created_at=NOW - timedelta(days=2), + ) + ancient = _generation( + test_db, + status=WikiGenerationStatus.FAILED, + created_at=NOW - timedelta(days=30), + ) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=0, + failed_retention_days=7, + now=NOW, + ) + + assert removed == (ancient.id,) + assert test_db.get(WikiGeneration, recent.id) is not None + + +def test_retention_never_collects_a_run_that_is_still_in_flight(test_db: Session): + """Its pages are being written to; deleting them would corrupt a live run. + + Reclamation normally fails an abandoned run long before it reaches this age, but + retention must not rely on that having happened. + """ + for status in (WikiGenerationStatus.PENDING, WikiGenerationStatus.RUNNING): + in_flight = _generation( + test_db, status=status, created_at=NOW - timedelta(days=30) + ) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=0, + failed_retention_days=7, + now=NOW, + ) + + assert in_flight.id not in removed + assert test_db.get(WikiGeneration, in_flight.id) is not None + + +def test_an_aware_reference_time_is_converted_rather_than_truncated(test_db: Session): + """Dropping the offset instead of converting shifts the cutoff by that offset. + + The version sits just inside the retention window once the +14:00 reference is + properly converted to UTC, and just outside it if the offset is merely discarded. + Anything further from the boundary passes either way and would prove nothing. + """ + _generation( + test_db, + status=WikiGenerationStatus.FAILED, + created_at=NOW - timedelta(days=7, hours=6), + ) + aware_now = NOW.replace(tzinfo=timezone(timedelta(hours=14))) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=0, + failed_retention_days=7, + now=aware_now, + ) + + assert removed == () + + +def test_retention_is_scoped_to_one_knowledge_base(test_db: Session): + other = _generation( + test_db, kind_id=KIND_ID + 1, created_at=NOW - timedelta(days=999) + ) + + removed = apply_retention( + test_db, kind_id=KIND_ID, published_generation_id=0, now=NOW + ) + + assert other.id not in removed diff --git a/backend/tests/services/knowledge/test_content_scope.py b/backend/tests/services/knowledge/test_content_scope.py new file mode 100644 index 0000000000..d55049a61a --- /dev/null +++ b/backend/tests/services/knowledge/test_content_scope.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for knowledge content query scopes and content ownership defaults.""" + +import pytest +from sqlalchemy.orm import Session + +from app.models.knowledge import ContentOrigin, KnowledgeDocument, KnowledgeFolder +from app.services.knowledge.content_scope import ( + CODE_TARGET_SOURCE_TYPE, + NO_FOLDER, + code_targets, + generated_folders, + generated_wiki_pages, + wiki_pages, +) + +KIND_ID = 4242 + + +def _add_document( + db: Session, + name: str, + *, + origin: str, + source_type: str = "text", + folder_id: int = 0, +) -> KnowledgeDocument: + document = KnowledgeDocument( + kind_id=KIND_ID, + name=name, + file_extension="md", + user_id=1, + origin=origin, + source_type=source_type, + folder_id=folder_id, + ) + db.add(document) + db.flush() + return document + + +def _add_folder(db: Session, name: str, *, origin: str) -> KnowledgeFolder: + folder = KnowledgeFolder(kind_id=KIND_ID, parent_id=0, name=name, origin=origin) + db.add(folder) + db.flush() + return folder + + +@pytest.fixture +def content(test_db: Session) -> dict[str, KnowledgeDocument]: + """A knowledge base holding every combination of ownership and target kind.""" + items = { + "generated_page": _add_document( + test_db, "architecture", origin=ContentOrigin.GENERATED.value + ), + "user_page": _add_document( + test_db, "our-gotchas", origin=ContentOrigin.USER.value + ), + "code_target": _add_document( + test_db, + "src/foo/bar.py", + origin=ContentOrigin.GENERATED.value, + source_type=CODE_TARGET_SOURCE_TYPE, + folder_id=NO_FOLDER, + ), + } + return items + + +def _names(query) -> set[str]: + return {document.name for document in query.all()} + + +def test_wiki_pages_excludes_code_targets(test_db: Session, content): + result = wiki_pages( + test_db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == KIND_ID) + ) + + assert _names(result) == {"architecture", "our-gotchas"} + + +def test_generated_wiki_pages_excludes_user_content_and_code_targets( + test_db: Session, content +): + result = generated_wiki_pages( + test_db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == KIND_ID) + ) + + assert _names(result) == {"architecture"} + + +def test_code_targets_returns_only_indexed_source_files(test_db: Session, content): + result = code_targets( + test_db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == KIND_ID) + ) + + assert _names(result) == {"src/foo/bar.py"} + + +def test_generated_folders_excludes_user_folders(test_db: Session): + _add_folder(test_db, "architecture", origin=ContentOrigin.GENERATED.value) + _add_folder(test_db, "notes", origin=ContentOrigin.USER.value) + + result = generated_folders( + test_db.query(KnowledgeFolder).filter(KnowledgeFolder.kind_id == KIND_ID) + ) + + assert {folder.name for folder in result.all()} == {"architecture"} + + +def test_content_defaults_to_user_owned(test_db: Session): + """Unmarked content must be treated as user-owned. + + Mislabelling generated content as user-owned only stops automatic cleanup, while + the reverse would let generation delete a person's documents. + """ + document = KnowledgeDocument( + kind_id=KIND_ID, name="uploaded", file_extension="pdf", user_id=1 + ) + folder = KnowledgeFolder(kind_id=KIND_ID, parent_id=0, name="uploads") + test_db.add_all([document, folder]) + test_db.flush() + + assert document.origin == ContentOrigin.USER.value + assert folder.origin == ContentOrigin.USER.value + + +def test_listing_documents_never_returns_code_targets(test_db: Session): + """The scope must be wired into the listing, not merely available to it. + + An earlier revision defined these scopes but left every production query + unfiltered, which reads as protection while providing none. + """ + _add_document(test_db, "architecture", origin=ContentOrigin.GENERATED.value) + _add_document( + test_db, + "src/main.py", + origin=ContentOrigin.GENERATED.value, + source_type=CODE_TARGET_SOURCE_TYPE, + folder_id=NO_FOLDER, + ) + + listed = wiki_pages( + test_db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == KIND_ID) + ).all() + + assert [document.name for document in listed] == ["architecture"] diff --git a/backend/tests/services/knowledge/test_mermaid_check.py b/backend/tests/services/knowledge/test_mermaid_check.py new file mode 100644 index 0000000000..47d65171b8 --- /dev/null +++ b/backend/tests/services/knowledge/test_mermaid_check.py @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for structural checks on Mermaid diagrams.""" + +import pytest + +from app.services.knowledge.mermaid_check import ( + check_mermaid_blocks, + describe_warnings, +) + + +def _fence(body: str, info: str = "mermaid") -> str: + return f"```{info}\n{body}\n```" + + +def test_a_valid_diagram_produces_no_warnings(): + markdown = "# Architecture\n\n" + _fence("flowchart TD\n A[Start] --> B[End]") + + assert check_mermaid_blocks(markdown) == [] + + +def test_documents_without_diagrams_produce_no_warnings(): + assert check_mermaid_blocks("# Just prose\n\nNo diagrams here.") == [] + + +def test_a_misspelled_diagram_type_is_reported(): + markdown = _fence("flowchat TD\n A --> B") + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "flowchat" in warnings[0].message + assert "not render" in warnings[0].message + + +def test_an_unclosed_fence_is_reported(): + markdown = "```mermaid\nflowchart TD\n A --> B\n" + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "never closed" in warnings[0].message + + +def test_an_empty_diagram_is_reported(): + markdown = _fence("") + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "empty" in warnings[0].message + + +def test_an_unbalanced_bracket_is_reported(): + markdown = _fence("flowchart TD\n A[Start --> B[End]") + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "unclosed '['" in warnings[0].message + + +def test_a_mismatched_closing_bracket_is_reported(): + markdown = _fence("flowchart TD\n A[Start) --> B") + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "mismatched" in warnings[0].message + + +def test_brackets_inside_quoted_labels_are_not_misread(): + markdown = _fence('flowchart TD\n A["build (release)"] --> B["ship"]') + + assert check_mermaid_blocks(markdown) == [] + + +def test_an_unclosed_quote_is_reported(): + markdown = _fence('flowchart TD\n A["Start --> B') + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "unclosed quote" in warnings[0].message + + +def test_comments_before_the_diagram_type_are_skipped(): + markdown = _fence("%% layout notes\nflowchart LR\n A --> B") + + assert check_mermaid_blocks(markdown) == [] + + +def test_diagram_type_with_a_direction_or_colon_is_accepted(): + assert check_mermaid_blocks(_fence("graph LR\n A --> B")) == [] + assert check_mermaid_blocks(_fence("stateDiagram-v2\n [*] --> Idle")) == [] + assert check_mermaid_blocks(_fence("pie title Share\n 'a' : 10")) == [] + + +def test_a_mermaid_example_quoted_inside_another_fence_is_not_checked(): + """A code sample showing broken Mermaid is documentation, not a diagram.""" + markdown = "````markdown\n```mermaid\nflowchat TD\n```\n````" + + assert check_mermaid_blocks(markdown) == [] + + +def test_entity_relationship_cardinality_is_not_mistaken_for_a_bracket_error(): + """ER diagrams write cardinality as ``||--o{``, which only looks unbalanced. + + Reporting it would send the agent off to fix a diagram that already renders. + """ + markdown = _fence( + "erDiagram\n" + " CUSTOMER ||--o{ ORDER : places\n" + " ORDER }|..|{ LINE_ITEM : contains" + ) + + assert check_mermaid_blocks(markdown) == [] + + +def test_text_heavy_diagrams_may_contain_stray_brackets_in_labels(): + assert check_mermaid_blocks(_fence("gantt\n title Release [Q3\n")) == [] + assert check_mermaid_blocks(_fence("journey\n Browse [start: 3")) == [] + + +def test_every_diagram_in_a_page_is_checked(): + markdown = "\n\n".join( + [ + _fence("flowchart TD\n A --> B"), + _fence("nonsense\n A --> B"), + _fence("sequenceDiagram\n Alice->>Bob: hi"), + _fence("erDiagram\n A ||--o{ B : has"), + ] + ) + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "nonsense" in warnings[0].message + + +def test_warnings_carry_the_line_of_their_diagram(): + markdown = "# Title\n\nProse.\n\n" + _fence("flowchat TD\n A --> B") + + warnings = check_mermaid_blocks(markdown) + + assert warnings[0].line == 5 + + +def test_warnings_render_as_an_instruction_for_the_agent(): + warnings = check_mermaid_blocks(_fence("flowchat TD\n A --> B")) + + described = describe_warnings(warnings) + + assert "write the page again" in described + assert "flowchat" in described + + +def test_no_warnings_render_as_nothing(): + assert describe_warnings([]) == "" + + +def test_a_shorter_fence_does_not_close_a_longer_one(): + """A closing fence must be at least as long as the one it opens. + + Comparing only the fence character lets an inner ``` end an outer ````, which is + exactly how a diagram gets quoted rather than declared. Here the document is + genuinely unterminated, and treating the short fence as a close would hide that. + """ + markdown = "````mermaid\nflowchart TD\n A --> B\n```\n" + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "never closed" in warnings[0].message + + +def test_a_longer_fence_closes_normally(): + markdown = "````mermaid\nflowchart TD\n A --> B\n````\n" + + assert check_mermaid_blocks(markdown) == [] + + +@pytest.mark.parametrize( + "declaration", + ["kanban", "radar", "treemap", "classDiagram-v2", "flowchart-elk", "C4Container"], +) +def test_declarations_the_pinned_mermaid_supports_are_not_reported(declaration: str): + """The frontend resolves Mermaid 11.15, and a type missing from the allow-list is + sent back to the agent as a broken diagram — a wasted round spent "fixing" one + that renders.""" + assert check_mermaid_blocks(_fence(f"{declaration}\n A --> B")) == [] From 51fbe67740086e3620174cfb9eb75174b898bbab Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Mon, 3 Aug 2026 13:11:36 +0800 Subject: [PATCH 02/14] refactor(knowledge): act on review of the code wiki internals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Index cleanup moves into the publish path and the periodic sweep is deleted. Parking and draining were two writers doing read-modify-write on one spec key, and the interleaving that loses a ref needs no unusual timing: a sweep reads the list, a concurrent publish appends to it, the sweep writes back what it read. A distributed lock between sweepers would not have fixed that, because the publish never held it. One writer removes the race outright, and with it the lock, the beat entry and the frequency question. The cost is stated rather than hidden: a wiki nobody regenerates keeps its orphaned chunks. The debt stays recorded on the knowledge base and logged when parked, so it is visible if that ever turns out to matter. The agent can now read a page before revising it. Incremental runs were told to "revise only the pages the changes affect" while having no way to see a word of what any page said — so revising meant rewriting from source, and whatever a page held that was not re-derivable was silently dropped. Reading is scoped to the agent's own generation, which in an incremental run is a complete copy of the published wiki: the capability it needs, and the narrowest scope that provides it. Also from review: - The folder cleanup walks the tree once, deepest-first, instead of re-querying every folder and every document until the result stopped changing. Two tests were added for multi-level emptying, which the old loop handled and nothing covered. - Folder resolution loads the knowledge base's folders once instead of querying per distinct folder. - The attachments a plan supersedes are read in one query. They were being served from the session identity map because of what ran just before, which nothing stated and nothing enforced. - GitHub and Gitea shared a near-identical status mapping in two copies; it is now one. GitLab is left out because it reports booleans, not words. Co-Authored-By: Claude Opus 5 --- backend/app/api/endpoints/wiki.py | 30 +++ backend/app/core/celery_app.py | 7 - backend/app/repository/file_status.py | 36 ++++ backend/app/repository/gitea_provider.py | 15 +- backend/app/repository/github_provider.py | 14 +- backend/app/schemas/wiki.py | 8 + .../knowledge/code_wiki_projection.py | 110 +++++----- .../services/knowledge/code_wiki_prompts.py | 5 + .../services/knowledge/code_wiki_publisher.py | 32 ++- .../services/knowledge/code_wiki_runner.py | 95 -------- backend/app/services/wiki_service.py | 36 ++++ backend/app/tasks/knowledge_tasks.py | 16 -- backend/init_data/skills/wiki_submit/SKILL.md | 12 ++ .../skills/wiki_submit/wiki_submit.js | 69 ++++++ .../knowledge/test_code_wiki_cleanup_sweep.py | 202 ------------------ .../knowledge/test_code_wiki_content_write.py | 58 +++++ .../knowledge/test_code_wiki_projection.py | 40 ++++ .../knowledge/test_code_wiki_publisher.py | 70 ++++++ 18 files changed, 457 insertions(+), 398 deletions(-) create mode 100644 backend/app/repository/file_status.py delete mode 100644 backend/tests/services/knowledge/test_code_wiki_cleanup_sweep.py diff --git a/backend/app/api/endpoints/wiki.py b/backend/app/api/endpoints/wiki.py index 7111f1e6bb..68baee5843 100644 --- a/backend/app/api/endpoints/wiki.py +++ b/backend/app/api/endpoints/wiki.py @@ -20,6 +20,7 @@ WikiGenerationDetail, WikiGenerationInDB, WikiGenerationListResponse, + WikiPageRead, WikiProjectDetail, WikiProjectInDB, WikiProjectListResponse, @@ -217,6 +218,35 @@ def save_wiki_generation_contents( return None +@internal_router.get("/generations/{generation_id}/pages", response_model=WikiPageRead) +def read_wiki_generation_page( + generation_id: int, + path: str = Query(..., min_length=1, description="Stable page path to read"), + _: None = Depends(_verify_internal_token), + wiki_db: Session = Depends(get_wiki_db), +): + """Read one page of the version the agent is writing into (internal use). + + An incremental version begins as a complete copy of the published wiki, so the + agent's own generation is also the current wiki — which makes "read your own + version" both the capability it needs and the narrowest scope that provides it. + Without this the instruction to revise a page cannot be followed: the agent knows + the page's path and cannot see a word of what it says. + + Answers 404 when the path holds no page. That is a useful answer rather than a + failure — in an incremental run it means the page is new. + """ + page = wiki_service.get_generation_page( + wiki_db=wiki_db, generation_id=generation_id, path=path + ) + if page is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Generation {generation_id} has no page at '{path}'", + ) + return page + + @router.get( "/generations/{generation_id}/contents", response_model=list[WikiContentInDB] ) diff --git a/backend/app/core/celery_app.py b/backend/app/core/celery_app.py index 721a37ecd7..9aefb32e3d 100644 --- a/backend/app/core/celery_app.py +++ b/backend/app/core/celery_app.py @@ -84,13 +84,6 @@ "task": "app.tasks.knowledge_tasks.scan_stale_index_tasks", "schedule": 5 * 60, # every 5 minutes }, - "sweep-code-wiki-index-cleanup": { - "task": "app.tasks.knowledge_tasks.sweep_code_wiki_index_cleanup", - # Less often than the stale scan: this drains a list that only grows when - # the vector store is already failing, and retrying a broken one every - # five minutes adds load exactly when it is least welcome. - "schedule": 15 * 60, - }, }, # Beat scheduler class - Use default PersistentScheduler (file-based) # Note: Only run ONE Celery Beat instance in production diff --git a/backend/app/repository/file_status.py b/backend/app/repository/file_status.py new file mode 100644 index 0000000000..e478ecca16 --- /dev/null +++ b/backend/app/repository/file_status.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Translating a host's words for a changed file into git's name-status letters. + +GitHub and Gitea both describe a compare entry with a word ("added", "modified"), +and they agree on all but one of them. The letters are what callers reason about, so +the translation lives here once rather than being restated per provider — two copies +of a near-identical mapping is exactly the shape that drifts. + +GitLab is absent on purpose: it reports booleans (``new_file``, ``deleted_file``, +``renamed_file``) rather than a word, so it has nothing to look up. +""" + +# Superset of what GitHub and Gitea emit. Sharing it is safe because the two never +# disagree on a word — Gitea says "deleted" where GitHub says "removed", and neither +# emits the other's spelling. +FILE_STATUS_LETTERS: dict[str, str] = { + "added": "A", + "changed": "M", + "copied": "A", + "deleted": "D", + "modified": "M", + "removed": "D", + "renamed": "R", +} + +# What a word we do not recognise becomes. An edit is the conservative reading: it +# neither invents a structural move nor hides a file from the diff. +DEFAULT_FILE_STATUS = "M" + + +def file_status_letter(status: str) -> str: + """Render one compare entry's status as a name-status letter.""" + return FILE_STATUS_LETTERS.get((status or "").lower(), DEFAULT_FILE_STATUS) diff --git a/backend/app/repository/gitea_provider.py b/backend/app/repository/gitea_provider.py index d2eda90f1d..fb29fce451 100644 --- a/backend/app/repository/gitea_provider.py +++ b/backend/app/repository/gitea_provider.py @@ -17,6 +17,7 @@ from app.core.cache import cache_manager from app.core.config import settings from app.models.user import User +from app.repository.file_status import file_status_letter from app.repository.interfaces.repository_provider import RepositoryProvider from app.schemas.github import Branch, Repository from shared.utils.sensitive_data_masker import mask_string @@ -26,18 +27,6 @@ # must still fail rather than hold the worker. REPO_STATE_TIMEOUT_SECONDS = 15 -# Compare statuses mapped to git's name-status letters, which is what the code wiki -# run-mode rules are written against. -_GITEA_FILE_STATUS = { - "added": "A", - "removed": "D", - "deleted": "D", - "modified": "M", - "renamed": "R", - "copied": "A", - "changed": "M", -} - class GiteaProvider(RepositoryProvider): """ @@ -922,7 +911,7 @@ def get_changed_files( return [ { "path": entry.get("filename", ""), - "status": _GITEA_FILE_STATUS.get(entry.get("status", ""), "M"), + "status": file_status_letter(entry.get("status", "")), } for entry in (payload.get("files") or []) if entry.get("filename") diff --git a/backend/app/repository/github_provider.py b/backend/app/repository/github_provider.py index 316efb738e..8356ae06b5 100644 --- a/backend/app/repository/github_provider.py +++ b/backend/app/repository/github_provider.py @@ -17,6 +17,7 @@ from app.core.cache import cache_manager from app.core.config import settings from app.models.user import User +from app.repository.file_status import file_status_letter from app.repository.interfaces.repository_provider import RepositoryProvider from app.schemas.github import Branch, Repository from shared.utils.sensitive_data_masker import mask_string @@ -31,17 +32,6 @@ # signal that it truncated, so hitting it is treated as "the diff is unknown". GITHUB_COMPARE_FILE_LIMIT = 300 -# Compare statuses mapped to git's name-status letters, which is what the code wiki -# run-mode rules are written against. -_GITHUB_FILE_STATUS = { - "added": "A", - "removed": "D", - "modified": "M", - "renamed": "R", - "copied": "A", - "changed": "M", -} - class GitHubProvider(RepositoryProvider): """ @@ -1038,7 +1028,7 @@ def get_changed_files( return [ { "path": entry.get("filename", ""), - "status": _GITHUB_FILE_STATUS.get(entry.get("status", ""), "M"), + "status": file_status_letter(entry.get("status", "")), } for entry in files if entry.get("filename") diff --git a/backend/app/schemas/wiki.py b/backend/app/schemas/wiki.py index a8d00776e6..deb05500d3 100644 --- a/backend/app/schemas/wiki.py +++ b/backend/app/schemas/wiki.py @@ -158,6 +158,14 @@ class WikiContentSummary(BaseModel): head_commit: Optional[str] = None +class WikiPageRead(BaseModel): + """One page of a version, as the agent reads it back.""" + + path: str + title: str + content: str + + class WikiContentWriteRequest(BaseModel): """Request payload for writing wiki contents""" diff --git a/backend/app/services/knowledge/code_wiki_projection.py b/backend/app/services/knowledge/code_wiki_projection.py index 027b5c00fd..2144cf60d3 100644 --- a/backend/app/services/knowledge/code_wiki_projection.py +++ b/backend/app/services/knowledge/code_wiki_projection.py @@ -92,35 +92,30 @@ def _folder_resolver(db: Session, kind_id: int, user_id: int): """Return a function creating (or finding) the folder chain for a page path.""" from app.models.knowledge import KnowledgeFolder - cache: dict[tuple[int, str], int] = {} + # Loaded once. A projection touches a handful of folders and asks about them + # repeatedly, so one query up front replaces a query per distinct folder. + cache: dict[tuple[int, str], int] = { + (folder.parent_id, folder.name.casefold()): folder.id + for folder in db.query(KnowledgeFolder) + .filter(KnowledgeFolder.kind_id == kind_id) + .all() + } def resolve(segments: Sequence[str]) -> int: parent_id = 0 for segment in segments: key = (parent_id, segment.casefold()) - if key in cache: - parent_id = cache[key] - continue - existing = ( - db.query(KnowledgeFolder) - .filter( - KnowledgeFolder.kind_id == kind_id, - KnowledgeFolder.parent_id == parent_id, - KnowledgeFolder.name == segment, - ) - .first() - ) - if existing is None: - existing = KnowledgeFolder( + if key not in cache: + created = KnowledgeFolder( kind_id=kind_id, parent_id=parent_id, name=segment, origin=ContentOrigin.GENERATED.value, ) - db.add(existing) + db.add(created) db.flush() - cache[key] = existing.id - parent_id = existing.id + cache[key] = created.id + parent_id = cache[key] return parent_id return resolve @@ -308,35 +303,54 @@ def _remove_emptied_generated_folders(db: Session, kind_id: int) -> None: """ from app.models.knowledge import KnowledgeFolder - # Repeated because emptying a leaf can empty its parent. - while True: - folders = ( - db.query(KnowledgeFolder) - .filter( - KnowledgeFolder.kind_id == kind_id, - KnowledgeFolder.origin == ContentOrigin.GENERATED.value, - ) - .all() + folders = ( + db.query(KnowledgeFolder) + .filter( + KnowledgeFolder.kind_id == kind_id, + KnowledgeFolder.origin == ContentOrigin.GENERATED.value, ) - if not folders: - return - - occupied_parents = {folder.parent_id for folder in folders if folder.parent_id} - used_folder_ids = { - row[0] - for row in db.query(KnowledgeDocument.folder_id) - .filter(KnowledgeDocument.kind_id == kind_id) - .distinct() - .all() - } - - empties = [ - folder - for folder in folders - if folder.id not in used_folder_ids and folder.id not in occupied_parents - ] - if not empties: - return - for folder in empties: + .all() + ) + if not folders: + return + + used_folder_ids = { + row[0] + for row in db.query(KnowledgeDocument.folder_id) + .filter(KnowledgeDocument.kind_id == kind_id) + .distinct() + .all() + } + + # Walked deepest-first. Emptying a leaf can empty its parent, so a parent is only + # judged once its children have been — which is what the old repeat-until-stable + # loop was doing, one re-query per level. + children: dict[int, list[KnowledgeFolder]] = {} + for folder in folders: + children.setdefault(folder.parent_id, []).append(folder) + + doomed: set[int] = set() + + def survives(folder: KnowledgeFolder) -> bool: + # Every child is visited before the decision: a short circuit would leave the + # ones after the first survivor unexamined, and they may be empty themselves. + kept_child = any([survives(child) for child in children.get(folder.id, ())]) + if folder.id in used_folder_ids or kept_child: + return True + doomed.add(folder.id) + return False + + # Roots are the generated folders whose parent is not itself generated: the tree + # top (parent 0) and anything a user folder contains. Depth is capped at four, so + # the recursion cannot run away. + generated_ids = {folder.id for folder in folders} + for folder in folders: + if folder.parent_id not in generated_ids: + survives(folder) + + if not doomed: + return + for folder in folders: + if folder.id in doomed: db.delete(folder) - db.flush() + db.flush() diff --git a/backend/app/services/knowledge/code_wiki_prompts.py b/backend/app/services/knowledge/code_wiki_prompts.py index c0a773e821..cbc728b246 100644 --- a/backend/app/services/knowledge/code_wiki_prompts.py +++ b/backend/app/services/knowledge/code_wiki_prompts.py @@ -212,6 +212,11 @@ def build_incremental_prompt(context: WikiRunContext) -> str: - Revise only the pages the changes actually affect. Leave the rest untouched — an unchanged page costs nothing to keep and is expensive to rewrite. +- **Read a page before you revise it.** Your version already holds every published + page, and what you send replaces the whole page, so revising one without reading it + first silently discards whatever it said: + `node wiki_submit.js read --generation-id --path ` + It exits with no output when the page does not exist yet, which means it is new. - Refresh the `index` overview if the set of pages changed. - If a change makes a page's subject disappear, remove that page explicitly. - Add pages only where the changes introduced something the wiki has no place for. diff --git a/backend/app/services/knowledge/code_wiki_publisher.py b/backend/app/services/knowledge/code_wiki_publisher.py index a06a29575e..4840fddfb8 100644 --- a/backend/app/services/knowledge/code_wiki_publisher.py +++ b/backend/app/services/knowledge/code_wiki_publisher.py @@ -188,6 +188,15 @@ def publish_generation( db.commit() return PublishResult(published=False, verdict=verdict, reason=verdict.reason) + # Settle what the last publish could not before computing this one. Doing it here + # rather than on a timer keeps this function the only writer of the parked list, + # which is what removes the race: two writers doing read-modify-write on one spec + # key will eventually drop a ref parked between a sweep's read and its write. + # + # It runs before the gate on purpose. The debt has nothing to do with whether this + # version is publishable, and a rejected version should still clear it. + retry_pending_index_cleanup(db, knowledge_base=knowledge_base, effects=effects) + desired = read_version_pages(db, generation.id) existing = read_projected_pages(db, knowledge_base.id) @@ -250,14 +259,18 @@ def _attachments_being_replaced(db: Session, plan: ProjectionPlan) -> tuple[int, Includes each page's converted attachment: overwriting the source without it leaves a stale conversion that the document still points at. """ - doomed: list[int] = [] touched = [update.existing.document_id for update in plan.updates] touched += [page.document_id for page in plan.deletes] + if not touched: + return () - for document_id in touched: - document = db.get(KnowledgeDocument, document_id) - if document is None: - continue + doomed: list[int] = [] + # One query rather than a lookup per document. Those lookups were served from the + # session's identity map today, because read_projected_pages had just loaded the + # same rows — a dependency on call order that nothing states and nothing enforces. + for document in ( + db.query(KnowledgeDocument).filter(KnowledgeDocument.id.in_(touched)).all() + ): if document.attachment_id: doomed.append(document.attachment_id) converted = document.converted_attachment_id @@ -300,6 +313,15 @@ def retry_pending_index_cleanup( ) -> tuple[str, ...]: """Attempt the index deletions a previous publish could not finish. + Called at the start of every publish rather than on a timer. The cost is that a + wiki nobody regenerates keeps its orphaned chunks; the gain is that parking and + draining happen in one place, so neither can overwrite the other's view of the + list. A periodic sweeper would be a second writer of the same spec key, and the + interleaving that loses a ref needs no unusual timing to happen. + + The debt stays visible in the meantime: it is recorded on the knowledge base and + logged at warning level when it is parked. + Returns: The refs still outstanding. """ diff --git a/backend/app/services/knowledge/code_wiki_runner.py b/backend/app/services/knowledge/code_wiki_runner.py index ae5b3d37c5..b0d7a8f33d 100644 --- a/backend/app/services/knowledge/code_wiki_runner.py +++ b/backend/app/services/knowledge/code_wiki_runner.py @@ -27,7 +27,6 @@ from datetime import datetime, timezone from typing import Optional, Sequence -from sqlalchemy import func from sqlalchemy.orm import Session from app.core.wiki_config import wiki_settings @@ -258,100 +257,6 @@ def finish_run( ) -def sweep_pending_index_cleanup(db: Session) -> dict: - """Finish the index deletions that publishes could not. - - Deleting a page removes its row inside the publish transaction and its chunks - outside one, because the vector store cannot join a database transaction. When - the second half fails the publish still succeeds — refusing a good version over - an external hiccup would be worse — and the owed deletions are parked on the - knowledge base for this sweep to finish. - - Without it they are never retried. The orphaned chunks outlive the page, and - retrieval goes on answering from it while citing a document id that no longer - resolves. Nothing else clears them: that would take deleting the same page a - second time, and it is already gone. - - Refs that keep failing stay parked and are tried again on the next sweep. Giving - up would make the orphan permanent, which is the state this exists to end. - """ - from app.services.knowledge.code_wiki_publisher import retry_pending_index_cleanup - - swept = 0 - drained = 0 - outstanding = 0 - for knowledge_base in _knowledge_bases_owing_cleanup(db): - owner = db.get(User, knowledge_base.user_id) - if owner is None: - logger.warning( - "[code_wiki] kb %s owes index cleanup but its owner %s is gone", - knowledge_base.id, - knowledge_base.user_id, - ) - continue - - swept += 1 - try: - still_pending = retry_pending_index_cleanup( - db, - knowledge_base=knowledge_base, - effects=build_projection_side_effects( - db, knowledge_base=knowledge_base, user=owner - ), - ) - except Exception as exc: - # One knowledge base's failure must not stop the others: they are - # independent, and the whole point is to keep draining. - db.rollback() - logger.error( - "[code_wiki] index cleanup sweep failed for kb %s: %s", - knowledge_base.id, - exc, - ) - continue - - outstanding += len(still_pending) - if not still_pending: - drained += 1 - - if swept: - logger.info( - "[code_wiki] index cleanup sweep: %s knowledge base(s), %s drained, " - "%s deletion(s) still owed", - swept, - drained, - outstanding, - ) - return {"swept": swept, "drained": drained, "outstanding": outstanding} - - -def _knowledge_bases_owing_cleanup(db: Session) -> list[Kind]: - """Knowledge bases with parked index deletions. - - The JSON filter narrows the scan to knowledge bases that have ever parked - anything; emptiness is then checked in Python, because a drained list is stored - as ``[]`` and comparing against that in SQL renders differently per database. - """ - from app.services.knowledge.code_wiki_projection import PENDING_INDEX_CLEANUP_KEY - - candidates = ( - db.query(Kind) - .filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, # noqa: E712 - func.json_extract(Kind.json, f"$.spec.{PENDING_INDEX_CLEANUP_KEY}").isnot( - None - ), - ) - .all() - ) - return [ - candidate - for candidate in candidates - if (candidate.json or {}).get("spec", {}).get(PENDING_INDEX_CLEANUP_KEY) - ] - - def is_code_wiki_generation(db: Session, generation: WikiGeneration) -> bool: """Whether this run belongs to a code wiki rather than the legacy wiki path.""" return _knowledge_base_of(db, generation) is not None diff --git a/backend/app/services/wiki_service.py b/backend/app/services/wiki_service.py index 3904da6673..047af02d45 100644 --- a/backend/app/services/wiki_service.py +++ b/backend/app/services/wiki_service.py @@ -26,6 +26,7 @@ WikiContentSummary, WikiContentWriteRequest, WikiGenerationCreate, + WikiPageRead, WikiProjectCreate, ) from app.services.adapters.task_kinds import task_kinds_service @@ -821,6 +822,41 @@ def save_generation_contents( if finishes_a_code_wiki: self._finish_code_wiki(wiki_db, generation, summary) + def get_generation_page( + self, + wiki_db: Session, + generation_id: int, + path: str, + ) -> Optional[WikiPageRead]: + """Read one page of a version by its path. + + Matched the same way a write is — normalised, then compared case-insensitively + — so that a path which would update a page also reads it. Resolving them + differently would let the agent read one page and overwrite another. + + Returns: + The page, or ``None`` when the version holds none at that path. + """ + try: + normalized = normalize_page_path(path) + except InvalidPagePath as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + wanted = collation_key(normalized) + for content in ( + wiki_db.query(WikiContent) + .filter(WikiContent.generation_id == generation_id) + .all() + ): + content_path = page_path_of(content) + if content_path and collation_key(content_path) == wanted: + return WikiPageRead( + path=content_path, + title=content.title, + content=content.content, + ) + return None + def _apply_removals( self, wiki_db: Session, diff --git a/backend/app/tasks/knowledge_tasks.py b/backend/app/tasks/knowledge_tasks.py index 4edd5dd1be..ae8ad46c4f 100644 --- a/backend/app/tasks/knowledge_tasks.py +++ b/backend/app/tasks/knowledge_tasks.py @@ -537,19 +537,3 @@ def scan_stale_index_tasks(): f"[StaleScanner] Scan complete, marked {marked_count} documents as FAILED" ) return {"marked_count": marked_count, "scanned_count": len(active_docs)} - - -@celery_app.task(name="app.tasks.knowledge_tasks.sweep_code_wiki_index_cleanup") -def sweep_code_wiki_index_cleanup(): - """Finish index deletions a code wiki publish could not complete. - - Publishing removes a deleted page's row inside a transaction and its chunks - outside one, so a vector store failure leaves the chunks behind. The publish is - allowed to succeed anyway and parks what it still owes; this is the sweep that - settles it. Without it, retrieval keeps answering from pages that no longer - exist and citing document ids that no longer resolve. - """ - from app.services.knowledge.code_wiki_runner import sweep_pending_index_cleanup - - with SessionLocal() as db: - return sweep_pending_index_cleanup(db) diff --git a/backend/init_data/skills/wiki_submit/SKILL.md b/backend/init_data/skills/wiki_submit/SKILL.md index 26258a0614..c6620d9d83 100644 --- a/backend/init_data/skills/wiki_submit/SKILL.md +++ b/backend/init_data/skills/wiki_submit/SKILL.md @@ -50,6 +50,18 @@ node wiki_submit.js submit \ --content $'# Overview\n\nYour markdown content here...' ``` +### Read what a page currently says + +Only your own generation is readable, which in an incremental run is a complete copy +of the published wiki — so this is how you see a page before revising it. + +```bash +node wiki_submit.js read --generation-id 123 --path architecture/backend > current.md +``` + +Exits 0 with no output when the page does not exist yet. In an incremental run that +means the page is new. + ### Remove pages that no longer have a subject Only meaningful in an incremental run, where your version starts as a copy of the diff --git a/backend/init_data/skills/wiki_submit/wiki_submit.js b/backend/init_data/skills/wiki_submit/wiki_submit.js index dc0e84c7a7..160b38c6fa 100644 --- a/backend/init_data/skills/wiki_submit/wiki_submit.js +++ b/backend/init_data/skills/wiki_submit/wiki_submit.js @@ -148,6 +148,26 @@ function makeRequest(url, options, body) { }) } +/** + * Read one page of the generation being written. + * @param {string} endpoint - Write endpoint URL, used to derive the read URL + * @param {string} token - Authorization token + * @param {number} generationId - Wiki generation ID + * @param {string} pagePath - Stable page path + * @returns {Promise} + */ +async function readPage(endpoint, token, generationId, pagePath) { + // The read endpoint sits beside the write one under /generations. + const base = endpoint.replace(/\/generations\/contents\/?$/, '') + const url = `${base}/generations/${generationId}/pages?path=${encodeURIComponent(pagePath)}` + + return makeRequest( + url, + { method: 'GET', headers: { Authorization: `Bearer ${token}` } }, + null + ) +} + /** * Submit wiki sections to the backend API. * @param {string} endpoint - API endpoint URL @@ -245,6 +265,46 @@ async function cmdSubmit(args) { return 0 } +/** + * Handle read command: print a page's current content. + * @param {object} args - Command arguments + * @returns {Promise} + */ +async function cmdRead(args) { + const endpoint = getWikiEndpoint(args.endpoint) + const token = getAuthToken(args.token) + if (!token) { + console.error('Error: Authorization token is required. It can be obtained from TASK_INFO, WIKI_TOKEN env var, or --token argument.') + process.exit(1) + } + if (!args.generationId) { + console.error('Error: --generation-id is required.') + process.exit(1) + } + if (!args.path) { + console.error('Error: --path is required for read command') + return 1 + } + const generationId = parseInt(args.generationId, 10) + + const result = await readPage(endpoint, token, generationId, args.path) + + if (result.status === 'error') { + // A page that does not exist yet is an answer, not a failure: in an incremental + // run it means this page is new. Reported on stderr so it cannot be mistaken + // for content when stdout is redirected to a file. + if (/404|no page at/i.test(result.message || '')) { + console.error(`Page '${args.path}' does not exist yet`) + return 0 + } + console.error(`❌ Error: ${result.message}`) + return 1 + } + + process.stdout.write(result.content || '') + return 0 +} + /** * Handle remove command: declare pages as gone. * @param {object} args - Command arguments @@ -470,6 +530,7 @@ Usage: node wiki_submit.js [options] Commands: submit Submit a wiki page + read Print a page's current content remove Declare wiki pages as gone complete Mark wiki generation as completed fail Mark wiki generation as failed @@ -493,6 +554,10 @@ Submit Options: --content, -c Page content (alternative to --file) --ext Extension data as JSON string +Read Options: + --path Page path to read. Exits 0 with no output when the page + does not exist yet. + Remove Options: --path Page path to remove. Repeat for several pages. @@ -507,6 +572,7 @@ Fail Options: Examples: node wiki_submit.js submit --generation-id 123 --path architecture/backend --title "Backend Architecture" --file ./page.md + node wiki_submit.js read --generation-id 123 --path architecture/backend > current.md node wiki_submit.js remove --generation-id 123 --path modules/legacy-sync node wiki_submit.js complete --generation-id 123 --head-commit $(git rev-parse HEAD) node wiki_submit.js fail --generation-id 123 --error-message "Failed to analyze repository" @@ -540,6 +606,9 @@ async function main() { } exitCode = await cmdSubmit(args) break + case 'read': + exitCode = await cmdRead(args) + break case 'remove': exitCode = await cmdRemove(args) break diff --git a/backend/tests/services/knowledge/test_code_wiki_cleanup_sweep.py b/backend/tests/services/knowledge/test_code_wiki_cleanup_sweep.py deleted file mode 100644 index 86939e30f5..0000000000 --- a/backend/tests/services/knowledge/test_code_wiki_cleanup_sweep.py +++ /dev/null @@ -1,202 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Weibo, Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for draining the index deletions a publish could not finish. - -A page's row is deleted inside the publish transaction and its chunks outside one, -because the vector store cannot join a database transaction. The publish is allowed -to succeed when the second half fails — refusing a good version over an external -hiccup would be worse — and parks what it still owes. - -Nothing else settles that debt. It cannot self-heal by deleting the page again, -because the page is already gone. So these tests are about the sweep finding the -debt, clearing what it can, keeping what it cannot, and not letting one broken -knowledge base stop the others. -""" - -from dataclasses import dataclass, field - -import pytest -from sqlalchemy.orm import Session - -from app.models.kind import Kind -from app.models.user import User -from app.services.knowledge.code_wiki_projection import ( - PENDING_INDEX_CLEANUP_KEY, - ProjectionSideEffects, -) -from app.services.knowledge.code_wiki_runner import sweep_pending_index_cleanup - - -@dataclass -class FakeIndex: - """Stands in for the vector store.""" - - deleted: list[int] = field(default_factory=list) - refuses: set[int] = field(default_factory=set) - explodes_for_kb: set[int] = field(default_factory=set) - - def effects_for(self, kind_id: int) -> ProjectionSideEffects: - def delete_rag_document(document_id: int) -> None: - if kind_id in self.explodes_for_kb: - raise RuntimeError("vector store unreachable") - if document_id in self.refuses: - raise RuntimeError(f"index deletion refused for {document_id}") - self.deleted.append(document_id) - - return ProjectionSideEffects( - write_attachment=lambda **_: 0, - delete_attachment=lambda _: None, - delete_rag_document=delete_rag_document, - enqueue_reindex=lambda _: None, - ) - - -@pytest.fixture -def index(monkeypatch) -> FakeIndex: - from app.services.knowledge import code_wiki_runner - - fake = FakeIndex() - monkeypatch.setattr( - code_wiki_runner, - "build_projection_side_effects", - lambda db, *, knowledge_base, user: fake.effects_for(knowledge_base.id), - ) - return fake - - -def _knowledge_base( - test_db: Session, user: User, name: str, pending: list[str] | None = None -) -> Kind: - spec = {"name": name, "kbType": "code_wiki"} - if pending is not None: - spec[PENDING_INDEX_CLEANUP_KEY] = pending - kind = Kind( - kind="KnowledgeBase", - name=name, - namespace="default", - user_id=user.id, - json={"spec": spec}, - is_active=True, - ) - test_db.add(kind) - test_db.flush() - return kind - - -def _pending(test_db: Session, knowledge_base: Kind) -> list[str]: - test_db.refresh(knowledge_base) - return (knowledge_base.json or {}).get("spec", {}).get( - PENDING_INDEX_CLEANUP_KEY - ) or [] - - -def test_owed_deletions_are_finished( - test_db: Session, test_user: User, index: FakeIndex -): - kb = _knowledge_base(test_db, test_user, "owes", pending=["11", "12"]) - - result = sweep_pending_index_cleanup(test_db) - - assert sorted(index.deleted) == [11, 12] - assert _pending(test_db, kb) == [] - assert result["drained"] == 1 - - -def test_a_deletion_that_still_fails_stays_owed( - test_db: Session, test_user: User, index: FakeIndex -): - """Giving up would make the orphaned chunks permanent, which is the state this - sweep exists to end.""" - index.refuses = {12} - kb = _knowledge_base(test_db, test_user, "partly", pending=["11", "12"]) - - result = sweep_pending_index_cleanup(test_db) - - assert index.deleted == [11] - assert _pending(test_db, kb) == ["12"] - assert result["outstanding"] == 1 - assert result["drained"] == 0 - - -def test_a_failing_ref_is_tried_again_on_the_next_sweep( - test_db: Session, test_user: User, index: FakeIndex -): - index.refuses = {12} - kb = _knowledge_base(test_db, test_user, "recovers", pending=["12"]) - sweep_pending_index_cleanup(test_db) - - index.refuses = set() - sweep_pending_index_cleanup(test_db) - - assert index.deleted == [12] - assert _pending(test_db, kb) == [] - - -def test_knowledge_bases_owing_nothing_are_not_touched( - test_db: Session, test_user: User, index: FakeIndex -): - _knowledge_base(test_db, test_user, "never-parked") - _knowledge_base(test_db, test_user, "already-drained", pending=[]) - - result = sweep_pending_index_cleanup(test_db) - - assert result == {"swept": 0, "drained": 0, "outstanding": 0} - assert index.deleted == [] - - -def test_one_broken_knowledge_base_does_not_stop_the_others( - test_db: Session, test_user: User, index: FakeIndex -): - """They are independent, and the whole point is to keep draining.""" - broken = _knowledge_base(test_db, test_user, "broken", pending=["21"]) - index.explodes_for_kb = {broken.id} - healthy = _knowledge_base(test_db, test_user, "healthy", pending=["22"]) - - sweep_pending_index_cleanup(test_db) - - assert index.deleted == [22] - assert _pending(test_db, healthy) == [] - - -def test_a_knowledge_base_whose_owner_is_gone_is_skipped_not_fatal( - test_db: Session, test_user: User, index: FakeIndex -): - orphaned = _knowledge_base(test_db, test_user, "orphaned", pending=["31"]) - orphaned.user_id = 999999 - healthy = _knowledge_base(test_db, test_user, "healthy", pending=["32"]) - test_db.flush() - - result = sweep_pending_index_cleanup(test_db) - - assert index.deleted == [32] - assert result["swept"] == 1 - assert _pending(test_db, healthy) == [] - - -def test_a_ref_that_can_never_be_deleted_is_dropped_not_retried_forever( - test_db: Session, test_user: User, index: FakeIndex -): - """It is not a document id, so no sweep can ever settle it. Kept, it would fail - on every future sweep and the list would never drain.""" - kb = _knowledge_base(test_db, test_user, "unusable", pending=["not-an-id", "41"]) - - sweep_pending_index_cleanup(test_db) - - assert index.deleted == [41] - assert _pending(test_db, kb) == [] - - -def test_a_ref_parked_twice_is_only_recorded_once( - test_db: Session, test_user: User, index: FakeIndex -): - """The retry writes refs back as strings, so parking an int afterwards would - miss the membership test and accumulate a duplicate.""" - from app.services.knowledge.code_wiki_publisher import _park_unfinished_cleanup - - kb = _knowledge_base(test_db, test_user, "dupes", pending=["51"]) - - _park_unfinished_cleanup(test_db, kb, [51, "51"]) - - assert _pending(test_db, kb) == ["51"] diff --git a/backend/tests/services/knowledge/test_code_wiki_content_write.py b/backend/tests/services/knowledge/test_code_wiki_content_write.py index 81a96ed20b..3f4e2394a3 100644 --- a/backend/tests/services/knowledge/test_code_wiki_content_write.py +++ b/backend/tests/services/knowledge/test_code_wiki_content_write.py @@ -287,3 +287,61 @@ def test_a_payload_with_nothing_in_it_is_still_refused( ) assert exc.value.status_code == 400 + + +# --- reading a page back ---------------------------------------------------- + + +def test_a_written_page_can_be_read_back(test_db: Session, generation: WikiGeneration): + """Without this the instruction to revise a page cannot be followed: the agent + knows the path and cannot see a word of what the page says.""" + _write(test_db, generation, _section("architecture/backend", "Backend", "v1")) + + page = WikiService().get_generation_page( + test_db, generation.id, "architecture/backend" + ) + + assert page is not None + assert page.path == "architecture/backend" + assert page.title == "Backend" + assert page.content == "v1" + + +def test_a_page_is_read_by_the_same_path_that_would_write_it( + test_db: Session, generation: WikiGeneration +): + """Resolving reads and writes differently would let the agent read one page and + overwrite another.""" + _write(test_db, generation, _section("modules/sync", "Sync", "body")) + + page = WikiService().get_generation_page( + test_db, generation.id, " Modules//Sync.md " + ) + + assert page is not None and page.content == "body" + + +def test_a_path_holding_no_page_reads_as_absent( + test_db: Session, generation: WikiGeneration +): + """An answer, not a failure: in an incremental run it means the page is new.""" + assert WikiService().get_generation_page(test_db, generation.id, "nothing") is None + + +def test_a_malformed_path_is_refused_rather_than_read_as_absent( + test_db: Session, generation: WikiGeneration +): + with pytest.raises(HTTPException) as exc: + WikiService().get_generation_page(test_db, generation.id, "../escape") + + assert exc.value.status_code == 400 + + +def test_a_pathless_legacy_entry_is_not_readable( + test_db: Session, generation: WikiGeneration +): + """It has no identity to ask for, and guessing would hand back a page the caller + did not name.""" + _write(test_db, generation, _section(None, "Overview", "legacy")) + + assert WikiService().get_generation_page(test_db, generation.id, "overview") is None diff --git a/backend/tests/services/knowledge/test_code_wiki_projection.py b/backend/tests/services/knowledge/test_code_wiki_projection.py index a8dc0fe58a..f8d4232e98 100644 --- a/backend/tests/services/knowledge/test_code_wiki_projection.py +++ b/backend/tests/services/knowledge/test_code_wiki_projection.py @@ -296,6 +296,46 @@ def test_a_folder_emptied_by_deletion_is_removed( ) +def test_a_whole_branch_emptied_by_deletion_is_removed( + test_db: Session, effects: RecordingEffects +): + """Emptying a leaf empties its parent, and that one its parent. Judging a folder + before its children would keep every level above the last page alive.""" + existing = _existing_document(test_db, "architecture/backend/api", "body") + plan = compute_projection_plan([], [_projected(existing)]) + + _apply(test_db, plan, effects) + + assert ( + test_db.query(KnowledgeFolder) + .filter(KnowledgeFolder.kind_id == KIND_ID) + .count() + == 0 + ) + + +def test_a_branch_is_kept_while_any_page_below_it_survives( + test_db: Session, effects: RecordingEffects +): + """The other half of the same rule: a deep survivor keeps its whole ancestry.""" + doomed = _existing_document(test_db, "architecture/legacy", "body") + kept = _existing_document(test_db, "architecture/backend/api", "body") + plan = compute_projection_plan( + [_source("architecture/backend/api", "body")], + [_projected(doomed), _projected(kept)], + ) + + _apply(test_db, plan, effects) + + names = { + folder.name + for folder in test_db.query(KnowledgeFolder) + .filter(KnowledgeFolder.kind_id == KIND_ID) + .all() + } + assert names == {"architecture", "backend"} + + def test_an_empty_user_folder_is_left_alone( test_db: Session, effects: RecordingEffects ): diff --git a/backend/tests/services/knowledge/test_code_wiki_publisher.py b/backend/tests/services/knowledge/test_code_wiki_publisher.py index b2f9fafbaa..80c12386e7 100644 --- a/backend/tests/services/knowledge/test_code_wiki_publisher.py +++ b/backend/tests/services/knowledge/test_code_wiki_publisher.py @@ -400,3 +400,73 @@ def test_cleanup_that_still_fails_stays_parked(test_db: Session, knowledge_base: def test_the_published_pointer_starts_at_nothing(knowledge_base: Kind): assert published_generation_id(knowledge_base) == 0 assert PUBLISHED_GENERATION_KEY not in knowledge_base.json["spec"] + + +def test_a_publish_settles_what_the_last_one_could_not( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """There is no sweeper. Publishing is the only thing that drains the debt, which + is what keeps this function the single writer of the parked list — two writers + doing read-modify-write on one spec key eventually drop a ref.""" + knowledge_base.json = { + "spec": { + "name": "wiki", + "kbType": "code_wiki", + PENDING_INDEX_CLEANUP_KEY: ["4242"], + } + } + test_db.flush() + + generation = _generation(test_db, knowledge_base.id) + _page(test_db, generation, "index", "body") + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert result.published + assert 4242 in effects.deleted_rag + assert (knowledge_base.json or {})["spec"][PENDING_INDEX_CLEANUP_KEY] == [] + + +def test_a_rejected_publish_still_settles_the_old_debt( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """The debt has nothing to do with whether this version is publishable.""" + first = _generation(test_db, knowledge_base.id) + for path in ("index", "one", "two", "three"): + _page(test_db, first, path, "body") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=first, + user_id=USER_ID, + effects=effects.build(), + ) + _update_spec_pending(test_db, knowledge_base, ["4242"]) + + collapsed = _generation(test_db, knowledge_base.id) + _page(test_db, collapsed, "index", "body") + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=collapsed, + user_id=USER_ID, + effects=effects.build(), + ) + + assert not result.published + assert 4242 in effects.deleted_rag + assert (knowledge_base.json or {})["spec"][PENDING_INDEX_CLEANUP_KEY] == [] + + +def _update_spec_pending(test_db: Session, knowledge_base: Kind, refs: list[str]): + payload = dict(knowledge_base.json or {}) + spec = dict(payload.get("spec", {})) + spec[PENDING_INDEX_CLEANUP_KEY] = refs + payload["spec"] = spec + knowledge_base.json = payload + test_db.flush() From 7931f707e9c6afbbdd9abc0433921ee249699b01 Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Mon, 3 Aug 2026 14:03:49 +0800 Subject: [PATCH 03/14] refactor(knowledge): collect the code wiki into a package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourteen modules named code_wiki_* sat flat in services/knowledge/, which is the point at which a prefix is standing in for a directory. They move to services/knowledge/code_wiki/ with the prefix dropped, mirrored by the tests. This is package-by-feature, matching services/subscription/ next door. mermaid_check moves in with them: the publish gate is its only caller. content_scope stays put, because knowledge_service uses it too. __init__.py re-exports nothing. app.services.knowledge resolves its own exports lazily to avoid import cycles, and eagerly importing this package's modules from there would pull that whole chain in at package-import time. Importing by module path also keeps publisher and publish_gate — different things with similar names — distinguishable at the call site. Dropping the prefix collided test_generation.py with tests/schemas/, because tests/services/knowledge/ had no __init__.py while its siblings all do. Added to both levels, following the convention already in the tree. Renames are recorded as renames, so git log --follow and blame still reach the reasoning behind each of these files. Co-Authored-By: Claude Opus 5 --- backend/app/api/endpoints/knowledge.py | 8 +++--- .../services/knowledge/code_wiki/__init__.py | 26 +++++++++++++++++++ .../generation.py} | 10 +++---- .../{ => code_wiki}/mermaid_check.py | 0 .../page_path.py} | 0 .../projection.py} | 4 +-- .../projection_plan.py} | 2 +- .../prompts.py} | 0 .../publish_gate.py} | 9 ++++--- .../publisher.py} | 8 +++--- .../repo_state.py} | 4 +-- .../run_mode.py} | 0 .../runner.py} | 14 +++++----- .../side_effects.py} | 2 +- .../source.py} | 0 .../version_store.py} | 2 +- .../app/services/knowledge/orchestrator.py | 2 +- backend/app/services/wiki_service.py | 6 ++--- backend/tests/api/test_knowledge_code_wiki.py | 4 +-- backend/tests/services/knowledge/__init__.py | 3 +++ .../services/knowledge/code_wiki/__init__.py | 3 +++ .../test_content_write.py} | 2 +- .../test_generation.py} | 10 +++---- .../{ => code_wiki}/test_mermaid_check.py | 2 +- .../test_page_path.py} | 2 +- .../test_projection.py} | 4 +-- .../test_projection_plan.py} | 2 +- .../test_prompts.py} | 2 +- .../test_publish_end_to_end.py} | 10 +++---- .../test_publish_gate.py} | 4 +-- .../test_publisher.py} | 10 +++---- .../test_repo_state.py} | 8 +++--- .../test_run_mode.py} | 2 +- .../test_runner.py} | 26 +++++++++---------- .../test_source.py} | 24 ++++++++--------- .../test_submit_to_publish.py} | 14 +++++----- .../test_version_store.py} | 4 +-- 37 files changed, 134 insertions(+), 99 deletions(-) create mode 100644 backend/app/services/knowledge/code_wiki/__init__.py rename backend/app/services/knowledge/{code_wiki_generation.py => code_wiki/generation.py} (96%) rename backend/app/services/knowledge/{ => code_wiki}/mermaid_check.py (100%) rename backend/app/services/knowledge/{code_wiki_page_path.py => code_wiki/page_path.py} (100%) rename backend/app/services/knowledge/{code_wiki_projection.py => code_wiki/projection.py} (99%) rename backend/app/services/knowledge/{code_wiki_projection_plan.py => code_wiki/projection_plan.py} (98%) rename backend/app/services/knowledge/{code_wiki_prompts.py => code_wiki/prompts.py} (100%) rename backend/app/services/knowledge/{code_wiki_publish_gate.py => code_wiki/publish_gate.py} (95%) rename backend/app/services/knowledge/{code_wiki_publisher.py => code_wiki/publisher.py} (98%) rename backend/app/services/knowledge/{code_wiki_repo_state.py => code_wiki/repo_state.py} (97%) rename backend/app/services/knowledge/{code_wiki_run_mode.py => code_wiki/run_mode.py} (100%) rename backend/app/services/knowledge/{code_wiki_runner.py => code_wiki/runner.py} (96%) rename backend/app/services/knowledge/{code_wiki_side_effects.py => code_wiki/side_effects.py} (98%) rename backend/app/services/knowledge/{code_wiki_source.py => code_wiki/source.py} (100%) rename backend/app/services/knowledge/{code_wiki_version_store.py => code_wiki/version_store.py} (99%) create mode 100644 backend/tests/services/knowledge/__init__.py create mode 100644 backend/tests/services/knowledge/code_wiki/__init__.py rename backend/tests/services/knowledge/{test_code_wiki_content_write.py => code_wiki/test_content_write.py} (99%) rename backend/tests/services/knowledge/{test_code_wiki_generation.py => code_wiki/test_generation.py} (96%) rename backend/tests/services/knowledge/{ => code_wiki}/test_mermaid_check.py (98%) rename backend/tests/services/knowledge/{test_code_wiki_page_path.py => code_wiki/test_page_path.py} (98%) rename backend/tests/services/knowledge/{test_code_wiki_projection.py => code_wiki/test_projection.py} (99%) rename backend/tests/services/knowledge/{test_code_wiki_projection_plan.py => code_wiki/test_projection_plan.py} (98%) rename backend/tests/services/knowledge/{test_code_wiki_prompts.py => code_wiki/test_prompts.py} (99%) rename backend/tests/services/knowledge/{test_code_wiki_publish_end_to_end.py => code_wiki/test_publish_end_to_end.py} (97%) rename backend/tests/services/knowledge/{test_code_wiki_publish_gate.py => code_wiki/test_publish_gate.py} (97%) rename backend/tests/services/knowledge/{test_code_wiki_publisher.py => code_wiki/test_publisher.py} (97%) rename backend/tests/services/knowledge/{test_code_wiki_repo_state.py => code_wiki/test_repo_state.py} (96%) rename backend/tests/services/knowledge/{test_code_wiki_run_mode.py => code_wiki/test_run_mode.py} (99%) rename backend/tests/services/knowledge/{test_code_wiki_runner.py => code_wiki/test_runner.py} (95%) rename backend/tests/services/knowledge/{test_code_wiki_source.py => code_wiki/test_source.py} (91%) rename backend/tests/services/knowledge/{test_code_wiki_submit_to_publish.py => code_wiki/test_submit_to_publish.py} (95%) rename backend/tests/services/knowledge/{test_code_wiki_version_store.py => code_wiki/test_version_store.py} (99%) diff --git a/backend/app/api/endpoints/knowledge.py b/backend/app/api/endpoints/knowledge.py index c1491abaa9..70bc9538db 100644 --- a/backend/app/api/endpoints/knowledge.py +++ b/backend/app/api/endpoints/knowledge.py @@ -68,10 +68,10 @@ KnowledgeService, knowledge_base_qa_service, ) -from app.services.knowledge.code_wiki_generation import GenerationInFlight -from app.services.knowledge.code_wiki_run_mode import ChangedPath -from app.services.knowledge.code_wiki_runner import CodeWikiRunError, start_run -from app.services.knowledge.code_wiki_source import ( +from app.services.knowledge.code_wiki.generation import GenerationInFlight +from app.services.knowledge.code_wiki.run_mode import ChangedPath +from app.services.knowledge.code_wiki.runner import CodeWikiRunError, start_run +from app.services.knowledge.code_wiki.source import ( SourceAccessDenied, SourceRepository, assert_user_can_read_source, diff --git a/backend/app/services/knowledge/code_wiki/__init__.py b/backend/app/services/knowledge/code_wiki/__init__.py new file mode 100644 index 0000000000..6b5336fb2b --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/__init__.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""A knowledge base generated and maintained by an agent from a source repository. + +The modules divide along what each one decides: + +- ``source`` / ``repo_state`` — which repository, and what it is at right now. +- ``run_mode`` — whether a run is needed at all, and how much of the wiki it rebuilds. +- ``version_store`` / ``page_path`` — the versions the agent writes into, and the page + identity that lets an unchanged page keep its document id across runs. +- ``prompts`` — what the agent is told, which differs by mode. +- ``projection_plan`` / ``projection`` — what publishing a version would change, and + the fixed ordering that applies it. +- ``publish_gate`` / ``publisher`` — whether a finished version may go live, and the + single place that moves ``spec.publishedGenerationId``. +- ``generation`` / ``runner`` — the spine: starting a run and concluding it. +- ``side_effects`` — the adapters for the work that cannot join a transaction. + +Nothing is re-exported here on purpose. ``app.services.knowledge`` resolves its own +exports lazily to avoid import cycles, and eagerly importing this package's modules +from here would pull that whole chain in at package-import time. Importing by module +path also keeps neighbours like ``publisher`` and ``publish_gate`` distinguishable at +the call site. +""" diff --git a/backend/app/services/knowledge/code_wiki_generation.py b/backend/app/services/knowledge/code_wiki/generation.py similarity index 96% rename from backend/app/services/knowledge/code_wiki_generation.py rename to backend/app/services/knowledge/code_wiki/generation.py index 5c1d63fe68..c476514dc9 100644 --- a/backend/app/services/knowledge/code_wiki_generation.py +++ b/backend/app/services/knowledge/code_wiki/generation.py @@ -29,22 +29,22 @@ from app.models.kind import Kind from app.models.user import User from app.models.wiki import WikiGeneration, WikiGenerationStatus, WikiGenerationType -from app.services.knowledge.code_wiki_projection import ProjectionSideEffects -from app.services.knowledge.code_wiki_publish_gate import PublishPolicy -from app.services.knowledge.code_wiki_publisher import ( +from app.services.knowledge.code_wiki.projection import ProjectionSideEffects +from app.services.knowledge.code_wiki.publish_gate import PublishPolicy +from app.services.knowledge.code_wiki.publisher import ( PublishResult, publish_generation, published_generation_id, read_version_pages, ) -from app.services.knowledge.code_wiki_run_mode import ( +from app.services.knowledge.code_wiki.run_mode import ( ChangedPath, RunMode, RunModeDecision, RunModePolicy, decide_run_mode, ) -from app.services.knowledge.code_wiki_version_store import ( +from app.services.knowledge.code_wiki.version_store import ( reclaim_stale_generations, seed_from_published, ) diff --git a/backend/app/services/knowledge/mermaid_check.py b/backend/app/services/knowledge/code_wiki/mermaid_check.py similarity index 100% rename from backend/app/services/knowledge/mermaid_check.py rename to backend/app/services/knowledge/code_wiki/mermaid_check.py diff --git a/backend/app/services/knowledge/code_wiki_page_path.py b/backend/app/services/knowledge/code_wiki/page_path.py similarity index 100% rename from backend/app/services/knowledge/code_wiki_page_path.py rename to backend/app/services/knowledge/code_wiki/page_path.py diff --git a/backend/app/services/knowledge/code_wiki_projection.py b/backend/app/services/knowledge/code_wiki/projection.py similarity index 99% rename from backend/app/services/knowledge/code_wiki_projection.py rename to backend/app/services/knowledge/code_wiki/projection.py index 2144cf60d3..97c9114c30 100644 --- a/backend/app/services/knowledge/code_wiki_projection.py +++ b/backend/app/services/knowledge/code_wiki/projection.py @@ -36,8 +36,8 @@ from sqlalchemy.orm import Session from app.models.knowledge import ContentOrigin, KnowledgeDocument -from app.services.knowledge.code_wiki_page_path import split_page_path -from app.services.knowledge.code_wiki_projection_plan import ( +from app.services.knowledge.code_wiki.page_path import split_page_path +from app.services.knowledge.code_wiki.projection_plan import ( CONTENT_HASH_KEY, PAGE_PATH_KEY, PageSource, diff --git a/backend/app/services/knowledge/code_wiki_projection_plan.py b/backend/app/services/knowledge/code_wiki/projection_plan.py similarity index 98% rename from backend/app/services/knowledge/code_wiki_projection_plan.py rename to backend/app/services/knowledge/code_wiki/projection_plan.py index 11583593d6..2399af7b78 100644 --- a/backend/app/services/knowledge/code_wiki_projection_plan.py +++ b/backend/app/services/knowledge/code_wiki/projection_plan.py @@ -22,7 +22,7 @@ from dataclasses import dataclass from typing import Iterable, Mapping -from app.services.knowledge.code_wiki_page_path import collation_key +from app.services.knowledge.code_wiki.page_path import collation_key # Keys under which the projection records a page's identity and content fingerprint on # the document it owns. They live in ``source_config`` (an existing JSON column) rather diff --git a/backend/app/services/knowledge/code_wiki_prompts.py b/backend/app/services/knowledge/code_wiki/prompts.py similarity index 100% rename from backend/app/services/knowledge/code_wiki_prompts.py rename to backend/app/services/knowledge/code_wiki/prompts.py diff --git a/backend/app/services/knowledge/code_wiki_publish_gate.py b/backend/app/services/knowledge/code_wiki/publish_gate.py similarity index 95% rename from backend/app/services/knowledge/code_wiki_publish_gate.py rename to backend/app/services/knowledge/code_wiki/publish_gate.py index 0b05259d3d..206d575153 100644 --- a/backend/app/services/knowledge/code_wiki_publish_gate.py +++ b/backend/app/services/knowledge/code_wiki/publish_gate.py @@ -28,9 +28,12 @@ from dataclasses import dataclass, field from typing import Optional, Sequence -from app.services.knowledge.code_wiki_page_path import collation_key -from app.services.knowledge.code_wiki_projection_plan import PageSource -from app.services.knowledge.mermaid_check import check_mermaid_blocks, describe_warnings +from app.services.knowledge.code_wiki.mermaid_check import ( + check_mermaid_blocks, + describe_warnings, +) +from app.services.knowledge.code_wiki.page_path import collation_key +from app.services.knowledge.code_wiki.projection_plan import PageSource logger = logging.getLogger(__name__) diff --git a/backend/app/services/knowledge/code_wiki_publisher.py b/backend/app/services/knowledge/code_wiki/publisher.py similarity index 98% rename from backend/app/services/knowledge/code_wiki_publisher.py rename to backend/app/services/knowledge/code_wiki/publisher.py index 4840fddfb8..98fcf48533 100644 --- a/backend/app/services/knowledge/code_wiki_publisher.py +++ b/backend/app/services/knowledge/code_wiki/publisher.py @@ -37,13 +37,13 @@ from app.models.kind import Kind from app.models.knowledge import KnowledgeDocument from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus -from app.services.knowledge.code_wiki_projection import ( +from app.services.knowledge.code_wiki.projection import ( PENDING_INDEX_CLEANUP_KEY, ProjectionSideEffects, apply_projection_plan, finish_projection, ) -from app.services.knowledge.code_wiki_projection_plan import ( +from app.services.knowledge.code_wiki.projection_plan import ( CONTENT_HASH_KEY, PAGE_PATH_KEY, PageSource, @@ -51,13 +51,13 @@ ProjectionPlan, compute_projection_plan, ) -from app.services.knowledge.code_wiki_publish_gate import ( +from app.services.knowledge.code_wiki.publish_gate import ( PUBLISH_GATE_EXT_KEY, GateVerdict, PublishPolicy, evaluate_publish_gate, ) -from app.services.knowledge.code_wiki_version_store import page_path_of +from app.services.knowledge.code_wiki.version_store import page_path_of from app.services.knowledge.content_scope import generated_wiki_pages logger = logging.getLogger(__name__) diff --git a/backend/app/services/knowledge/code_wiki_repo_state.py b/backend/app/services/knowledge/code_wiki/repo_state.py similarity index 97% rename from backend/app/services/knowledge/code_wiki_repo_state.py rename to backend/app/services/knowledge/code_wiki/repo_state.py index 8f7ff6f4f7..137730985e 100644 --- a/backend/app/services/knowledge/code_wiki_repo_state.py +++ b/backend/app/services/knowledge/code_wiki/repo_state.py @@ -28,8 +28,8 @@ from sqlalchemy.orm import Session from app.services.git_skill.utils import get_user_git_info -from app.services.knowledge.code_wiki_run_mode import ChangedPath -from app.services.knowledge.code_wiki_source import SourceRepository, provider_for +from app.services.knowledge.code_wiki.run_mode import ChangedPath +from app.services.knowledge.code_wiki.source import SourceRepository, provider_for logger = logging.getLogger(__name__) diff --git a/backend/app/services/knowledge/code_wiki_run_mode.py b/backend/app/services/knowledge/code_wiki/run_mode.py similarity index 100% rename from backend/app/services/knowledge/code_wiki_run_mode.py rename to backend/app/services/knowledge/code_wiki/run_mode.py diff --git a/backend/app/services/knowledge/code_wiki_runner.py b/backend/app/services/knowledge/code_wiki/runner.py similarity index 96% rename from backend/app/services/knowledge/code_wiki_runner.py rename to backend/app/services/knowledge/code_wiki/runner.py index b0d7a8f33d..bb8e3e9521 100644 --- a/backend/app/services/knowledge/code_wiki_runner.py +++ b/backend/app/services/knowledge/code_wiki/runner.py @@ -35,18 +35,18 @@ from app.models.wiki import WikiGeneration, WikiGenerationStatus from app.schemas.knowledge import KnowledgeBaseType from app.schemas.task import TaskCreate -from app.services.knowledge.code_wiki_generation import ( +from app.services.knowledge.code_wiki.generation import ( SOURCE_COMMIT_KEY, finish_generation, published_commit, start_generation, ) -from app.services.knowledge.code_wiki_prompts import WikiRunContext, build_prompt -from app.services.knowledge.code_wiki_publisher import PublishResult, read_version_pages -from app.services.knowledge.code_wiki_repo_state import read_repository_state -from app.services.knowledge.code_wiki_run_mode import ChangedPath, RunMode -from app.services.knowledge.code_wiki_side_effects import build_projection_side_effects -from app.services.knowledge.code_wiki_source import SourceRepository +from app.services.knowledge.code_wiki.prompts import WikiRunContext, build_prompt +from app.services.knowledge.code_wiki.publisher import PublishResult, read_version_pages +from app.services.knowledge.code_wiki.repo_state import read_repository_state +from app.services.knowledge.code_wiki.run_mode import ChangedPath, RunMode +from app.services.knowledge.code_wiki.side_effects import build_projection_side_effects +from app.services.knowledge.code_wiki.source import SourceRepository logger = logging.getLogger(__name__) diff --git a/backend/app/services/knowledge/code_wiki_side_effects.py b/backend/app/services/knowledge/code_wiki/side_effects.py similarity index 98% rename from backend/app/services/knowledge/code_wiki_side_effects.py rename to backend/app/services/knowledge/code_wiki/side_effects.py index b8e31cf30b..7436a3e663 100644 --- a/backend/app/services/knowledge/code_wiki_side_effects.py +++ b/backend/app/services/knowledge/code_wiki/side_effects.py @@ -20,7 +20,7 @@ from app.models.kind import Kind from app.models.knowledge import KnowledgeDocument from app.models.user import User -from app.services.knowledge.code_wiki_projection import ProjectionSideEffects +from app.services.knowledge.code_wiki.projection import ProjectionSideEffects logger = logging.getLogger(__name__) diff --git a/backend/app/services/knowledge/code_wiki_source.py b/backend/app/services/knowledge/code_wiki/source.py similarity index 100% rename from backend/app/services/knowledge/code_wiki_source.py rename to backend/app/services/knowledge/code_wiki/source.py diff --git a/backend/app/services/knowledge/code_wiki_version_store.py b/backend/app/services/knowledge/code_wiki/version_store.py similarity index 99% rename from backend/app/services/knowledge/code_wiki_version_store.py rename to backend/app/services/knowledge/code_wiki/version_store.py index d2e19a9e66..013d23cc58 100644 --- a/backend/app/services/knowledge/code_wiki_version_store.py +++ b/backend/app/services/knowledge/code_wiki/version_store.py @@ -32,7 +32,7 @@ from sqlalchemy.orm import Session from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus -from app.services.knowledge.code_wiki_page_path import ( +from app.services.knowledge.code_wiki.page_path import ( collation_key, normalize_page_path, ) diff --git a/backend/app/services/knowledge/orchestrator.py b/backend/app/services/knowledge/orchestrator.py index 288eb708c6..8f5e760a24 100644 --- a/backend/app/services/knowledge/orchestrator.py +++ b/backend/app/services/knowledge/orchestrator.py @@ -42,7 +42,7 @@ KnowledgeDocumentResponse, ResourceScope, ) -from app.services.knowledge.code_wiki_source import SourceRepository +from app.services.knowledge.code_wiki.source import SourceRepository from app.services.knowledge.document_read_service import ( DOCUMENT_READ_ERROR_NOT_FOUND, document_read_service, diff --git a/backend/app/services/wiki_service.py b/backend/app/services/wiki_service.py index 047af02d45..e491963fde 100644 --- a/backend/app/services/wiki_service.py +++ b/backend/app/services/wiki_service.py @@ -31,14 +31,14 @@ ) from app.services.adapters.task_kinds import task_kinds_service from app.services.adapters.team_kinds import team_kinds_service -from app.services.knowledge.code_wiki_page_path import ( +from app.services.knowledge.code_wiki.page_path import ( InvalidPagePath, assert_unique_within_version, collation_key, normalize_page_path, ) -from app.services.knowledge.code_wiki_runner import finish_run, is_code_wiki_generation -from app.services.knowledge.code_wiki_version_store import ( +from app.services.knowledge.code_wiki.runner import finish_run, is_code_wiki_generation +from app.services.knowledge.code_wiki.version_store import ( page_path_of, remove_page, set_page_path, diff --git a/backend/tests/api/test_knowledge_code_wiki.py b/backend/tests/api/test_knowledge_code_wiki.py index 3b763dd067..558deddcdb 100644 --- a/backend/tests/api/test_knowledge_code_wiki.py +++ b/backend/tests/api/test_knowledge_code_wiki.py @@ -71,7 +71,7 @@ def test_creation_is_refused_without_repository_access( test_client: TestClient, auth_headers: dict[str, str] ): """A wiki must not be built for a repository the requester cannot read.""" - from app.services.knowledge.code_wiki_source import SourceAccessDenied + from app.services.knowledge.code_wiki.source import SourceAccessDenied with patch( "app.api.endpoints.knowledge.assert_user_can_read_source", @@ -197,7 +197,7 @@ def test_a_second_run_while_one_is_live_is_a_conflict( auth_headers: dict[str, str], kind_services_use_test_db, ): - from app.services.knowledge.code_wiki_generation import GenerationInFlight + from app.services.knowledge.code_wiki.generation import GenerationInFlight kb_id = _create_wiki(test_client, auth_headers) diff --git a/backend/tests/services/knowledge/__init__.py b/backend/tests/services/knowledge/__init__.py new file mode 100644 index 0000000000..fd017d8796 --- /dev/null +++ b/backend/tests/services/knowledge/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/backend/tests/services/knowledge/code_wiki/__init__.py b/backend/tests/services/knowledge/code_wiki/__init__.py new file mode 100644 index 0000000000..fd017d8796 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/backend/tests/services/knowledge/test_code_wiki_content_write.py b/backend/tests/services/knowledge/code_wiki/test_content_write.py similarity index 99% rename from backend/tests/services/knowledge/test_code_wiki_content_write.py rename to backend/tests/services/knowledge/code_wiki/test_content_write.py index 3f4e2394a3..a0c97e7f69 100644 --- a/backend/tests/services/knowledge/test_code_wiki_content_write.py +++ b/backend/tests/services/knowledge/code_wiki/test_content_write.py @@ -21,7 +21,7 @@ WikiGenerationType, ) from app.schemas.wiki import WikiContentSection, WikiContentWriteRequest -from app.services.knowledge.code_wiki_version_store import page_path_of +from app.services.knowledge.code_wiki.version_store import page_path_of from app.services.wiki_service import WikiService KIND_ID = 91 diff --git a/backend/tests/services/knowledge/test_code_wiki_generation.py b/backend/tests/services/knowledge/code_wiki/test_generation.py similarity index 96% rename from backend/tests/services/knowledge/test_code_wiki_generation.py rename to backend/tests/services/knowledge/code_wiki/test_generation.py index fe0ee1f318..7c9d5d3539 100644 --- a/backend/tests/services/knowledge/test_code_wiki_generation.py +++ b/backend/tests/services/knowledge/code_wiki/test_generation.py @@ -19,16 +19,16 @@ from app.models.knowledge import KnowledgeDocument from app.models.user import User from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus -from app.services.knowledge.code_wiki_generation import ( +from app.services.knowledge.code_wiki.generation import ( GenerationInFlight, finish_generation, published_commit, start_generation, ) -from app.services.knowledge.code_wiki_projection import ProjectionSideEffects -from app.services.knowledge.code_wiki_publisher import published_generation_id -from app.services.knowledge.code_wiki_run_mode import ChangedPath, RunMode -from app.services.knowledge.code_wiki_version_store import ( +from app.services.knowledge.code_wiki.projection import ProjectionSideEffects +from app.services.knowledge.code_wiki.publisher import published_generation_id +from app.services.knowledge.code_wiki.run_mode import ChangedPath, RunMode +from app.services.knowledge.code_wiki.version_store import ( STALE_RUN_AFTER_HOURS, set_page_path, ) diff --git a/backend/tests/services/knowledge/test_mermaid_check.py b/backend/tests/services/knowledge/code_wiki/test_mermaid_check.py similarity index 98% rename from backend/tests/services/knowledge/test_mermaid_check.py rename to backend/tests/services/knowledge/code_wiki/test_mermaid_check.py index 47d65171b8..8e3a966306 100644 --- a/backend/tests/services/knowledge/test_mermaid_check.py +++ b/backend/tests/services/knowledge/code_wiki/test_mermaid_check.py @@ -6,7 +6,7 @@ import pytest -from app.services.knowledge.mermaid_check import ( +from app.services.knowledge.code_wiki.mermaid_check import ( check_mermaid_blocks, describe_warnings, ) diff --git a/backend/tests/services/knowledge/test_code_wiki_page_path.py b/backend/tests/services/knowledge/code_wiki/test_page_path.py similarity index 98% rename from backend/tests/services/knowledge/test_code_wiki_page_path.py rename to backend/tests/services/knowledge/code_wiki/test_page_path.py index e364f7e93f..c06dcc975a 100644 --- a/backend/tests/services/knowledge/test_code_wiki_page_path.py +++ b/backend/tests/services/knowledge/code_wiki/test_page_path.py @@ -11,7 +11,7 @@ import pytest -from app.services.knowledge.code_wiki_page_path import ( +from app.services.knowledge.code_wiki.page_path import ( MAX_DIRECTORY_DEPTH, MAX_PATH_LENGTH, MAX_SEGMENT_LENGTH, diff --git a/backend/tests/services/knowledge/test_code_wiki_projection.py b/backend/tests/services/knowledge/code_wiki/test_projection.py similarity index 99% rename from backend/tests/services/knowledge/test_code_wiki_projection.py rename to backend/tests/services/knowledge/code_wiki/test_projection.py index f8d4232e98..af608ee65d 100644 --- a/backend/tests/services/knowledge/test_code_wiki_projection.py +++ b/backend/tests/services/knowledge/code_wiki/test_projection.py @@ -15,12 +15,12 @@ from sqlalchemy.orm import Session from app.models.knowledge import ContentOrigin, KnowledgeDocument, KnowledgeFolder -from app.services.knowledge.code_wiki_projection import ( +from app.services.knowledge.code_wiki.projection import ( ProjectionSideEffects, apply_projection_plan, finish_projection, ) -from app.services.knowledge.code_wiki_projection_plan import ( +from app.services.knowledge.code_wiki.projection_plan import ( CONTENT_HASH_KEY, PAGE_PATH_KEY, PageSource, diff --git a/backend/tests/services/knowledge/test_code_wiki_projection_plan.py b/backend/tests/services/knowledge/code_wiki/test_projection_plan.py similarity index 98% rename from backend/tests/services/knowledge/test_code_wiki_projection_plan.py rename to backend/tests/services/knowledge/code_wiki/test_projection_plan.py index b8774dbfb1..fcd14a4086 100644 --- a/backend/tests/services/knowledge/test_code_wiki_projection_plan.py +++ b/backend/tests/services/knowledge/code_wiki/test_projection_plan.py @@ -8,7 +8,7 @@ it deletes, so these tests are mostly about it not reaching further than it should. """ -from app.services.knowledge.code_wiki_projection_plan import ( +from app.services.knowledge.code_wiki.projection_plan import ( PageSource, ProjectedPage, compute_projection_plan, diff --git a/backend/tests/services/knowledge/test_code_wiki_prompts.py b/backend/tests/services/knowledge/code_wiki/test_prompts.py similarity index 99% rename from backend/tests/services/knowledge/test_code_wiki_prompts.py rename to backend/tests/services/knowledge/code_wiki/test_prompts.py index 88db3ece5a..622dd2a55d 100644 --- a/backend/tests/services/knowledge/test_code_wiki_prompts.py +++ b/backend/tests/services/knowledge/code_wiki/test_prompts.py @@ -10,7 +10,7 @@ loss, so those are pinned here. """ -from app.services.knowledge.code_wiki_prompts import ( +from app.services.knowledge.code_wiki.prompts import ( REQUIRED_COVERAGE, WikiRunContext, build_diagram_correction, diff --git a/backend/tests/services/knowledge/test_code_wiki_publish_end_to_end.py b/backend/tests/services/knowledge/code_wiki/test_publish_end_to_end.py similarity index 97% rename from backend/tests/services/knowledge/test_code_wiki_publish_end_to_end.py rename to backend/tests/services/knowledge/code_wiki/test_publish_end_to_end.py index c0b6c104c9..4ca89ab85e 100644 --- a/backend/tests/services/knowledge/test_code_wiki_publish_end_to_end.py +++ b/backend/tests/services/knowledge/code_wiki/test_publish_end_to_end.py @@ -31,15 +31,15 @@ WikiGenerationStatus, WikiGenerationType, ) -from app.services.knowledge.code_wiki_projection_plan import PAGE_PATH_KEY -from app.services.knowledge.code_wiki_publisher import ( +from app.services.knowledge.code_wiki.projection_plan import PAGE_PATH_KEY +from app.services.knowledge.code_wiki.publisher import ( publish_generation, published_generation_id, ) -from app.services.knowledge.code_wiki_side_effects import ( +from app.services.knowledge.code_wiki.side_effects import ( build_projection_side_effects, ) -from app.services.knowledge.code_wiki_version_store import set_page_path +from app.services.knowledge.code_wiki.version_store import set_page_path from app.services.knowledge.index_state_machine import ( mark_document_index_succeeded, ) @@ -100,7 +100,7 @@ def _publish(test_db, knowledge_base, generation, test_user, enqueued): test_db, knowledge_base=knowledge_base, user=test_user ) with patch( - "app.services.knowledge.code_wiki_side_effects._enqueue_reindex", + "app.services.knowledge.code_wiki.side_effects._enqueue_reindex", side_effect=lambda db, **kw: enqueued.append(kw["document_id"]), ): return publish_generation( diff --git a/backend/tests/services/knowledge/test_code_wiki_publish_gate.py b/backend/tests/services/knowledge/code_wiki/test_publish_gate.py similarity index 97% rename from backend/tests/services/knowledge/test_code_wiki_publish_gate.py rename to backend/tests/services/knowledge/code_wiki/test_publish_gate.py index e9f73effc5..c273106c0c 100644 --- a/backend/tests/services/knowledge/test_code_wiki_publish_gate.py +++ b/backend/tests/services/knowledge/code_wiki/test_publish_gate.py @@ -11,8 +11,8 @@ set of paths currently published rather than with a number. """ -from app.services.knowledge.code_wiki_projection_plan import PageSource -from app.services.knowledge.code_wiki_publish_gate import ( +from app.services.knowledge.code_wiki.projection_plan import PageSource +from app.services.knowledge.code_wiki.publish_gate import ( PublishPolicy, evaluate_publish_gate, ) diff --git a/backend/tests/services/knowledge/test_code_wiki_publisher.py b/backend/tests/services/knowledge/code_wiki/test_publisher.py similarity index 97% rename from backend/tests/services/knowledge/test_code_wiki_publisher.py rename to backend/tests/services/knowledge/code_wiki/test_publisher.py index 80c12386e7..e59287803a 100644 --- a/backend/tests/services/knowledge/test_code_wiki_publisher.py +++ b/backend/tests/services/knowledge/code_wiki/test_publisher.py @@ -22,21 +22,21 @@ WikiGenerationStatus, WikiGenerationType, ) -from app.services.knowledge.code_wiki_projection import ( +from app.services.knowledge.code_wiki.projection import ( PENDING_INDEX_CLEANUP_KEY, ProjectionSideEffects, ) -from app.services.knowledge.code_wiki_publish_gate import ( +from app.services.knowledge.code_wiki.publish_gate import ( PUBLISH_GATE_EXT_KEY, PublishPolicy, ) -from app.services.knowledge.code_wiki_publisher import ( +from app.services.knowledge.code_wiki.publisher import ( PUBLISHED_GENERATION_KEY, publish_generation, published_generation_id, retry_pending_index_cleanup, ) -from app.services.knowledge.code_wiki_version_store import set_page_path +from app.services.knowledge.code_wiki.version_store import set_page_path USER_ID = 11 @@ -126,7 +126,7 @@ def _page(test_db: Session, generation: WikiGeneration, path: str, content: str) def _live_paths(test_db: Session, kind_id: int) -> set[str]: - from app.services.knowledge.code_wiki_projection_plan import PAGE_PATH_KEY + from app.services.knowledge.code_wiki.projection_plan import PAGE_PATH_KEY return { (document.source_config or {}).get(PAGE_PATH_KEY) diff --git a/backend/tests/services/knowledge/test_code_wiki_repo_state.py b/backend/tests/services/knowledge/code_wiki/test_repo_state.py similarity index 96% rename from backend/tests/services/knowledge/test_code_wiki_repo_state.py rename to backend/tests/services/knowledge/code_wiki/test_repo_state.py index 8e3cd52797..ce9bc65baa 100644 --- a/backend/tests/services/knowledge/test_code_wiki_repo_state.py +++ b/backend/tests/services/knowledge/code_wiki/test_repo_state.py @@ -19,11 +19,11 @@ import pytest from sqlalchemy.orm import Session -from app.services.knowledge.code_wiki_repo_state import ( +from app.services.knowledge.code_wiki.repo_state import ( RepositoryState, read_repository_state, ) -from app.services.knowledge.code_wiki_source import ( +from app.services.knowledge.code_wiki.source import ( SUPPORTED_SOURCE_TYPES, SourceRepository, provider_for, @@ -66,11 +66,11 @@ def _with(provider, token="t0ken"): """Patch the provider lookup and credentials this module reaches for.""" return ( patch( - "app.services.knowledge.code_wiki_repo_state.provider_for", + "app.services.knowledge.code_wiki.repo_state.provider_for", return_value=provider, ), patch( - "app.services.knowledge.code_wiki_repo_state.get_user_git_info", + "app.services.knowledge.code_wiki.repo_state.get_user_git_info", return_value=({"token": token} if token else None), ), ) diff --git a/backend/tests/services/knowledge/test_code_wiki_run_mode.py b/backend/tests/services/knowledge/code_wiki/test_run_mode.py similarity index 99% rename from backend/tests/services/knowledge/test_code_wiki_run_mode.py rename to backend/tests/services/knowledge/code_wiki/test_run_mode.py index 14c5c65590..f0f07ea63f 100644 --- a/backend/tests/services/knowledge/test_code_wiki_run_mode.py +++ b/backend/tests/services/knowledge/code_wiki/test_run_mode.py @@ -8,7 +8,7 @@ these tests guard against the failure that would delete a whole wiki. """ -from app.services.knowledge.code_wiki_run_mode import ( +from app.services.knowledge.code_wiki.run_mode import ( ChangedPath, RunMode, RunModeDecision, diff --git a/backend/tests/services/knowledge/test_code_wiki_runner.py b/backend/tests/services/knowledge/code_wiki/test_runner.py similarity index 95% rename from backend/tests/services/knowledge/test_code_wiki_runner.py rename to backend/tests/services/knowledge/code_wiki/test_runner.py index b5a1e80c1f..81f521d6ba 100644 --- a/backend/tests/services/knowledge/test_code_wiki_runner.py +++ b/backend/tests/services/knowledge/code_wiki/test_runner.py @@ -20,17 +20,17 @@ from app.models.user import User from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus from app.schemas.task import TaskCreate -from app.services.knowledge.code_wiki_generation import published_commit -from app.services.knowledge.code_wiki_publisher import published_generation_id -from app.services.knowledge.code_wiki_run_mode import ChangedPath -from app.services.knowledge.code_wiki_runner import ( +from app.services.knowledge.code_wiki.generation import published_commit +from app.services.knowledge.code_wiki.publisher import published_generation_id +from app.services.knowledge.code_wiki.run_mode import ChangedPath +from app.services.knowledge.code_wiki.runner import ( CodeWikiRunError, finish_run, is_code_wiki_generation, source_of, start_run, ) -from app.services.knowledge.code_wiki_version_store import set_page_path +from app.services.knowledge.code_wiki.version_store import set_page_path HEAD = "aaaaaaa" NEXT_HEAD = "bbbbbbb" @@ -128,12 +128,12 @@ def _write(self, *, filename: str, content: str) -> int: @pytest.fixture def no_side_effects(monkeypatch) -> FakeEffects: """Publish without touching attachment storage, the index or the queue.""" - from app.services.knowledge import code_wiki_runner - from app.services.knowledge.code_wiki_projection import ProjectionSideEffects + from app.services.knowledge.code_wiki import runner + from app.services.knowledge.code_wiki.projection import ProjectionSideEffects fake = FakeEffects() monkeypatch.setattr( - code_wiki_runner, + runner, "build_projection_side_effects", lambda db, *, knowledge_base, user: ProjectionSideEffects( write_attachment=fake._write, @@ -436,11 +436,11 @@ def test_a_run_belonging_to_no_knowledge_base_is_not_a_code_wiki_run( def _repository_at(monkeypatch, head: str, changed=None): """Answer as the provider would, without reaching one.""" - from app.services.knowledge import code_wiki_runner - from app.services.knowledge.code_wiki_repo_state import RepositoryState + from app.services.knowledge.code_wiki import runner + from app.services.knowledge.code_wiki.repo_state import RepositoryState monkeypatch.setattr( - code_wiki_runner, + runner, "read_repository_state", lambda db, *, user_id, source, since_commit: RepositoryState( head_commit=head, branch="main", changed_paths=changed @@ -510,12 +510,12 @@ def test_a_supplied_commit_is_not_second_guessed( tasks: FakeTasks, ): """A caller that knows the commit — a webhook, a test — must be believed.""" - from app.services.knowledge import code_wiki_runner + from app.services.knowledge.code_wiki import runner def refuse(*args, **kwargs): raise AssertionError("the provider must not be consulted") - monkeypatch.setattr(code_wiki_runner, "read_repository_state", refuse) + monkeypatch.setattr(runner, "read_repository_state", refuse) started = start_run( test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD diff --git a/backend/tests/services/knowledge/test_code_wiki_source.py b/backend/tests/services/knowledge/code_wiki/test_source.py similarity index 91% rename from backend/tests/services/knowledge/test_code_wiki_source.py rename to backend/tests/services/knowledge/code_wiki/test_source.py index 142cf5f037..e8446e2bf6 100644 --- a/backend/tests/services/knowledge/test_code_wiki_source.py +++ b/backend/tests/services/knowledge/code_wiki/test_source.py @@ -8,7 +8,7 @@ import pytest -from app.services.knowledge.code_wiki_source import ( +from app.services.knowledge.code_wiki.source import ( SourceAccessDenied, SourceRepository, assert_user_can_read_source, @@ -32,11 +32,11 @@ def test_access_granted_returns_provider_details(): with ( patch( - "app.services.knowledge.code_wiki_source.get_user_git_info", + "app.services.knowledge.code_wiki.source.get_user_git_info", return_value={"type": "github", "token": "t0ken"}, ), patch( - "app.services.knowledge.code_wiki_source.provider_for", + "app.services.knowledge.code_wiki.source.provider_for", return_value=provider, ), ): @@ -54,11 +54,11 @@ def test_gitlab_is_identified_by_project_id(): with ( patch( - "app.services.knowledge.code_wiki_source.get_user_git_info", + "app.services.knowledge.code_wiki.source.get_user_git_info", return_value={"type": "gitlab", "token": "t0ken"}, ), patch( - "app.services.knowledge.code_wiki_source.provider_for", + "app.services.knowledge.code_wiki.source.provider_for", return_value=provider, ), ): @@ -71,7 +71,7 @@ def test_gitlab_is_identified_by_project_id(): def test_missing_credentials_are_denied(): with patch( - "app.services.knowledge.code_wiki_source.get_user_git_info", + "app.services.knowledge.code_wiki.source.get_user_git_info", return_value=None, ): with pytest.raises(SourceAccessDenied, match="No credentials configured"): @@ -87,11 +87,11 @@ def test_no_repository_access_is_denied(): with ( patch( - "app.services.knowledge.code_wiki_source.get_user_git_info", + "app.services.knowledge.code_wiki.source.get_user_git_info", return_value={"type": "github", "token": "t0ken"}, ), patch( - "app.services.knowledge.code_wiki_source.provider_for", + "app.services.knowledge.code_wiki.source.provider_for", return_value=provider, ), ): @@ -106,11 +106,11 @@ def test_provider_error_denies_rather_than_allows(): with ( patch( - "app.services.knowledge.code_wiki_source.get_user_git_info", + "app.services.knowledge.code_wiki.source.get_user_git_info", return_value={"type": "github", "token": "t0ken"}, ), patch( - "app.services.knowledge.code_wiki_source.provider_for", + "app.services.knowledge.code_wiki.source.provider_for", return_value=provider, ), ): @@ -165,7 +165,7 @@ def test_an_unusable_url_is_refused(): def test_credentials_for_another_platform_are_refused(): """Asking one provider about another's repository gives a meaningless answer.""" with patch( - "app.services.knowledge.code_wiki_source.get_user_git_info", + "app.services.knowledge.code_wiki.source.get_user_git_info", return_value={"type": "gitlab", "token": "t0ken"}, ): with pytest.raises(SourceAccessDenied, match="configured as 'gitlab'"): @@ -239,7 +239,7 @@ def test_a_credential_record_without_a_usable_token_denies_access(git_info): """Every shape of "configured but unusable" has to deny. A record that exists but carries no token would otherwise reach the provider with ``None``.""" with patch( - "app.services.knowledge.code_wiki_source.get_user_git_info", + "app.services.knowledge.code_wiki.source.get_user_git_info", return_value=git_info, ): with pytest.raises(SourceAccessDenied, match="No credentials configured"): diff --git a/backend/tests/services/knowledge/test_code_wiki_submit_to_publish.py b/backend/tests/services/knowledge/code_wiki/test_submit_to_publish.py similarity index 95% rename from backend/tests/services/knowledge/test_code_wiki_submit_to_publish.py rename to backend/tests/services/knowledge/code_wiki/test_submit_to_publish.py index e48a5bf978..597f103c33 100644 --- a/backend/tests/services/knowledge/test_code_wiki_submit_to_publish.py +++ b/backend/tests/services/knowledge/code_wiki/test_submit_to_publish.py @@ -31,10 +31,10 @@ WikiContentSummary, WikiContentWriteRequest, ) -from app.services.knowledge.code_wiki_generation import published_commit -from app.services.knowledge.code_wiki_publish_gate import PUBLISH_GATE_EXT_KEY -from app.services.knowledge.code_wiki_publisher import published_generation_id -from app.services.knowledge.code_wiki_version_store import set_page_path +from app.services.knowledge.code_wiki.generation import published_commit +from app.services.knowledge.code_wiki.publish_gate import PUBLISH_GATE_EXT_KEY +from app.services.knowledge.code_wiki.publisher import published_generation_id +from app.services.knowledge.code_wiki.version_store import set_page_path from app.services.wiki_service import WikiService HEAD = "aaaaaaa" @@ -53,12 +53,12 @@ def _write(self, *, filename: str, content: str) -> int: @pytest.fixture def no_side_effects(monkeypatch) -> FakeEffects: - from app.services.knowledge import code_wiki_runner - from app.services.knowledge.code_wiki_projection import ProjectionSideEffects + from app.services.knowledge.code_wiki import runner + from app.services.knowledge.code_wiki.projection import ProjectionSideEffects fake = FakeEffects() monkeypatch.setattr( - code_wiki_runner, + runner, "build_projection_side_effects", lambda db, *, knowledge_base, user: ProjectionSideEffects( write_attachment=fake._write, diff --git a/backend/tests/services/knowledge/test_code_wiki_version_store.py b/backend/tests/services/knowledge/code_wiki/test_version_store.py similarity index 99% rename from backend/tests/services/knowledge/test_code_wiki_version_store.py rename to backend/tests/services/knowledge/code_wiki/test_version_store.py index 6547d365a5..a50cc2d7fe 100644 --- a/backend/tests/services/knowledge/test_code_wiki_version_store.py +++ b/backend/tests/services/knowledge/code_wiki/test_version_store.py @@ -19,8 +19,8 @@ WikiGenerationStatus, WikiGenerationType, ) -from app.services.knowledge.code_wiki_page_path import InvalidPagePath -from app.services.knowledge.code_wiki_version_store import ( +from app.services.knowledge.code_wiki.page_path import InvalidPagePath +from app.services.knowledge.code_wiki.version_store import ( STALE_RUN_AFTER_HOURS, apply_retention, page_path_of, From 783c460e004fd48af0ee5577640bb426bcc87537 Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Mon, 3 Aug 2026 20:41:02 +0800 Subject: [PATCH 04/14] feat(knowledge): give a code wiki page a name, a place and an order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The frontend builds its navigation from page paths, and three things it needs were either discarded or never recorded. **The title was thrown away.** Documents were named after the path's last segment, so "Backend Architecture" at architecture/backend became a document called `backend`, and two pages at architecture/backend and services/backend were indistinguishable anywhere the path was not also shown. The title is now the document's name; the path stays the identity in source_config, so rewording a heading renames the page in place and the id the RAG index is keyed on survives. That inverts an earlier trade-off, which a test recorded as "the projection compares content, not headings". That was right while the title was discarded. Now that it reaches the knowledge base, the fingerprint has to cover it, and the cost is rewriting an attachment whose bytes did not change — paid rarely, because a title almost always moves with the body it heads. **Order was nowhere.** The agent has been sending structure_order since this work began and it was recorded and never read. Paths carry hierarchy and say nothing about which section comes first, and alphabetically "api" precedes the overview. It is now written to spec.pageOrder — as one array in the publish transaction, not per document, because a reorder must not touch a page whose content did not change: the fingerprint would still match, the projection would skip it, and the new position would never land. **lastPublishedAt / lastPublishedCommit** join it there, so a list can show when a wiki was last built without joining every wiki against its generations. A section that holds pages but has no page of its own is reported as a warning, not refused. It renders as a heading a reader cannot open — worse to read, and nowhere near worth discarding an otherwise complete version, which is the same trade already made for diagrams that will not render. The instructions ask for section pages so the warning stays rare. `block_on_mermaid` deliberately sees only the diagram warnings; a policy named for diagrams must not start rejecting versions over a navigation nit. Co-Authored-By: Claude Opus 5 --- .../knowledge/code_wiki/projection.py | 28 ++++++- .../knowledge/code_wiki/projection_plan.py | 17 +++- .../services/knowledge/code_wiki/prompts.py | 7 ++ .../knowledge/code_wiki/publish_gate.py | 38 ++++++++- .../services/knowledge/code_wiki/publisher.py | 66 ++++++++++++++- .../knowledge/code_wiki/test_projection.py | 77 ++++++++++++++++- .../code_wiki/test_projection_plan.py | 26 ++++-- .../knowledge/code_wiki/test_prompts.py | 14 ++++ .../knowledge/code_wiki/test_publish_gate.py | 44 ++++++++++ .../knowledge/code_wiki/test_publisher.py | 83 +++++++++++++++++++ .../code_wiki/test_submit_to_publish.py | 9 +- 11 files changed, 380 insertions(+), 29 deletions(-) diff --git a/backend/app/services/knowledge/code_wiki/projection.py b/backend/app/services/knowledge/code_wiki/projection.py index 97c9114c30..ef2f844009 100644 --- a/backend/app/services/knowledge/code_wiki/projection.py +++ b/backend/app/services/knowledge/code_wiki/projection.py @@ -121,6 +121,27 @@ def resolve(segments: Sequence[str]) -> int: return resolve +# What ``knowledge_documents.name`` will hold. The column is 255 wide while a page +# path may be 500, which is one reason the path cannot simply be the name. +MAX_DOCUMENT_NAME = 255 + + +def _display_name(source: PageSource) -> str: + """What a reader sees this page called. + + The agent's title, which was previously discarded — documents were named after + the path's last segment, so "Backend Architecture" at ``architecture/backend`` + became a document called ``backend``, and two pages at ``architecture/backend`` + and ``services/backend`` were indistinguishable in any flat listing. + + Falls back to the leaf when a title is missing, because a page has to be called + something, and truncates rather than letting the insert fail on a long one. + """ + _, leaf = split_page_path(source.path) + title = (source.title or "").strip() or leaf + return title[:MAX_DOCUMENT_NAME] + + def _stamp(document: KnowledgeDocument, source: PageSource) -> None: """Record the page identity and fingerprint the next plan compares against.""" config = dict(document.source_config or {}) @@ -167,11 +188,11 @@ def apply_projection_plan( def add_document(source: PageSource) -> int: """Create the row for a page that is not in the knowledge base.""" - folders, leaf = split_page_path(source.path) + folders, _ = split_page_path(source.path) document = KnowledgeDocument( kind_id=kind_id, attachment_id=new_attachments[source.path], - name=leaf, + name=_display_name(source), file_extension=DOCUMENT_EXTENSION, file_size=len(source.content.encode("utf-8")), user_id=user_id, @@ -209,6 +230,9 @@ def add_document(source: PageSource) -> int: superseded_attachments.append(document.attachment_id) document.attachment_id = new_attachments[update.source.path] document.file_size = len(update.source.content.encode("utf-8")) + # Follows the title, which is why the fingerprint covers it: a reworded + # heading has to reach the document's name, not just its content. + document.name = _display_name(update.source) _stamp(document, update.source) updated.append(document.id) diff --git a/backend/app/services/knowledge/code_wiki/projection_plan.py b/backend/app/services/knowledge/code_wiki/projection_plan.py index 2399af7b78..5ed87bd0fa 100644 --- a/backend/app/services/knowledge/code_wiki/projection_plan.py +++ b/backend/app/services/knowledge/code_wiki/projection_plan.py @@ -32,9 +32,18 @@ CONTENT_HASH_KEY = "wiki_content_hash" -def content_fingerprint(content: str) -> str: - """Return the fingerprint used to decide whether a page needs rewriting.""" - return hashlib.sha256(content.encode("utf-8")).hexdigest() +def content_fingerprint(title: str, content: str) -> str: + """Return the fingerprint used to decide whether a page needs rewriting. + + Covers the title as well as the body, because the title is now what the document + is named. Fingerprinting the body alone would classify a page whose heading was + reworded as unchanged, and the rename would be silently dropped. + + The cost is that a title-only edit rewrites the attachment, which is wasted work + — but a title almost always appears in the body as its heading, so the two change + together in practice. + """ + return hashlib.sha256(f"{title}\n{content}".encode("utf-8")).hexdigest() @dataclass(frozen=True) @@ -47,7 +56,7 @@ class PageSource: @property def fingerprint(self) -> str: - return content_fingerprint(self.content) + return content_fingerprint(self.title, self.content) @dataclass(frozen=True) diff --git a/backend/app/services/knowledge/code_wiki/prompts.py b/backend/app/services/knowledge/code_wiki/prompts.py index cbc728b246..51b40fce04 100644 --- a/backend/app/services/knowledge/code_wiki/prompts.py +++ b/backend/app/services/knowledge/code_wiki/prompts.py @@ -125,6 +125,13 @@ def _write_contract(deletion_allowed: bool) -> str: keep its place, its links and its search index. Changing it republishes the page as a deletion plus an insertion, so reword titles freely and move paths rarely. - Paths nest at most 4 folders deep, and two paths may not differ only by case. +- **A section that holds pages needs a page of its own.** If you write + `architecture/backend`, write `architecture` too — the navigation is built from + these paths, and a section with no page becomes a heading a reader cannot open. + Give it the overview of what its pages cover. +- The order you list in `--structure-order` when you finish is the order readers see + the pages in. Put the overview first and arrange sections so the wiki reads front + to back; anything you leave out of that list is shown after everything you ranked. - Write each page's **complete content** every time. There is no patch format; what you send replaces the page. - Write `index` as the overview page: a short list of links, each with one line diff --git a/backend/app/services/knowledge/code_wiki/publish_gate.py b/backend/app/services/knowledge/code_wiki/publish_gate.py index 206d575153..a6a4412ae3 100644 --- a/backend/app/services/knowledge/code_wiki/publish_gate.py +++ b/backend/app/services/knowledge/code_wiki/publish_gate.py @@ -102,7 +102,8 @@ def evaluate_publish_gate( The verdict. Warnings never cause a rejection on their own. """ policy = policy or DEFAULT_POLICY - warnings = _collect_warnings(pages) + diagram_warnings = _diagram_warnings(pages) + warnings = _structure_warnings(pages) + diagram_warnings if len(pages) < policy.min_pages: return GateVerdict( @@ -129,7 +130,9 @@ def evaluate_publish_gate( warnings=warnings, ) - if warnings and policy.block_on_mermaid: + # Only the diagram warnings, despite `warnings` carrying more: a policy named + # for diagrams must not start rejecting versions over a missing section page. + if diagram_warnings and policy.block_on_mermaid: return GateVerdict( passed=False, reason="diagram problems were found and the policy blocks on them", @@ -139,8 +142,35 @@ def evaluate_publish_gate( return GateVerdict(passed=True, warnings=warnings) -def _collect_warnings(pages: Sequence[PageSource]) -> tuple[str, ...]: - """Gather non-blocking content warnings across the version.""" +def _sections_without_a_page(pages: Sequence[PageSource]) -> list[str]: + """Sections that hold pages but have no page of their own. + + Reported, never blocking — the same trade as a broken diagram. The navigation is + built from paths, so such a section renders as a group heading that cannot be + opened: a little worse to read, and nowhere near worth discarding a version that + is otherwise complete. Asking for section pages belongs in the instructions, not + in a gate that throws away the run. + """ + present = {collation_key(page.path) for page in pages} + missing = { + page.path.rsplit("/", 1)[0] + for page in pages + if "/" in page.path + and collation_key(page.path.rsplit("/", 1)[0]) not in present + } + return sorted(missing) + + +def _structure_warnings(pages: Sequence[PageSource]) -> tuple[str, ...]: + """Navigation problems worth reporting but not worth refusing a version over.""" + return tuple( + f"{section}: holds pages but has no page of its own" + for section in _sections_without_a_page(pages) + ) + + +def _diagram_warnings(pages: Sequence[PageSource]) -> tuple[str, ...]: + """Diagrams that will not render, which the agent can be asked to fix.""" collected: list[str] = [] for page in pages: mermaid_warnings = check_mermaid_blocks(page.content) diff --git a/backend/app/services/knowledge/code_wiki/publisher.py b/backend/app/services/knowledge/code_wiki/publisher.py index 98fcf48533..1641215c03 100644 --- a/backend/app/services/knowledge/code_wiki/publisher.py +++ b/backend/app/services/knowledge/code_wiki/publisher.py @@ -37,6 +37,7 @@ from app.models.kind import Kind from app.models.knowledge import KnowledgeDocument from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.services.knowledge.code_wiki.page_path import collation_key from app.services.knowledge.code_wiki.projection import ( PENDING_INDEX_CLEANUP_KEY, ProjectionSideEffects, @@ -64,6 +65,18 @@ PUBLISHED_GENERATION_KEY = "publishedGenerationId" +# Written alongside the pointer, in the same transaction, because they describe what +# that transaction just did. A list needs them and would otherwise join every wiki +# against its generations to get two fields. +PUBLISHED_AT_KEY = "lastPublishedAt" +PUBLISHED_COMMIT_KEY = "lastPublishedCommit" + +# The order pages are shown in, as a list of paths. Kept here rather than on each +# document because a reorder must not touch a page whose content did not change: +# the fingerprint would still match, so the projection would skip it and the new +# position would never be written. One array, one write, no document churn. +PAGE_ORDER_KEY = "pageOrder" + @dataclass(frozen=True) class PublishResult: @@ -82,7 +95,9 @@ def read_version_pages(db: Session, generation_id: int) -> tuple[PageSource, ... guessed at: projecting one would create a document the next run could not match, and so would delete and recreate on every publish. """ - pages: list[PageSource] = [] + declared = _declared_order(db, generation_id) + + ranked: list[tuple[int, int, PageSource]] = [] for entry in ( db.query(WikiContent).filter(WikiContent.generation_id == generation_id).all() ): @@ -93,8 +108,39 @@ def read_version_pages(db: Session, generation_id: int) -> tuple[PageSource, ... entry.id, ) continue - pages.append(PageSource(path=path, title=entry.title, content=entry.content)) - return tuple(pages) + # Pages the agent did not rank sort after the ones it did, in the order they + # were written. Alphabetical would be worse than arbitrary here: it puts "api" + # ahead of the overview, and a wiki read in that order reads wrong. + rank = declared.get(collation_key(path), len(declared)) + ranked.append( + ( + rank, + entry.id, + PageSource(path=path, title=entry.title, content=entry.content), + ) + ) + + ranked.sort(key=lambda item: (item[0], item[1])) + return tuple(page for _, _, page in ranked) + + +def _declared_order(db: Session, generation_id: int) -> dict[str, int]: + """The page order the agent declared when it finished, keyed for matching. + + Sent as ``summary.structure_order`` and until now recorded and never read. It is + the only statement of order there is: a path carries hierarchy but says nothing + about which section comes first. + """ + generation = db.get(WikiGeneration, generation_id) + if generation is None: + return {} + summary = (generation.ext or {}).get("content_write", {}).get("summary", {}) or {} + declared = summary.get("structure_order") or [] + return { + collation_key(str(path)): index + for index, path in enumerate(declared) + if str(path).strip() + } def read_projected_pages(db: Session, kind_id: int) -> tuple[ProjectedPage, ...]: @@ -238,7 +284,19 @@ def publish_generation( ) _record_verdict(generation, verdict) - _update_spec(knowledge_base, **{PUBLISHED_GENERATION_KEY: generation.id}) + _update_spec( + knowledge_base, + **{ + PUBLISHED_GENERATION_KEY: generation.id, + PUBLISHED_AT_KEY: datetime.now(timezone.utc) + .replace(tzinfo=None) + .isoformat(), + PUBLISHED_COMMIT_KEY: str( + (generation.source_snapshot or {}).get("commit", "") or "" + ), + PAGE_ORDER_KEY: [page.path for page in desired], + }, + ) db.commit() # Past this point the version is live. Nothing below may raise: the pages are diff --git a/backend/tests/services/knowledge/code_wiki/test_projection.py b/backend/tests/services/knowledge/code_wiki/test_projection.py index af608ee65d..ebc621cca6 100644 --- a/backend/tests/services/knowledge/code_wiki/test_projection.py +++ b/backend/tests/services/knowledge/code_wiki/test_projection.py @@ -94,10 +94,13 @@ def _existing_document( db.flush() parent_id = folder.id + # Mirrors _source: the title is the leaf, so an existing row and a fresh source + # for the same path fingerprint identically and count as unchanged. + title = path.rsplit("/", 1)[-1] document = KnowledgeDocument( kind_id=KIND_ID, attachment_id=attachment_id, - name=path.rsplit("/", 1)[-1], + name=title, file_extension="md", file_size=len(content), user_id=USER_ID, @@ -105,7 +108,7 @@ def _existing_document( origin=ContentOrigin.GENERATED.value, source_config={ PAGE_PATH_KEY: path, - CONTENT_HASH_KEY: content_fingerprint(content), + CONTENT_HASH_KEY: content_fingerprint(title, content), }, ) db.add(document) @@ -273,7 +276,9 @@ def test_the_fingerprint_is_stamped_so_the_next_run_can_skip( outcome = _apply(test_db, plan, effects) document = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]) - assert document.source_config[CONTENT_HASH_KEY] == content_fingerprint("v1") + assert document.source_config[CONTENT_HASH_KEY] == content_fingerprint( + "index", "v1" + ) assert document.source_config[PAGE_PATH_KEY] == "index" @@ -427,3 +432,69 @@ def test_user_content_is_never_touched(test_db: Session, effects: RecordingEffec _apply(test_db, compute_projection_plan([], []), effects) assert test_db.get(KnowledgeDocument, user_document.id) is not None + + +# --- what a page is called -------------------------------------------------- + + +def test_a_page_is_named_after_its_title_not_its_path( + test_db: Session, effects: RecordingEffects +): + """The title used to be discarded, so "Backend Architecture" at + architecture/backend became a document called "backend" — and two pages at + architecture/backend and services/backend were indistinguishable anywhere the + path was not also shown.""" + source = PageSource( + path="architecture/backend", title="Backend Architecture", content="body" + ) + plan = compute_projection_plan([source], []) + + outcome = _apply(test_db, plan, effects) + + document = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]) + assert document.name == "Backend Architecture" + # The path stays the identity, untouched by what the page is called. + assert document.source_config[PAGE_PATH_KEY] == "architecture/backend" + + +def test_a_page_without_a_title_falls_back_to_its_path( + test_db: Session, effects: RecordingEffects +): + """A page has to be called something, and refusing the publish over a missing + heading would discard a version that is otherwise fine.""" + source = PageSource(path="architecture/backend", title=" ", content="body") + plan = compute_projection_plan([source], []) + + outcome = _apply(test_db, plan, effects) + + assert test_db.get(KnowledgeDocument, outcome.created_document_ids[0]).name == ( + "backend" + ) + + +def test_an_overlong_title_is_truncated_rather_than_failing_the_insert( + test_db: Session, effects: RecordingEffects +): + """The column is 255 wide; a long heading must not take the whole publish down.""" + source = PageSource(path="index", title="x" * 400, content="body") + plan = compute_projection_plan([source], []) + + outcome = _apply(test_db, plan, effects) + + name = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]).name + assert len(name) == 255 + + +def test_rewording_a_title_renames_the_document_in_place( + test_db: Session, effects: RecordingEffects +): + """It keeps its id, so the RAG index entry and any stored citation survive.""" + existing = _existing_document(test_db, "index", "body") + original_id = existing.id + renamed = PageSource(path="index", title="A Better Heading", content="body") + plan = compute_projection_plan([renamed], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + + assert outcome.updated_document_ids == (original_id,) + assert test_db.get(KnowledgeDocument, original_id).name == "A Better Heading" diff --git a/backend/tests/services/knowledge/code_wiki/test_projection_plan.py b/backend/tests/services/knowledge/code_wiki/test_projection_plan.py index fcd14a4086..b576f880d2 100644 --- a/backend/tests/services/knowledge/code_wiki/test_projection_plan.py +++ b/backend/tests/services/knowledge/code_wiki/test_projection_plan.py @@ -20,11 +20,13 @@ def _source(path: str, content: str = "body") -> PageSource: return PageSource(path=path, title=path.rsplit("/", 1)[-1], content=content) -def _projected(document_id: int, path: str, content: str = "body") -> ProjectedPage: +def _projected( + document_id: int, path: str, content: str = "body", title: str | None = None +) -> ProjectedPage: return ProjectedPage( document_id=document_id, path=path, - content_hash=content_fingerprint(content), + content_hash=content_fingerprint(title or path.rsplit("/", 1)[-1], content), ) @@ -77,23 +79,29 @@ def test_a_moved_page_is_an_add_and_a_delete(): def test_matching_ignores_case_because_the_database_does(): plan = compute_projection_plan( [_source("Architecture/Backend", "same")], - [_projected(1, "architecture/backend", "same")], + # Same title on both sides: this is about the path matching, and letting the + # helper derive two differently-cased titles would test something else. + [_projected(1, "architecture/backend", "same", title="Backend")], ) assert plan.skips == ("Architecture/Backend",) assert plan.deletes == () -def test_a_rewritten_title_alone_does_not_touch_the_page(): - """Titles live in the version; the projection compares content, not headings.""" - existing = _projected(1, "index", "unchanged body") +def test_a_rewritten_title_alone_still_updates_the_page(): + """The title is what the document is named, so a reworded heading has to reach + the knowledge base. It used to be discarded, and comparing content alone was + right then; now that trade-off is inverted — the cost is rewriting an attachment + whose bytes did not change, and a title almost always moves with its body.""" + existing = _projected(1, "index", "unchanged body", title="Index") renamed = PageSource( path="index", title="A Better Heading", content="unchanged body" ) plan = compute_projection_plan([renamed], [existing]) - assert plan.skips == ("index",) + assert plan.skips == () + assert [update.source.title for update in plan.updates] == ["A Better Heading"] def test_an_empty_version_removes_everything_it_owns(): @@ -135,5 +143,5 @@ def test_an_identical_snapshot_produces_no_work(): def test_the_fingerprint_distinguishes_content(): - assert content_fingerprint("a") != content_fingerprint("b") - assert content_fingerprint("a") == content_fingerprint("a") + assert content_fingerprint("T", "a") != content_fingerprint("T", "b") + assert content_fingerprint("T", "a") == content_fingerprint("T", "a") diff --git a/backend/tests/services/knowledge/code_wiki/test_prompts.py b/backend/tests/services/knowledge/code_wiki/test_prompts.py index 622dd2a55d..7fc979d06f 100644 --- a/backend/tests/services/knowledge/code_wiki/test_prompts.py +++ b/backend/tests/services/knowledge/code_wiki/test_prompts.py @@ -167,3 +167,17 @@ def test_diagram_problems_come_back_as_a_correction(): def test_no_diagram_problems_means_no_follow_up(): assert build_diagram_correction([]) is None + + +def test_both_modes_ask_for_a_page_at_every_section(): + """A section with no page becomes a heading a reader cannot open. Asking for it + here is what keeps the publish gate's warning rare.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "section that holds pages needs a page of its own" in prompt + + +def test_both_modes_explain_that_the_declared_order_is_what_readers_see(): + """Otherwise the field looks like bookkeeping and gets filled in arbitrarily.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "--structure-order" in prompt + assert "order readers see" in prompt diff --git a/backend/tests/services/knowledge/code_wiki/test_publish_gate.py b/backend/tests/services/knowledge/code_wiki/test_publish_gate.py index c273106c0c..48d3120f87 100644 --- a/backend/tests/services/knowledge/code_wiki/test_publish_gate.py +++ b/backend/tests/services/knowledge/code_wiki/test_publish_gate.py @@ -175,3 +175,47 @@ def test_a_path_kept_under_a_different_case_is_not_a_removal(): verdict = evaluate_publish_gate(recased, published_paths=_published(10)) assert verdict.passed + + +# --- sections that hold pages but are not pages ------------------------------ + + +def test_a_section_with_no_page_of_its_own_is_reported_not_refused(): + """The navigation is built from paths, so it renders as a group heading that + cannot be opened — worse to read, nowhere near worth discarding a version.""" + pages = [ + PageSource(path="index", title="Index", content="body"), + PageSource(path="architecture/backend", title="Backend", content="body"), + ] + + verdict = evaluate_publish_gate(pages, published_paths=[]) + + assert verdict.passed + assert any("architecture" in warning for warning in verdict.warnings) + + +def test_a_section_that_is_itself_a_page_draws_no_warning(): + pages = [ + PageSource(path="architecture", title="Architecture", content="body"), + PageSource(path="architecture/backend", title="Backend", content="body"), + ] + + verdict = evaluate_publish_gate(pages, published_paths=[]) + + assert verdict.warnings == () + + +def test_the_diagram_policy_does_not_reject_over_a_missing_section_page(): + """`block_on_mermaid` is named for diagrams; letting it fire on a navigation nit + would make its name a lie and throw away versions nobody meant to refuse.""" + pages = [ + PageSource(path="index", title="Index", content="body"), + PageSource(path="architecture/backend", title="Backend", content="body"), + ] + + verdict = evaluate_publish_gate( + pages, published_paths=[], policy=PublishPolicy(block_on_mermaid=True) + ) + + assert verdict.passed + assert verdict.warnings diff --git a/backend/tests/services/knowledge/code_wiki/test_publisher.py b/backend/tests/services/knowledge/code_wiki/test_publisher.py index e59287803a..6f2ec7e99e 100644 --- a/backend/tests/services/knowledge/code_wiki/test_publisher.py +++ b/backend/tests/services/knowledge/code_wiki/test_publisher.py @@ -31,6 +31,9 @@ PublishPolicy, ) from app.services.knowledge.code_wiki.publisher import ( + PAGE_ORDER_KEY, + PUBLISHED_AT_KEY, + PUBLISHED_COMMIT_KEY, PUBLISHED_GENERATION_KEY, publish_generation, published_generation_id, @@ -470,3 +473,83 @@ def _update_spec_pending(test_db: Session, knowledge_base: Kind, refs: list[str] payload["spec"] = spec knowledge_base.json = payload test_db.flush() + + +# --- what a publish records for readers ------------------------------------- + + +def test_publishing_records_the_commit_and_time_for_the_list( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """A list would otherwise join every wiki against its generations for two + fields. Written in the publish transaction, so they cannot drift from the + pointer they sit beside.""" + generation = _generation(test_db, knowledge_base.id) + generation.source_snapshot = {"commit": "abc1234"} + _page(test_db, generation, "index", "body") + test_db.flush() + + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + spec = (knowledge_base.json or {})["spec"] + assert spec[PUBLISHED_COMMIT_KEY] == "abc1234" + assert spec[PUBLISHED_AT_KEY] + + +def test_the_agents_declared_order_is_what_gets_recorded( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """Paths carry hierarchy and say nothing about which section comes first. + Alphabetically "api" precedes the overview, and a wiki read that way reads + wrong.""" + generation = _generation(test_db, knowledge_base.id) + for path in ("api", "index", "architecture"): + _page(test_db, generation, path, "body") + generation.ext = { + "content_write": { + "summary": {"structure_order": ["index", "architecture", "api"]} + } + } + test_db.flush() + + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert (knowledge_base.json or {})["spec"][PAGE_ORDER_KEY] == [ + "index", + "architecture", + "api", + ] + + +def test_pages_the_agent_did_not_rank_follow_the_ones_it_did( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """A page added without updating the declared order must still appear, and + appear somewhere predictable rather than at a position nobody chose.""" + generation = _generation(test_db, knowledge_base.id) + for path in ("index", "stray"): + _page(test_db, generation, path, "body") + generation.ext = {"content_write": {"summary": {"structure_order": ["index"]}}} + test_db.flush() + + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert (knowledge_base.json or {})["spec"][PAGE_ORDER_KEY] == ["index", "stray"] diff --git a/backend/tests/services/knowledge/code_wiki/test_submit_to_publish.py b/backend/tests/services/knowledge/code_wiki/test_submit_to_publish.py index 597f103c33..31cd325859 100644 --- a/backend/tests/services/knowledge/code_wiki/test_submit_to_publish.py +++ b/backend/tests/services/knowledge/code_wiki/test_submit_to_publish.py @@ -247,9 +247,12 @@ def test_pages_sent_with_the_final_summary_are_published_too( ), ) - # Named for the path's leaf, not the title: the path is the identity, and a - # reworded heading must not move the document. - assert [doc.name for doc in _documents(test_db, knowledge_base.id)] == ["backend"] + # Named for the title, which is what a reader sees. The path stays the identity + # in source_config, so rewording the heading renames the document without moving + # it or changing the id the RAG index is keyed on. + assert [doc.name for doc in _documents(test_db, knowledge_base.id)] == [ + "Backend Architecture" + ] def test_a_legacy_generation_only_records_its_status( From edf8b67ad3b93ab4c603b2d4a16e1d013fced300 Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Tue, 4 Aug 2026 10:50:26 +0800 Subject: [PATCH 05/14] feat(knowledge): a code wiki belongs to its repository, not to whoever asked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ownership moves to the configured wiki account. That account's Git credentials are what clone the repository, so it is the identity the wiki actually depends on; attributing it to the requester made a repository's documentation disappear when one person left, and let two colleagues each own a private copy of the same thing. The consequence is deliberate and load-bearing. Knowledge-base ACLs now grant nobody else anything, so **who may read a code wiki is decided by who may read its repository** — the rule that needs no second copy of the repository's membership kept in sync. That check runs wegent → git, using the signed-in user's own credentials, and this is why it works where syncing members would not: syncing runs git → wegent and needs every member mapped back to a wegent account through credentials stored in a JSON column, silently omitting everyone who has never configured a token. Here there is no mapping — the identity is theirs by construction. It usually costs nothing, because the provider layer already keeps each user's accessible repositories in Redis; only a cold cache asks about the one repository, and an unreachable provider is refused rather than waved through. One repository has one wiki, registered on wiki_projects, whose source_url is already UNIQUE. That constraint is what settles two simultaneous requests; a check against a JSON field on the knowledge base would leave a window exactly where it matters. The loser of that race gets the winner's wiki, and asking for a repository that already has one answers 200 rather than 201. Code wikis are kept out of general knowledge base listings by a scope, wired into all four paths that produce them — including the MCP tool, where an agent shown one may write into it and have the next publish delete what it added. Leaving that to ownership would be incidental rather than stated: it holds only while the wiki account is somebody else, and stops holding for that account or for an administrator. The scope spells out NULL, because a knowledge base predating kbType compares NULL against the literal and would vanish from every listing at once. Also: - GET /knowledge-bases/code-wikis lists them, reading the repository, published time and commit straight from spec. - The model-binding check can be asked about the code wiki team; it reported on the legacy one, which says a model is bound when the run about to start has none. - Creation now verifies the wiki account can read the repository, not just the requester. That mismatch used to surface as a failed generation, with nothing in the failure saying the account simply needed adding to the repository. Co-Authored-By: Claude Opus 5 --- ...a_link_wiki_projects_to_their_code_wiki.py | 48 ++++ backend/app/api/endpoints/knowledge.py | 161 ++++++++++++- backend/app/api/endpoints/wiki.py | 13 +- backend/app/models/wiki.py | 13 ++ backend/app/schemas/knowledge.py | 39 +++- .../knowledge/code_wiki/read_access.py | 131 +++++++++++ .../services/knowledge/code_wiki/registry.py | 124 ++++++++++ .../app/services/knowledge/content_scope.py | 33 +++ .../services/knowledge/knowledge_service.py | 66 +++--- backend/tests/api/test_knowledge_code_wiki.py | 212 ++++++++++++++++++ .../knowledge/code_wiki/test_read_access.py | 126 +++++++++++ .../services/knowledge/test_content_scope.py | 54 +++++ 12 files changed, 969 insertions(+), 51 deletions(-) create mode 100644 backend/alembic/versions/20260804_2b5791acc5fa_link_wiki_projects_to_their_code_wiki.py create mode 100644 backend/app/services/knowledge/code_wiki/read_access.py create mode 100644 backend/app/services/knowledge/code_wiki/registry.py create mode 100644 backend/tests/services/knowledge/code_wiki/test_read_access.py diff --git a/backend/alembic/versions/20260804_2b5791acc5fa_link_wiki_projects_to_their_code_wiki.py b/backend/alembic/versions/20260804_2b5791acc5fa_link_wiki_projects_to_their_code_wiki.py new file mode 100644 index 0000000000..c027999416 --- /dev/null +++ b/backend/alembic/versions/20260804_2b5791acc5fa_link_wiki_projects_to_their_code_wiki.py @@ -0,0 +1,48 @@ +"""link wiki projects to their code wiki + +Revision ID: 2b5791acc5fa +Revises: bd9c871a93d2 + +``wiki_projects.source_url`` is already UNIQUE, which makes this table the only place +that can enforce "one repository, one code wiki" against two people creating at the +same moment. Recording the knowledge base here, rather than checking a JSON field on +the knowledge base itself, turns a check-then-insert into a database constraint. + +``0`` marks a project row with no code wiki, which is every row that exists today. +""" + +from collections.abc import Sequence +from typing import Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "2b5791acc5fa" +down_revision: Union[str, Sequence[str], None] = "bd9c871a93d2" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.add_column( + "wiki_projects", + sa.Column( + "kind_id", + sa.Integer(), + nullable=False, + server_default="0", + comment="Code wiki knowledge base built from this repository; 0 = none", + ), + ) + op.create_index( + "ix_wiki_projects_kind_id", "wiki_projects", ["kind_id"], unique=False + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index("ix_wiki_projects_kind_id", table_name="wiki_projects") + op.drop_column("wiki_projects", "kind_id") diff --git a/backend/app/api/endpoints/knowledge.py b/backend/app/api/endpoints/knowledge.py index 70bc9538db..6cb9630f81 100644 --- a/backend/app/api/endpoints/knowledge.py +++ b/backend/app/api/endpoints/knowledge.py @@ -13,7 +13,15 @@ from datetime import datetime from typing import Dict, List, Optional -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status +from fastapi import ( + APIRouter, + BackgroundTasks, + Depends, + HTTPException, + Query, + Response, + status, +) from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -29,6 +37,7 @@ from app.core.config import settings from app.core.exceptions import CustomHTTPException from app.db.session import SessionLocal +from app.models.kind import Kind from app.models.user import User from app.schemas.knowledge import ( AccessibleKnowledgeResponse, @@ -36,6 +45,8 @@ BatchDocumentIds, BatchOperationResult, CodeWikiCreate, + CodeWikiListItem, + CodeWikiListResponse, CodeWikiRunCreate, CodeWikiRunResponse, DocumentContentUpdate, @@ -69,6 +80,19 @@ knowledge_base_qa_service, ) from app.services.knowledge.code_wiki.generation import GenerationInFlight +from app.services.knowledge.code_wiki.publisher import ( + PUBLISHED_AT_KEY, + PUBLISHED_COMMIT_KEY, +) +from app.services.knowledge.code_wiki.read_access import readable_wiki_ids +from app.services.knowledge.code_wiki.registry import ( + CODE_WIKI_NAMESPACE, + CodeWikiOwnerMissing, + claim_repository, + existing_wiki_id, + project_for, + wiki_owner, +) from app.services.knowledge.code_wiki.run_mode import ChangedPath from app.services.knowledge.code_wiki.runner import CodeWikiRunError, start_run from app.services.knowledge.code_wiki.source import ( @@ -76,6 +100,7 @@ SourceRepository, assert_user_can_read_source, ) +from app.services.knowledge.content_scope import only_code_wikis from app.services.knowledge.orchestrator import ( DEFAULT_KNOWLEDGE_LIST_LIMIT, MAX_DOCUMENT_READ_LIMIT, @@ -492,6 +517,93 @@ def create_knowledge_base( ) +def _assert_the_wiki_account_can_clone( + db: Session, owner: User, requester: User, source: SourceRepository +) -> None: + """Check the account that will actually clone, not just the one asking. + + The requester's access was verified above, but the repository is cloned with the + wiki account's credentials. Without this the mismatch surfaces much later as a + failed generation, and nothing in that failure says the account simply needs + adding to the repository. + """ + if owner.id == requester.id: + return + try: + assert_user_can_read_source(db, owner.id, source) + except SourceAccessDenied as e: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=( + f"The wiki account cannot read '{source.project_name}', so the " + f"repository could be checked out by you but not by the account " + f"that generates the wiki. Give '{owner.user_name}' read access to " + f"the repository, or set WIKI_DEFAULT_USER_ID=0 to generate with " + f"the requester's own credentials. ({e})" + ), + ) from e + + +def _code_wiki_response(db: Session, knowledge_base_id: int) -> KnowledgeBaseResponse: + """Render a code wiki without a knowledge-base ACL check. + + The wiki belongs to the wiki account, so the requester never passes that check — + including for the wiki they just asked to have built. Authorisation for this + endpoint is the repository gate above, which they have already passed. + """ + kind = KnowledgeService._get_knowledge_base_record(db, knowledge_base_id) + if kind is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Code wiki not found" + ) + return KnowledgeBaseResponse.from_kind(kind) + + +@router.get("/code-wikis", response_model=CodeWikiListResponse) +@trace_sync("list_code_wikis", "knowledge.api") +def list_code_wikis( + current_user: User = Depends(security.get_current_user), + db: Session = Depends(get_db), +): + """Code wikis the caller may read. + + Separate from the general knowledge base list rather than a filter on it: a code + wiki belongs to the wiki account, so it matches none of that list's scopes, and + who may read one is decided by the repository instead. + """ + wikis = ( + only_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, # noqa: E712 + ) + ) + .order_by(Kind.updated_at.desc()) + .all() + ) + readable = readable_wiki_ids(db, current_user, wikis) + + items = [_code_wiki_list_item(db, kind) for kind in wikis if kind.id in readable] + return CodeWikiListResponse(items=items, total=len(items)) + + +def _code_wiki_list_item(db: Session, kind: Kind) -> CodeWikiListItem: + spec = (kind.json or {}).get("spec", {}) + source = spec.get("source") or {} + return CodeWikiListItem( + id=kind.id, + name=spec.get("name", kind.name), + description=spec.get("description"), + project_name=str(source.get("projectName", "") or ""), + source_url=str(source.get("sourceUrl", "") or ""), + last_published_at=spec.get(PUBLISHED_AT_KEY), + last_published_commit=str(spec.get(PUBLISHED_COMMIT_KEY, "") or ""), + document_count=KnowledgeService.get_document_count(db, kind.id), + created_at=kind.created_at, + updated_at=kind.updated_at, + ) + + @router.post( "/code-wikis", response_model=KnowledgeBaseResponse, @@ -500,16 +612,20 @@ def create_knowledge_base( @trace_sync("create_code_wiki", "knowledge.api") def create_code_wiki( data: CodeWikiCreate, + response: Response, current_user: User = Depends(security.get_current_user), db: Session = Depends(get_db), ): - """Create a code wiki bound to a source repository. + """Create a code wiki bound to a source repository, or return the existing one. The requester must be able to read the repository, so that a wiki cannot be built - for a private repository they have no access to. Reading the resulting wiki is - then governed by knowledge-base permissions alone: place it in an organization - namespace to make it readable by everyone signed in, or keep it in a restricted - namespace and share it explicitly. + for one they have no access to. Who may read the result is decided the same way: + the wiki belongs to the configured wiki account rather than to the requester, so + knowledge-base ACLs grant nobody else access and the repository is what does. + + One repository has one wiki. Asking for a repository that already has one returns + it — the caller wanted that repository's wiki, not the act of creating it — and + the response is 200 rather than 201 to say which happened. """ try: source = SourceRepository.from_url(data.source_type, data.source_url) @@ -517,33 +633,56 @@ def create_code_wiki( except SourceAccessDenied as e: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) from e + existing_id = existing_wiki_id(db, source) + if existing_id: + response.status_code = status.HTTP_200_OK + return _code_wiki_response(db, existing_id) + try: + owner = wiki_owner(db, current_user) + _assert_the_wiki_account_can_clone(db, owner, current_user, source) + project = project_for(db, source) result = knowledge_orchestrator.create_knowledge_base( db=db, - user=current_user, + user=owner, name=data.name, description=data.description, - namespace=data.namespace or "default", + namespace=CODE_WIKI_NAMESPACE, kb_type=KnowledgeBaseType.CODE_WIKI.value, source=source, ) + claim_repository(db, project, result.id) + db.commit() add_span_event( "knowledge.code_wiki.created", { "kb_id": str(result.id), "project_name": source.project_name, - "namespace": data.namespace or "default", - "user_id": str(current_user.id), + "requested_by": str(current_user.id), + "owner_id": str(owner.id), }, ) return result + except CodeWikiOwnerMissing as e: + db.rollback() + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) + ) from e except IntegrityError as e: + # Two requests for one repository, both past the check above. The UNIQUE on + # wiki_projects.source_url is what settles it; the loser returns the winner's + # wiki rather than an error, since that is what it asked for. db.rollback() + settled = existing_wiki_id(db, source) + if settled: + response.status_code = status.HTTP_200_OK + return _code_wiki_response(db, settled) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Knowledge base with name '{data.name}' already exists in this namespace", + detail=f"Knowledge base with name '{data.name}' already exists", ) from e except ValueError as e: + db.rollback() raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) ) from e diff --git a/backend/app/api/endpoints/wiki.py b/backend/app/api/endpoints/wiki.py index 68baee5843..e9a4162f31 100644 --- a/backend/app/api/endpoints/wiki.py +++ b/backend/app/api/endpoints/wiki.py @@ -432,13 +432,24 @@ def get_wiki_stats_summary( # ========== Config Endpoints ========== @router.get("/config") def get_wiki_config( + code_wiki: bool = Query( + default=False, + description="Report on the code wiki team rather than the legacy wiki team", + ), current_user: User = Depends(security.get_current_user), main_db: Session = Depends(get_db), ): """Get wiki configuration including default team info and bound model""" from app.services.adapters.team_kinds import team_kinds_service - default_team_name = wiki_settings.DEFAULT_TEAM_NAME + # Which team to report on. The legacy wiki and a code wiki run different teams, + # so answering for the wrong one tells the caller a model is bound when the run + # it is about to start has none. + default_team_name = ( + wiki_settings.CODE_WIKI_TEAM_NAME + if code_wiki + else wiki_settings.DEFAULT_TEAM_NAME + ) default_user_id = wiki_settings.DEFAULT_USER_ID default_team = None has_bound_model = False diff --git a/backend/app/models/wiki.py b/backend/app/models/wiki.py index c3ef86f1db..3fe2a5a703 100644 --- a/backend/app/models/wiki.py +++ b/backend/app/models/wiki.py @@ -38,6 +38,19 @@ class WikiProject(WikiBase): source_domain = Column(String(100), nullable=True) description = Column(Text) ext = Column(JSON, comment="Project extension data") + # The code wiki built from this repository, or 0 for a legacy project that has + # none. Recorded here rather than on the knowledge base because `source_url` + # above is UNIQUE: that constraint is what makes "one repository, one wiki" hold + # under two people creating at the same moment, which a check-then-insert on a + # JSON field could not. + kind_id = Column( + Integer, + nullable=False, + default=0, + server_default="0", + index=True, + comment="Code wiki knowledge base built from this repository; 0 = none", + ) is_active = Column(Boolean, nullable=False, default=True) created_at = Column(DateTime, default=func.now()) updated_at = Column(DateTime, default=func.now(), onupdate=func.now()) diff --git a/backend/app/schemas/knowledge.py b/backend/app/schemas/knowledge.py index 67ad5fc820..f8212938a1 100644 --- a/backend/app/schemas/knowledge.py +++ b/backend/app/schemas/knowledge.py @@ -307,14 +307,6 @@ class CodeWikiCreate(BaseModel): name: str = Field(..., min_length=1, max_length=100) description: Optional[str] = Field(None, max_length=500) - namespace: str = Field( - default="default", - max_length=255, - description=( - "Namespace to create the wiki in. Use an organization namespace to make " - "it readable by everyone signed in." - ), - ) source_type: Literal["github", "gitlab", "gitea"] = Field( ..., description=( @@ -343,6 +335,37 @@ class CodeWikiChangedPath(BaseModel): ) +class CodeWikiListItem(BaseModel): + """One code wiki, as a list shows it. + + Everything here is read from the knowledge base's spec, written when the version + was published. A list that had to join every wiki against its generations for + these three fields would pay for them on every page load. + """ + + id: int + name: str + description: Optional[str] = None + project_name: str = Field("", description="Repository the wiki documents") + source_url: str = Field("", description="Repository URL") + last_published_at: Optional[str] = Field( + None, description="When the live version was published; null if never" + ) + last_published_commit: str = Field( + "", description="Commit the live version documents" + ) + document_count: int = 0 + created_at: datetime + updated_at: datetime + + +class CodeWikiListResponse(BaseModel): + """Code wikis the caller may read.""" + + items: List[CodeWikiListItem] + total: int + + class CodeWikiRunCreate(BaseModel): """Request to regenerate a code wiki now. diff --git a/backend/app/services/knowledge/code_wiki/read_access.py b/backend/app/services/knowledge/code_wiki/read_access.py new file mode 100644 index 0000000000..e11a3a59e0 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/read_access.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Who may read a code wiki. + +A code wiki belongs to the wiki account, so knowledge-base ACLs grant nobody else +anything. What decides access instead is the repository: **if you can read the code, +you can read its documentation**, which is the only rule that does not need a second +place to keep in sync with the repository's own membership. + +The direction matters, and it is why this works where syncing members would not. +Syncing runs git → wegent and needs every repository member mapped back to a wegent +account: a many-to-many lookup against credentials stored in a JSON column, which +silently omits everyone who has never configured a token. Asking at read time runs +wegent → git and uses the signed-in user's own credentials, so there is no mapping at +all — the identity is theirs by construction, and someone with no token is correctly +unable to demonstrate anything. + +The answer usually costs nothing: the provider layer already keeps each user's +accessible repositories in Redis for two hours, built once behind a lock because it +is expensive. This reads that. Only a cold cache falls through to asking about the +one repository, which is the same call the creation gate makes. + +Two hours is also how long someone removed from a repository keeps access. +""" + +import logging +from typing import Any + +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.user import User +from app.services.knowledge.code_wiki.source import SourceRepository, provider_for +from app.services.knowledge.knowledge_service import _run_async_in_new_loop + +logger = logging.getLogger(__name__) + + +def may_read_code_wiki(db: Session, user: User, knowledge_base: Kind) -> bool: + """Whether ``user`` may read this code wiki, judged by its repository.""" + spec = (knowledge_base.json or {}).get("spec", {}) + source = SourceRepository.from_spec(spec.get("source")) + if source is None or not source.project_name: + # No repository to judge by. Refusing is the safe direction: a wiki nobody + # can attribute to a repository is one nobody should inherit access to. + logger.warning( + "[code_wiki] kb %s has no source repository; read refused", + knowledge_base.id, + ) + return False + + if _repository_is_in_the_users_cache(user, source): + return True + + return _repository_is_readable_now(db, user, source) + + +def _repository_is_in_the_users_cache(user: User, source: SourceRepository) -> bool: + """Look for the repository in the list the provider layer already caches.""" + provider = provider_for(source.source_type) + reader = getattr(provider, "_get_all_repositories_from_cache", None) + if reader is None: + return False + + try: + cached = _run_async_in_new_loop(reader(user, source.source_domain)) + except Exception as exc: + logger.debug( + "[code_wiki] could not read the repository cache for user %s: %s", + user.id, + exc, + ) + return False + + if not cached: + return False + + wanted = source.project_name.casefold() + return any(_full_name(entry).casefold() == wanted for entry in cached) + + +def _full_name(entry: Any) -> str: + if isinstance(entry, dict): + return str(entry.get("full_name") or entry.get("name") or "") + return "" + + +def _repository_is_readable_now( + db: Session, user: User, source: SourceRepository +) -> bool: + """Ask the provider about this one repository. + + Reached when the cache is cold, which is a user who has not opened the repository + picker yet. Building the whole list here would make a first read pay for every + repository they can see; asking about one is the same call the creation gate + makes and is bounded. + """ + from app.services.knowledge.code_wiki.source import ( + SourceAccessDenied, + assert_user_can_read_source, + ) + + try: + assert_user_can_read_source(db, user.id, source) + return True + except SourceAccessDenied: + return False + except Exception as exc: + # An unreachable provider must not become an open door. + logger.warning( + "[code_wiki] could not check %s for user %s: %s", + source.project_name, + user.id, + exc, + ) + return False + + +def readable_wiki_ids(db: Session, user: User, knowledge_bases: list[Kind]) -> set[int]: + """Which of these code wikis ``user`` may read. + + Used by the list, where the cache lookup is one Redis read shared across every + wiki rather than one per wiki. + """ + readable: set[int] = set() + for knowledge_base in knowledge_bases: + if may_read_code_wiki(db, user, knowledge_base): + readable.add(knowledge_base.id) + return readable diff --git a/backend/app/services/knowledge/code_wiki/registry.py b/backend/app/services/knowledge/code_wiki/registry.py new file mode 100644 index 0000000000..1b30f84320 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/registry.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Which repository a code wiki documents, and who owns the result. + +**One repository, one wiki.** Two people asking for the same repository should get +the same wiki, not two of them — two full generations, two indexes, two sets of +citations, for one repository's worth of documentation. The claim is registered on +``wiki_projects``, whose ``source_url`` is already UNIQUE, so the database refuses the +second one even when both requests are in flight at once. A check against a JSON field +on the knowledge base could not: it would leave a window between the read and the +insert exactly wide enough for the case worth preventing. + +**A code wiki belongs to the configured wiki account, not to whoever asked for it.** +The account is the one whose Git credentials clone the repository, so it is the +identity the wiki actually depends on. Attributing it to the requester instead would +make a repository's documentation disappear when one person leaves, and would let two +colleagues each own a private copy of the same thing. + +The consequence is deliberate and load-bearing: knowledge-base ACLs no longer grant +anyone but that account access, so **who may read a code wiki is decided by who may +read its repository**. That check is not an extra convenience here — it is the only +one, which is why it has a fallback rather than being allowed to fail closed on a +cold cache. +""" + +import logging +from typing import Optional + +from sqlalchemy.orm import Session + +from app.core.wiki_config import wiki_settings +from app.models.user import User +from app.models.wiki import WikiProject +from app.services.knowledge.code_wiki.source import SourceRepository + +logger = logging.getLogger(__name__) + +# Code wikis are not filed under anybody's team or organization: the repository +# decides who may read them, so the namespace would be a second, weaker answer to a +# question already settled. +CODE_WIKI_NAMESPACE = "default" + + +class CodeWikiOwnerMissing(RuntimeError): + """Raised when the configured wiki account does not exist.""" + + +def wiki_owner(db: Session, requester: User) -> User: + """The account a code wiki is created under. + + Falls back to the requester only when no wiki account is configured + (``WIKI_DEFAULT_USER_ID=0``), which is the single-user setup where the two are + the same thing anyway. + """ + configured_id = wiki_settings.DEFAULT_USER_ID + if configured_id <= 0: + return requester + + # Queried directly: user_service.get_user_by_id raises a 404 for a missing user, + # which would surface as "not found" on a request that found everything it asked + # for and is really a misconfiguration. + owner = db.query(User).filter(User.id == configured_id).first() + if owner is None: + raise CodeWikiOwnerMissing( + f"Configured wiki account {configured_id} does not exist. " + "Check WIKI_DEFAULT_USER_ID." + ) + return owner + + +def project_for(db: Session, source: SourceRepository) -> WikiProject: + """The registry row for a repository, created if this is the first time. + + Shared with the legacy wiki deliberately — one row per repository, whichever + feature asked first. The two never mix beyond that, because a version line is + keyed by ``wiki_generations.kind_id`` and legacy rows carry ``0``. + """ + existing = ( + db.query(WikiProject) + .filter(WikiProject.source_url == source.source_url) + .first() + ) + if existing is not None: + return existing + + project = WikiProject( + project_name=source.project_name, + project_type="git", + source_type=source.source_type, + source_url=source.source_url, + source_domain=source.source_domain, + description="", + ext={}, + kind_id=0, + is_active=True, + ) + db.add(project) + db.flush() + logger.info( + "[code_wiki] registered repository %s as project %s", + source.project_name, + project.id, + ) + return project + + +def existing_wiki_id(db: Session, source: SourceRepository) -> Optional[int]: + """The code wiki already built from this repository, if there is one.""" + project = ( + db.query(WikiProject) + .filter(WikiProject.source_url == source.source_url) + .first() + ) + if project is None or not project.kind_id: + return None + return project.kind_id + + +def claim_repository(db: Session, project: WikiProject, kind_id: int) -> None: + """Record that this repository's wiki is ``kind_id``.""" + project.kind_id = kind_id + db.flush() diff --git a/backend/app/services/knowledge/content_scope.py b/backend/app/services/knowledge/content_scope.py index 538ead5a3c..6bcf29728b 100644 --- a/backend/app/services/knowledge/content_scope.py +++ b/backend/app/services/knowledge/content_scope.py @@ -22,14 +22,17 @@ them explicitly. """ +from sqlalchemy import or_ from sqlalchemy.orm import Query +from app.models.kind import Kind from app.models.knowledge import ( ContentOrigin, DocumentSourceType, KnowledgeDocument, KnowledgeFolder, ) +from app.schemas.knowledge import KnowledgeBaseType # Source type marking a document as an indexed source file rather than a wiki page. CODE_TARGET_SOURCE_TYPE = DocumentSourceType.CODE.value @@ -74,3 +77,33 @@ def code_targets(query: Query) -> Query: to name code targets deliberately. """ return query.filter(KnowledgeDocument.source_type == CODE_TARGET_SOURCE_TYPE) + + +def exclude_code_wikis(query: Query) -> Query: + """Keep code wikis out of a ``Kind`` query listing knowledge bases. + + The default for every general listing. A code wiki is a projection an agent + rewrites on a schedule, so it belongs in the reader built for that, not among + knowledge bases a person fills by hand — and an agent shown one through the MCP + tool may well try to write into it, where the next publish silently deletes + whatever it added. + + Relying on ownership to hide them instead would be incidental rather than + stated: it holds only while the wiki account is somebody else, and stops holding + for that account, for an administrator, or if that decision is ever revisited. + """ + kb_type = Kind.json["spec"]["kbType"].as_string() + # NULL has to be spelled out: a knowledge base predating kbType compares NULL + # against the literal, which is neither true nor false, and the row would vanish + # from every listing. Rendered through SQLAlchemy's JSON accessor rather than + # json_unquote, which only exists on MySQL. + return query.filter( + or_(kb_type.is_(None), kb_type != KnowledgeBaseType.CODE_WIKI.value) + ) + + +def only_code_wikis(query: Query) -> Query: + """The other half: a ``Kind`` query restricted to code wikis.""" + return query.filter( + Kind.json["spec"]["kbType"].as_string() == KnowledgeBaseType.CODE_WIKI.value + ) diff --git a/backend/app/services/knowledge/knowledge_service.py b/backend/app/services/knowledge/knowledge_service.py index 66544ce21e..698c804256 100644 --- a/backend/app/services/knowledge/knowledge_service.py +++ b/backend/app/services/knowledge/knowledge_service.py @@ -60,7 +60,7 @@ get_user_groups, get_view_role_in_group, ) -from app.services.knowledge.content_scope import wiki_pages +from app.services.knowledge.content_scope import exclude_code_wikis, wiki_pages from app.services.knowledge.folder_policy import assert_document_can_be_placed_in_folder from app.services.knowledge.knowledge_access_policy import ( can_directly_access_knowledge_base as evaluate_direct_knowledge_base_access, @@ -512,16 +512,14 @@ def list_knowledge_bases( # Single query to get personal and explicitly shared knowledge bases # Personal: user_id matches and namespace is "default" # Shared: id is in shared_kb_ids - all_kbs = ( - db.query(Kind) - .filter( + all_kbs = exclude_code_wikis( + db.query(Kind).filter( Kind.kind == "KnowledgeBase", Kind.is_active == True, ((Kind.user_id == user_id) & (Kind.namespace == "default")) | (Kind.id.in_(shared_kb_ids) if shared_kb_ids else False), ) - .all() - ) + ).all() # Separate into personal and shared for sorting personal = [ @@ -548,11 +546,12 @@ def list_knowledge_bases( # KBs belonging to this group (native group KBs only) # Entity-authorized KBs are shown in personal shared_with_me instead group_kbs = ( - db.query(Kind) - .filter( - Kind.kind == "KnowledgeBase", - Kind.namespace == group_name, - Kind.is_active == True, + exclude_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.namespace == group_name, + Kind.is_active == True, + ) ) .order_by(Kind.updated_at.desc()) .all() @@ -623,9 +622,11 @@ def list_knowledge_bases( # Team: namespace is in accessible_groups # Organization: namespace has level='organization' # Shared: id is in shared_kb_ids - query = db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, + query = exclude_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + ) ) conditions = [(Kind.user_id == user_id) & (Kind.namespace == "default")] @@ -2097,11 +2098,12 @@ def get_accessible_knowledge( # Get knowledge bases in this group group_kbs = ( - db.query(Kind) - .filter( - Kind.kind == "KnowledgeBase", - Kind.namespace == group_name, - Kind.is_active == True, + exclude_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.namespace == group_name, + Kind.is_active == True, + ) ) .order_by(Kind.updated_at.desc()) .all() @@ -2679,12 +2681,13 @@ def get_all_knowledge_bases_grouped( # 1. Get personal knowledge bases created by user (single query) personal_created = ( - db.query(Kind) - .filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Kind.namespace == "default", - Kind.user_id == user_id, + exclude_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Kind.namespace == "default", + Kind.user_id == user_id, + ) ) .order_by(Kind.updated_at.desc()) .all() @@ -2708,12 +2711,13 @@ def get_all_knowledge_bases_grouped( shared_kbs: list[Kind] = [] if shared_kb_ids: shared_kbs = ( - db.query(Kind) - .filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Kind.id.in_(shared_kb_ids), - Kind.user_id != user_id, + exclude_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Kind.id.in_(shared_kb_ids), + Kind.user_id != user_id, + ) ) .order_by(Kind.updated_at.desc()) .all() diff --git a/backend/tests/api/test_knowledge_code_wiki.py b/backend/tests/api/test_knowledge_code_wiki.py index 558deddcdb..c5627d481e 100644 --- a/backend/tests/api/test_knowledge_code_wiki.py +++ b/backend/tests/api/test_knowledge_code_wiki.py @@ -13,6 +13,7 @@ from app.core.security import create_access_token from app.models.user import User from app.schemas.knowledge import KnowledgeBaseType +from app.services.knowledge.code_wiki.source import SourceAccessDenied CREATE_URL = "/api/knowledge-bases/code-wikis" @@ -239,3 +240,214 @@ def test_triggering_a_run_requires_authentication(test_client: TestClient): response = test_client.post(_run_url(1), json={}) assert response.status_code in (401, 403) + + +# --- who owns a code wiki, and how many there are --------------------------- + + +@pytest.fixture +def wiki_account(test_db: Session) -> User: + """A wiki account that is not the requester, which is the real deployment.""" + from app.core.security import get_password_hash + + account = User( + user_name="wiki-bot", + password_hash=get_password_hash("irrelevant"), + email="wiki-bot@example.com", + is_active=True, + git_info=None, + ) + test_db.add(account) + test_db.commit() + return account + + +def _point_wiki_account_at(monkeypatch: pytest.MonkeyPatch, user_id: int) -> None: + from app.core.wiki_config import wiki_settings + + monkeypatch.setattr(wiki_settings, "DEFAULT_USER_ID", user_id) + + +def test_a_code_wiki_belongs_to_the_wiki_account_not_the_requester( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + test_user: User, + wiki_account: User, + monkeypatch: pytest.MonkeyPatch, + kind_services_use_test_db, +): + """The account whose credentials clone the repository is the one the wiki + depends on. Attributing it to the requester would make a repository's + documentation disappear when one person leaves.""" + from app.models.kind import Kind + + _point_wiki_account_at(monkeypatch, wiki_account.id) + + kb_id = _create_wiki(test_client, auth_headers) + + kind = test_db.get(Kind, kb_id) + assert kind.user_id == wiki_account.id + assert kind.user_id != test_user.id + + +def test_asking_for_a_repository_that_already_has_a_wiki_returns_it( + test_client: TestClient, + auth_headers: dict[str, str], + wiki_account: User, + monkeypatch: pytest.MonkeyPatch, + kind_services_use_test_db, +): + """Two full generations and two indexes for one repository's documentation is + the thing worth preventing. 200 rather than 201 says which happened.""" + _point_wiki_account_at(monkeypatch, wiki_account.id) + first_id = _create_wiki(test_client, auth_headers) + + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + again = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + + assert again.status_code == 200, again.text + assert again.json()["id"] == first_id + + +def test_the_repository_is_registered_so_the_database_can_refuse_a_second_wiki( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + wiki_account: User, + monkeypatch: pytest.MonkeyPatch, + kind_services_use_test_db, +): + """source_url is UNIQUE, which is what holds when two requests race — a check + against a JSON field would leave a window exactly where it matters.""" + from app.models.wiki import WikiProject + + _point_wiki_account_at(monkeypatch, wiki_account.id) + kb_id = _create_wiki(test_client, auth_headers) + + project = test_db.query(WikiProject).one() + assert project.kind_id == kb_id + assert project.source_url == "https://github.com/wecode-ai/Wegent.git" + + +def test_a_missing_wiki_account_is_reported_rather_than_silently_reassigned( + test_client: TestClient, + auth_headers: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + kind_services_use_test_db, +): + """Falling back to the requester would create a wiki owned by someone the + operator did not choose, with credentials that may not reach the repository.""" + _point_wiki_account_at(monkeypatch, 987654) + + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + response = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + + assert response.status_code == 500 + assert "WIKI_DEFAULT_USER_ID" in response.json()["detail"] + + +# --- listing ---------------------------------------------------------------- + + +LIST_URL = "/api/knowledge-bases/code-wikis" + + +def test_the_list_shows_a_wiki_whose_repository_the_caller_can_read( + test_client: TestClient, + auth_headers: dict[str, str], + wiki_account: User, + monkeypatch: pytest.MonkeyPatch, + kind_services_use_test_db, +): + """The wiki belongs to the wiki account, so nothing in the knowledge-base ACL + would show it — the repository is what does.""" + _point_wiki_account_at(monkeypatch, wiki_account.id) + kb_id = _create_wiki(test_client, auth_headers) + + with patch("app.api.endpoints.knowledge.readable_wiki_ids", return_value={kb_id}): + response = test_client.get(LIST_URL, headers=auth_headers) + + assert response.status_code == 200, response.text + body = response.json() + assert body["total"] == 1 + assert body["items"][0]["project_name"] == "wecode-ai/Wegent" + + +def test_the_list_hides_a_wiki_whose_repository_the_caller_cannot_read( + test_client: TestClient, + auth_headers: dict[str, str], + wiki_account: User, + monkeypatch: pytest.MonkeyPatch, + kind_services_use_test_db, +): + _point_wiki_account_at(monkeypatch, wiki_account.id) + _create_wiki(test_client, auth_headers) + + with patch("app.api.endpoints.knowledge.readable_wiki_ids", return_value=set()): + response = test_client.get(LIST_URL, headers=auth_headers) + + assert response.json() == {"items": [], "total": 0} + + +def test_a_code_wiki_stays_out_of_the_general_knowledge_base_list( + test_client: TestClient, + auth_headers: dict[str, str], + wiki_account: User, + monkeypatch: pytest.MonkeyPatch, + kind_services_use_test_db, +): + """Stated by the scope rather than left to ownership: relying on the account not + matching holds only until an administrator looks, or the account itself does.""" + _point_wiki_account_at(monkeypatch, wiki_account.id) + kb_id = _create_wiki(test_client, auth_headers) + + grouped = test_client.get( + "/api/knowledge-bases/all-grouped", headers=auth_headers + ).json() + + listed = [ + kb["id"] + for bucket in ( + grouped["personal"]["created_by_me"], + grouped["personal"]["shared_with_me"], + ) + for kb in bucket + ] + assert kb_id not in listed + + +def test_creation_is_refused_when_the_wiki_account_cannot_clone( + test_client: TestClient, + auth_headers: dict[str, str], + test_user: User, + wiki_account: User, + monkeypatch: pytest.MonkeyPatch, + kind_services_use_test_db, +): + """The requester's access is not the one that matters at clone time. Without + this the mismatch surfaces as a failed generation, and nothing in that failure + says the account just needs adding to the repository.""" + _point_wiki_account_at(monkeypatch, wiki_account.id) + + def only_the_requester(db, user_id, source): + if user_id == test_user.id: + return {"has_access": True} + raise SourceAccessDenied("not a member") + + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + side_effect=only_the_requester, + ): + response = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + + assert response.status_code == 403 + detail = response.json()["detail"] + assert "wiki-bot" in detail + assert "WIKI_DEFAULT_USER_ID=0" in detail diff --git a/backend/tests/services/knowledge/code_wiki/test_read_access.py b/backend/tests/services/knowledge/code_wiki/test_read_access.py new file mode 100644 index 0000000000..a7d3572a57 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_read_access.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for deciding who may read a code wiki. + +A code wiki belongs to the wiki account, so knowledge-base ACLs grant nobody else +anything and this is the only check standing between a reader and the wiki. That +makes both directions worth pinning: it has to let repository members in without a +round trip per read, and it has to stay shut when the answer cannot be obtained. +""" + +from unittest.mock import patch + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.user import User +from app.services.knowledge.code_wiki.read_access import may_read_code_wiki +from app.services.knowledge.code_wiki.source import SourceAccessDenied + +SOURCE = { + "sourceType": "github", + "sourceUrl": "https://github.com/wecode-ai/Wegent.git", + "sourceDomain": "github.com", + "projectName": "wecode-ai/Wegent", +} + + +@pytest.fixture +def code_wiki(test_db: Session, test_user: User) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-read-access", + namespace="default", + user_id=99999, # the wiki account, not the reader + json={"spec": {"name": "wiki", "kbType": "code_wiki", "source": SOURCE}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def _with_cache(entries): + """Stand in for the repository list the provider layer keeps in Redis.""" + + async def reader(user, git_domain): + return entries + + # staticmethod: attached to a class it would bind and receive self, and the + # resulting TypeError would be swallowed as "no cache" — the test would pass for + # the wrong reason on the refusal cases and fail here. + return patch( + "app.services.knowledge.code_wiki.read_access.provider_for", + return_value=type( + "P", (), {"_get_all_repositories_from_cache": staticmethod(reader)} + )(), + ) + + +def _live_check(allowed: bool): + if allowed: + return patch( + "app.services.knowledge.code_wiki.source.assert_user_can_read_source", + return_value={"has_access": True}, + ) + return patch( + "app.services.knowledge.code_wiki.source.assert_user_can_read_source", + side_effect=SourceAccessDenied("no access"), + ) + + +def test_a_repository_in_the_users_cache_grants_access( + test_db: Session, test_user: User, code_wiki: Kind +): + """The common path, and it costs one Redis read rather than a call per view.""" + with _with_cache([{"full_name": "wecode-ai/Wegent"}]): + assert may_read_code_wiki(test_db, test_user, code_wiki) + + +def test_the_match_ignores_case(test_db: Session, test_user: User, code_wiki: Kind): + with _with_cache([{"full_name": "WeCode-AI/Wegent"}]): + assert may_read_code_wiki(test_db, test_user, code_wiki) + + +def test_a_cold_cache_falls_through_to_asking_about_the_one_repository( + test_db: Session, test_user: User, code_wiki: Kind +): + """A user who has never opened the repository picker must not be locked out; + building their whole list here would make a first read pay for every repository + they can see.""" + with _with_cache(None), _live_check(allowed=True): + assert may_read_code_wiki(test_db, test_user, code_wiki) + + +def test_a_repository_the_user_cannot_read_is_refused( + test_db: Session, test_user: User, code_wiki: Kind +): + with _with_cache([{"full_name": "someone-else/other"}]), _live_check(allowed=False): + assert not may_read_code_wiki(test_db, test_user, code_wiki) + + +def test_an_unreachable_provider_does_not_become_an_open_door( + test_db: Session, test_user: User, code_wiki: Kind +): + with ( + _with_cache(None), + patch( + "app.services.knowledge.code_wiki.source.assert_user_can_read_source", + side_effect=RuntimeError("provider is down"), + ), + ): + assert not may_read_code_wiki(test_db, test_user, code_wiki) + + +def test_a_wiki_with_no_repository_is_refused( + test_db: Session, test_user: User, code_wiki: Kind +): + """Nothing to judge by. A wiki nobody can attribute to a repository is one + nobody should inherit access to.""" + code_wiki.json = {"spec": {"name": "wiki", "kbType": "code_wiki"}} + test_db.flush() + + assert not may_read_code_wiki(test_db, test_user, code_wiki) diff --git a/backend/tests/services/knowledge/test_content_scope.py b/backend/tests/services/knowledge/test_content_scope.py index d55049a61a..99a9a99bdd 100644 --- a/backend/tests/services/knowledge/test_content_scope.py +++ b/backend/tests/services/knowledge/test_content_scope.py @@ -7,13 +7,16 @@ import pytest from sqlalchemy.orm import Session +from app.models.kind import Kind from app.models.knowledge import ContentOrigin, KnowledgeDocument, KnowledgeFolder from app.services.knowledge.content_scope import ( CODE_TARGET_SOURCE_TYPE, NO_FOLDER, code_targets, + exclude_code_wikis, generated_folders, generated_wiki_pages, + only_code_wikis, wiki_pages, ) @@ -148,3 +151,54 @@ def test_listing_documents_never_returns_code_targets(test_db: Session): ).all() assert [document.name for document in listed] == ["architecture"] + + +# --- keeping code wikis out of general listings ----------------------------- + + +def _kb(db: Session, name: str, kb_type: str | None) -> Kind: + spec: dict = {"name": name} + if kb_type is not None: + spec["kbType"] = kb_type + kind = Kind( + kind="KnowledgeBase", + name=name, + namespace="default", + user_id=1, + json={"spec": spec}, + is_active=True, + ) + db.add(kind) + db.flush() + return kind + + +def test_a_knowledge_base_predating_kb_type_is_still_listed(test_db: Session): + """NULL compared against a literal is neither true nor false, so spelling it out + is what keeps every knowledge base created before kbType existed from vanishing + out of every listing at once.""" + legacy = _kb(test_db, "legacy", None) + + listed = exclude_code_wikis(test_db.query(Kind)).all() + + assert legacy in listed + + +def test_code_wikis_are_kept_out_of_general_listings(test_db: Session): + """An agent shown one through the MCP tool may write into it, and the next + publish deletes whatever it added.""" + notebook = _kb(test_db, "notes", "notebook") + wiki = _kb(test_db, "wegent-wiki", "code_wiki") + + listed = exclude_code_wikis(test_db.query(Kind)).all() + + assert notebook in listed + assert wiki not in listed + + +def test_the_dedicated_scope_returns_only_code_wikis(test_db: Session): + _kb(test_db, "notes", "notebook") + _kb(test_db, "legacy", None) + wiki = _kb(test_db, "wegent-wiki", "code_wiki") + + assert only_code_wikis(test_db.query(Kind)).all() == [wiki] From 8c5b046c0694e85164c27aab4e9a5f57d8f7323f Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Tue, 4 Aug 2026 11:47:49 +0800 Subject: [PATCH 06/14] fix(knowledge): work through the review rounds, including one I had skipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-08-03 round went unread. Going back through every thread that is neither outdated nor resolved: **A documented scope that was not enforced.** The page-read endpoint says only your own generation is readable, but the internal dependency accepts any active user's JWT and never binds the caller to a generation — so any signed-in user could read any generation's pages, which are a copy of a wiki whose repository they may have no access to. The caller is now checked against the generation's owner; the fixed internal token stays unscoped, being an operator rather than a person. A run is now owned by the account that executes it rather than the one that asked, so that check agrees with the identity the agent authenticates as — and with the account that owns the knowledge base it publishes into. **Two branches that were never given their inputs.** decide_run_mode has had the periodic-rebuild thresholds since it was written and start_generation never passed them, so they defaulted to "no drift yet" and a wiki running on increments was never rebuilt however long it ran. Wired, with one query. The top-level-module guard is removed rather than wired. It exists for a diff filtered to documented file types, where a module of assets could vanish without appearing in the diff; ours is unfiltered, so the case it guards cannot arise, and an unwired branch is a defence that is not there. Performance, on the listing path: - readable_wiki_ids read the cache once per wiki while its docstring claimed once per host — N event loops and N Redis reads for N wikis, plus one live provider call each on a cold cache. Now keyed by host, with only the misses falling through, and a test that counts the reads so the docstring cannot drift again. - Document counts come from one grouped query rather than a COUNT per wiki, and the listing is paginated. Access is judged before paging, or a page would come back short with a wrong total. - Returning an existing wiki reported zero documents while the listing reported the real number for the same wiki. exclude_code_wikis reached six queries and missed siblings in the same methods. With WIKI_DEFAULT_USER_ID=0 the wiki account *is* the requester, so ownership hides nothing and every unfiltered listing exposed the wiki — the case the helper's own docstring warned about. Now applied to all thirteen. Also: Mermaid frontmatter no longer reads as the diagram type (the types added for the pinned version are exactly the ones that use it, so this was newly reachable); both the -beta and stable spellings are accepted rather than guessed between; a section is reported once however its prefix is cased; the skill fails loudly when it cannot derive the read URL instead of reporting the resulting 404 as "page does not exist"; and _folder_resolver drops a user_id it never used. Not taken: the Alembic graph is a single head (2b5791acc5fa) — verified, the branchpoints reconverge. Co-Authored-By: Claude Opus 5 --- backend/app/api/endpoints/knowledge.py | 27 +++++- backend/app/api/endpoints/wiki.py | 48 ++++++++- .../knowledge/code_wiki/generation.py | 44 +++++++++ .../knowledge/code_wiki/mermaid_check.py | 26 ++++- .../knowledge/code_wiki/projection.py | 4 +- .../knowledge/code_wiki/publish_gate.py | 18 ++-- .../knowledge/code_wiki/read_access.py | 63 +++++++++--- .../services/knowledge/code_wiki/run_mode.py | 18 ---- .../services/knowledge/code_wiki/runner.py | 6 +- .../services/knowledge/knowledge_service.py | 97 ++++++++++--------- .../skills/wiki_submit/wiki_submit.js | 16 ++- .../knowledge/code_wiki/test_generation.py | 39 ++++++++ .../knowledge/code_wiki/test_mermaid_check.py | 16 +++ .../knowledge/code_wiki/test_read_access.py | 86 ++++++++++++++++ .../knowledge/code_wiki/test_run_mode.py | 56 ----------- 15 files changed, 411 insertions(+), 153 deletions(-) diff --git a/backend/app/api/endpoints/knowledge.py b/backend/app/api/endpoints/knowledge.py index 6cb9630f81..ee8e6f1a85 100644 --- a/backend/app/api/endpoints/knowledge.py +++ b/backend/app/api/endpoints/knowledge.py @@ -556,12 +556,19 @@ def _code_wiki_response(db: Session, knowledge_base_id: int) -> KnowledgeBaseRes raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Code wiki not found" ) - return KnowledgeBaseResponse.from_kind(kind) + # Counted rather than left to default to zero: this path returns a wiki that + # already exists and normally holds pages, and reporting none of them would + # disagree with what the listing says about the same wiki. + return KnowledgeBaseResponse.from_kind( + kind, KnowledgeService.get_document_count(db, kind.id) + ) @router.get("/code-wikis", response_model=CodeWikiListResponse) @trace_sync("list_code_wikis", "knowledge.api") def list_code_wikis( + page: int = Query(1, ge=1, description="Page number"), + limit: int = Query(20, ge=1, le=100, description="Items per page"), current_user: User = Depends(security.get_current_user), db: Session = Depends(get_db), ): @@ -581,13 +588,23 @@ def list_code_wikis( .order_by(Kind.updated_at.desc()) .all() ) + # Access is judged before paging, because a page of rows the caller cannot read + # would come back short and the total would be wrong. The judgement itself is + # one cache read per host, not one per wiki. readable = readable_wiki_ids(db, current_user, wikis) + visible = [kind for kind in wikis if kind.id in readable] - items = [_code_wiki_list_item(db, kind) for kind in wikis if kind.id in readable] - return CodeWikiListResponse(items=items, total=len(items)) + window = visible[(page - 1) * limit : page * limit] + # One grouped query for the whole page rather than a COUNT per wiki. + counts = KnowledgeService.get_document_counts(db, [kind.id for kind in window]) + + return CodeWikiListResponse( + items=[_code_wiki_list_item(kind, counts.get(kind.id, 0)) for kind in window], + total=len(visible), + ) -def _code_wiki_list_item(db: Session, kind: Kind) -> CodeWikiListItem: +def _code_wiki_list_item(kind: Kind, document_count: int) -> CodeWikiListItem: spec = (kind.json or {}).get("spec", {}) source = spec.get("source") or {} return CodeWikiListItem( @@ -598,7 +615,7 @@ def _code_wiki_list_item(db: Session, kind: Kind) -> CodeWikiListItem: source_url=str(source.get("sourceUrl", "") or ""), last_published_at=spec.get(PUBLISHED_AT_KEY), last_published_commit=str(spec.get(PUBLISHED_COMMIT_KEY, "") or ""), - document_count=KnowledgeService.get_document_count(db, kind.id), + document_count=document_count, created_at=kind.created_at, updated_at=kind.updated_at, ) diff --git a/backend/app/api/endpoints/wiki.py b/backend/app/api/endpoints/wiki.py index e9a4162f31..380d49f90a 100644 --- a/backend/app/api/endpoints/wiki.py +++ b/backend/app/api/endpoints/wiki.py @@ -13,6 +13,7 @@ from app.core.wiki_config import wiki_settings from app.db.session import get_wiki_db from app.models.user import User +from app.models.wiki import WikiGeneration from app.schemas.wiki import ( WikiContentInDB, WikiContentWriteRequest, @@ -80,6 +81,50 @@ def _verify_internal_token( ) +def _internal_caller( + authorization: str = Header(default=""), + db: Session = Depends(get_db), +) -> Optional[User]: + """Authenticate an internal caller and say which user it is, if any. + + ``None`` means the fixed internal token was used, which is a trusted operator + rather than a person and is not scoped to one generation. + """ + _verify_internal_token(authorization=authorization, db=db) + token = authorization[7:].strip() + if token == wiki_settings.INTERNAL_API_TOKEN: + return None + try: + return security.get_current_user_from_token(token, db) + except Exception: # pragma: no cover - _verify_internal_token already accepted it + return None + + +def _assert_caller_owns_generation( + wiki_db: Session, caller: Optional[User], generation_id: int +) -> None: + """Refuse a caller asking about a generation that is not theirs. + + Authenticating a JWT says who is asking, not what they may ask about. Without + this any signed-in user could read any generation's pages, which is a different + thing from the documented "your own version" — and that version is a copy of a + wiki whose repository they may have no access to at all. + """ + if caller is None: + return + + generation = ( + wiki_db.query(WikiGeneration).filter(WikiGeneration.id == generation_id).first() + ) + if generation is None: + raise HTTPException(status_code=404, detail="Generation not found") + if generation.user_id != caller.id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="This generation belongs to another account", + ) + + def _resolve_user_id( account_id: Optional[int], current_user: User, main_db: Session ) -> int: @@ -222,7 +267,7 @@ def save_wiki_generation_contents( def read_wiki_generation_page( generation_id: int, path: str = Query(..., min_length=1, description="Stable page path to read"), - _: None = Depends(_verify_internal_token), + caller: Optional[User] = Depends(_internal_caller), wiki_db: Session = Depends(get_wiki_db), ): """Read one page of the version the agent is writing into (internal use). @@ -236,6 +281,7 @@ def read_wiki_generation_page( Answers 404 when the path holds no page. That is a useful answer rather than a failure — in an incremental run it means the page is new. """ + _assert_caller_owns_generation(wiki_db, caller, generation_id) page = wiki_service.get_generation_page( wiki_db=wiki_db, generation_id=generation_id, path=path ) diff --git a/backend/app/services/knowledge/code_wiki/generation.py b/backend/app/services/knowledge/code_wiki/generation.py index c476514dc9..d3adf0fa9b 100644 --- a/backend/app/services/knowledge/code_wiki/generation.py +++ b/backend/app/services/knowledge/code_wiki/generation.py @@ -149,11 +149,14 @@ def start_generation( ) last_commit = published_commit(db, knowledge_base) + since_full = _runs_since_the_last_full(db, knowledge_base.id, now=now) decision = decide_run_mode( head_commit=head_commit, last_commit=last_commit or None, changed_paths=changed_paths, total_source_files=total_source_files, + incrementals_since_full=since_full[0], + days_since_full=since_full[1], # Passed only when given: the callee defaults it, and forwarding ``None`` # would replace that default with nothing. **({"policy": policy} if policy is not None else {}), @@ -261,6 +264,47 @@ def finish_generation( ) +def _runs_since_the_last_full( + db: Session, kind_id: int, now: Optional[datetime] = None +) -> tuple[int, Optional[float]]: + """How much has accumulated since this wiki was last rebuilt from scratch. + + Feeds the periodic-rebuild thresholds, which had the branches for this from the + start and were never given the inputs — so they defaulted to "no drift yet" and + the wiki was never rebuilt no matter how long it ran on increments. + + Returns: + The number of completed incremental runs, and the age in days of the last + full one — ``None`` when there has never been a full run to age. + """ + latest_full = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == kind_id, + WikiGeneration.generation_type == WikiGenerationType.FULL, + WikiGeneration.status == WikiGenerationStatus.COMPLETED, + ) + .order_by(WikiGeneration.completed_at.desc()) + .first() + ) + if latest_full is None: + return 0, None + + incrementals = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == kind_id, + WikiGeneration.generation_type == WikiGenerationType.INCREMENTAL, + WikiGeneration.status == WikiGenerationStatus.COMPLETED, + WikiGeneration.completed_at > latest_full.completed_at, + ) + .count() + ) + reference = (now or datetime.now(timezone.utc)).replace(tzinfo=None) + age_days = (reference - latest_full.completed_at).total_seconds() / 86400 + return incrementals, max(0.0, age_days) + + def version_page_count(db: Session, generation_id: int) -> int: """How many pages a version holds, for status displays.""" return len(read_version_pages(db, generation_id)) diff --git a/backend/app/services/knowledge/code_wiki/mermaid_check.py b/backend/app/services/knowledge/code_wiki/mermaid_check.py index c7f74a8fea..1350ad8d13 100644 --- a/backend/app/services/knowledge/code_wiki/mermaid_check.py +++ b/backend/app/services/knowledge/code_wiki/mermaid_check.py @@ -29,6 +29,10 @@ # missing here is reported as a broken diagram and sent back to the agent to "fix", # which wastes a round on a diagram that renders perfectly well — so this list has to # be widened whenever the pinned version gains a declaration. +# +# Newer types carry a "-beta" suffix that Mermaid drops as they stabilise, and both +# spellings are listed rather than guessed between: the cost of an extra entry is +# nothing, and the cost of a missing one is a wasted round trip through the model. KNOWN_DIAGRAM_TYPES: frozenset[str] = frozenset( { "architecture-beta", @@ -54,6 +58,7 @@ "pie", "quadrantchart", "radar", + "radar-beta", "requirementdiagram", "sankey-beta", "sequencediagram", @@ -61,6 +66,7 @@ "statediagram-v2", "timeline", "treemap", + "treemap-beta", "xychart-beta", "zenuml", } @@ -100,7 +106,7 @@ def __str__(self) -> str: def _diagram_type_of(body: Sequence[str]) -> str: """First meaningful token of a diagram, lowercased; empty when there is none.""" - for line in body: + for line in _without_frontmatter(body): stripped = line.strip() if not stripped or stripped.startswith("%%"): continue @@ -109,6 +115,24 @@ def _diagram_type_of(body: Sequence[str]) -> str: return "" +def _without_frontmatter(body: Sequence[str]) -> Sequence[str]: + """Drop a leading ``--- ... ---`` block. + + Mermaid allows YAML frontmatter before the diagram type, which several of the + types added for the pinned version use. Read as the diagram itself, the opening + ``---`` becomes the type, and a diagram that renders is reported as unknown. + """ + first = next((index for index, line in enumerate(body) if line.strip()), None) + if first is None or body[first].strip() != "---": + return body + + for index in range(first + 1, len(body)): + if body[index].strip() == "---": + return body[index + 1 :] + # Unterminated: not frontmatter, whatever else it is. + return body + + def _unbalanced_bracket(body: Sequence[str]) -> str: """Report the first unbalanced bracket, ignoring anything inside quotes.""" stack: List[str] = [] diff --git a/backend/app/services/knowledge/code_wiki/projection.py b/backend/app/services/knowledge/code_wiki/projection.py index ef2f844009..7783631e4c 100644 --- a/backend/app/services/knowledge/code_wiki/projection.py +++ b/backend/app/services/knowledge/code_wiki/projection.py @@ -88,7 +88,7 @@ class ProjectionOutcome: unfinished_index_cleanup: tuple[str, ...] = field(default=()) -def _folder_resolver(db: Session, kind_id: int, user_id: int): +def _folder_resolver(db: Session, kind_id: int): """Return a function creating (or finding) the folder chain for a page path.""" from app.models.knowledge import KnowledgeFolder @@ -173,7 +173,7 @@ def apply_projection_plan( Returns: What was done, including any RAG cleanup left to retry. """ - resolve_folder = _folder_resolver(db, kind_id, user_id) + resolve_folder = _folder_resolver(db, kind_id) # --- before the transaction commits: content into storage ------------------- # Written first on purpose. An orphaned object is litter; content deleted before diff --git a/backend/app/services/knowledge/code_wiki/publish_gate.py b/backend/app/services/knowledge/code_wiki/publish_gate.py index a6a4412ae3..c0c3226609 100644 --- a/backend/app/services/knowledge/code_wiki/publish_gate.py +++ b/backend/app/services/knowledge/code_wiki/publish_gate.py @@ -152,13 +152,17 @@ def _sections_without_a_page(pages: Sequence[PageSource]) -> list[str]: in a gate that throws away the run. """ present = {collation_key(page.path) for page in pages} - missing = { - page.path.rsplit("/", 1)[0] - for page in pages - if "/" in page.path - and collation_key(page.path.rsplit("/", 1)[0]) not in present - } - return sorted(missing) + # Keyed the same way membership is tested. Collecting the raw string instead + # would report one section twice when two pages spell its prefix differently. + missing: dict[str, str] = {} + for page in pages: + if "/" not in page.path: + continue + section = page.path.rsplit("/", 1)[0] + key = collation_key(section) + if key not in present: + missing.setdefault(key, section) + return sorted(missing.values()) def _structure_warnings(pages: Sequence[PageSource]) -> tuple[str, ...]: diff --git a/backend/app/services/knowledge/code_wiki/read_access.py b/backend/app/services/knowledge/code_wiki/read_access.py index e11a3a59e0..59569fc2f3 100644 --- a/backend/app/services/knowledge/code_wiki/read_access.py +++ b/backend/app/services/knowledge/code_wiki/read_access.py @@ -51,18 +51,23 @@ def may_read_code_wiki(db: Session, user: User, knowledge_base: Kind) -> bool: ) return False - if _repository_is_in_the_users_cache(user, source): + if source.project_name.casefold() in _cached_repository_names(user, source): return True return _repository_is_readable_now(db, user, source) -def _repository_is_in_the_users_cache(user: User, source: SourceRepository) -> bool: - """Look for the repository in the list the provider layer already caches.""" +def _cached_repository_names(user: User, source: SourceRepository) -> frozenset[str]: + """The repositories this user can reach on this host, as the cache has them. + + Keyed by host rather than by repository, so judging many wikis on one host costs + one read. Returns an empty set for a cold cache, which the caller distinguishes + from "not a member" by falling through to the live check. + """ provider = provider_for(source.source_type) reader = getattr(provider, "_get_all_repositories_from_cache", None) if reader is None: - return False + return frozenset() try: cached = _run_async_in_new_loop(reader(user, source.source_domain)) @@ -72,13 +77,11 @@ def _repository_is_in_the_users_cache(user: User, source: SourceRepository) -> b user.id, exc, ) - return False - - if not cached: - return False + return frozenset() - wanted = source.project_name.casefold() - return any(_full_name(entry).casefold() == wanted for entry in cached) + return frozenset( + _full_name(entry).casefold() for entry in (cached or []) if _full_name(entry) + ) def _full_name(entry: Any) -> str: @@ -121,11 +124,41 @@ def _repository_is_readable_now( def readable_wiki_ids(db: Session, user: User, knowledge_bases: list[Kind]) -> set[int]: """Which of these code wikis ``user`` may read. - Used by the list, where the cache lookup is one Redis read shared across every - wiki rather than one per wiki. + The cache is read once per host, not once per wiki. Judging each separately would + open a fresh event loop and issue a fresh Redis read every time, and on a cold + cache would put one live provider call per wiki on the request thread — the cost + of a listing growing with the number of documented repositories. """ - readable: set[int] = set() + sources: dict[int, SourceRepository] = {} for knowledge_base in knowledge_bases: - if may_read_code_wiki(db, user, knowledge_base): - readable.add(knowledge_base.id) + source = SourceRepository.from_spec( + (knowledge_base.json or {}).get("spec", {}).get("source") + ) + if source is None or not source.project_name: + logger.warning( + "[code_wiki] kb %s has no source repository; read refused", + knowledge_base.id, + ) + continue + sources[knowledge_base.id] = source + + caches: dict[tuple[str, str], frozenset[str]] = {} + readable: set[int] = set() + unresolved: dict[int, SourceRepository] = {} + + for kind_id, source in sources.items(): + host = (source.source_type, source.source_domain) + if host not in caches: + caches[host] = _cached_repository_names(user, source) + if source.project_name.casefold() in caches[host]: + readable.add(kind_id) + else: + unresolved[kind_id] = source + + # Only what the cache could not answer. A warm cache leaves this empty, which is + # the common case; a cold one pays per repository, which is why it is last. + for kind_id, source in unresolved.items(): + if _repository_is_readable_now(db, user, source): + readable.add(kind_id) + return readable diff --git a/backend/app/services/knowledge/code_wiki/run_mode.py b/backend/app/services/knowledge/code_wiki/run_mode.py index 37c088a31d..26cd735617 100644 --- a/backend/app/services/knowledge/code_wiki/run_mode.py +++ b/backend/app/services/knowledge/code_wiki/run_mode.py @@ -119,8 +119,6 @@ def decide_run_mode( changed_paths: Optional[Sequence[ChangedPath]] = None, incrementals_since_full: int = 0, days_since_full: Optional[float] = None, - previous_top_level_dirs: Optional[frozenset[str]] = None, - current_top_level_dirs: Optional[frozenset[str]] = None, policy: RunModePolicy = DEFAULT_POLICY, total_source_files: Optional[int] = None, ) -> RunModeDecision: @@ -133,10 +131,6 @@ def decide_run_mode( the extent of the change is unknown, so a full rebuild is chosen. incrementals_since_full: Incremental runs completed since the last full one. days_since_full: Days since the last full run, if one has happened. - previous_top_level_dirs: Top-level directories at ``last_commit``. Compared - with ``current_top_level_dirs`` to spot modules appearing or disappearing; - skipped when either side is unknown. - current_top_level_dirs: Top-level directories at ``head_commit``. policy: Thresholds to apply. total_source_files: Files under consideration at ``head_commit``, used for the proportional threshold; skipped when unknown. @@ -152,18 +146,6 @@ def decide_run_mode( RunMode.FULL, "extent of changes unknown, rebuilding to stay correct" ) - # Checked before the empty-diff shortcut: the diff may be filtered to documented - # file types while the directory sets come from the whole tree, so removing a - # module of, say, protos or assets can show up here and nowhere else. Skipping on - # an empty diff first would orphan that module's pages with nothing left to catch - # it, since the periodic rules also sit behind the skip. - if previous_top_level_dirs is not None and current_top_level_dirs is not None: - appeared = current_top_level_dirs - previous_top_level_dirs - disappeared = previous_top_level_dirs - current_top_level_dirs - if appeared or disappeared: - moved = ", ".join(sorted(appeared | disappeared)) - return RunModeDecision(RunMode.FULL, f"top-level modules changed: {moved}") - if not changed_paths: # The commit moved but nothing we document did — treat as unchanged rather # than paying for a rebuild. diff --git a/backend/app/services/knowledge/code_wiki/runner.py b/backend/app/services/knowledge/code_wiki/runner.py index bb8e3e9521..24e30207b3 100644 --- a/backend/app/services/knowledge/code_wiki/runner.py +++ b/backend/app/services/knowledge/code_wiki/runner.py @@ -143,7 +143,11 @@ def start_run( started = start_generation( db, knowledge_base=knowledge_base, - user=user, + # The account that runs the task, not the one that asked: it is the identity + # the agent authenticates as, so anything scoped to "this run's owner" has to + # agree with it, and it is the account that owns the knowledge base being + # published into. + user=task_user, head_commit=head_commit, changed_paths=changed_paths, total_source_files=total_source_files, diff --git a/backend/app/services/knowledge/knowledge_service.py b/backend/app/services/knowledge/knowledge_service.py index 698c804256..18149def9c 100644 --- a/backend/app/services/knowledge/knowledge_service.py +++ b/backend/app/services/knowledge/knowledge_service.py @@ -562,13 +562,15 @@ def list_knowledge_bases( # Organization knowledge bases are visible to all users # Query knowledge bases in namespaces with level='organization' organization_kbs = ( - db.query(Kind) - .join(Namespace, Kind.namespace == Namespace.name) - .filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Namespace.level == GroupLevel.organization.value, - Namespace.is_active == True, + exclude_code_wikis( + db.query(Kind) + .join(Namespace, Kind.namespace == Namespace.name) + .filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Namespace.level == GroupLevel.organization.value, + Namespace.is_active == True, + ) ) .order_by(Kind.updated_at.desc()) .all() @@ -2009,12 +2011,13 @@ def get_accessible_knowledge( # Get personal knowledge bases (created by user) personal_kbs = ( - db.query(Kind) - .filter( - Kind.kind == "KnowledgeBase", - Kind.user_id == user_id, - Kind.namespace == "default", - Kind.is_active == True, + exclude_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.user_id == user_id, + Kind.namespace == "default", + Kind.is_active == True, + ) ) .order_by(Kind.updated_at.desc()) .all() @@ -2053,12 +2056,13 @@ def get_accessible_knowledge( ) - included_personal_ids if extra_shared_ids: shared_kbs = ( - db.query(Kind) - .filter( - Kind.kind == "KnowledgeBase", - Kind.id.in_(extra_shared_ids), - Kind.is_active == True, - Kind.user_id != user_id, + exclude_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.id.in_(extra_shared_ids), + Kind.is_active == True, + Kind.user_id != user_id, + ) ) .order_by(Kind.updated_at.desc()) .all() @@ -2216,12 +2220,13 @@ def get_personal_knowledge_bases_grouped( # Get KBs created by user (personal knowledge bases, namespace=default) created_kbs = ( - db.query(Kind) - .filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Kind.namespace == "default", - Kind.user_id == user_id, + exclude_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Kind.namespace == "default", + Kind.user_id == user_id, + ) ) .order_by(Kind.updated_at.desc()) .all() @@ -2249,13 +2254,14 @@ def get_personal_knowledge_bases_grouped( shared_kbs = [] if shared_kb_ids: shared_kbs = ( - db.query(Kind) - .filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Kind.id.in_(shared_kb_ids), - Kind.namespace == "default", - Kind.user_id != user_id, # Exclude KBs created by current user + exclude_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Kind.id.in_(shared_kb_ids), + Kind.namespace == "default", + Kind.user_id != user_id, # Exclude KBs created by current user + ) ) .order_by(Kind.updated_at.desc()) .all() @@ -2791,11 +2797,12 @@ def get_all_knowledge_bases_grouped( group_kbs: list[Kind] = [] if accessible_groups: group_kbs = ( - db.query(Kind) - .filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Kind.namespace.in_(accessible_groups), + exclude_code_wikis( + db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Kind.namespace.in_(accessible_groups), + ) ) .order_by(Kind.updated_at.desc()) .all() @@ -2840,13 +2847,15 @@ def get_all_knowledge_bases_grouped( # 5. Get organization knowledge bases (single query) org_kbs = ( - db.query(Kind) - .join(Namespace, Kind.namespace == Namespace.name) - .filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Namespace.level == GroupLevel.organization.value, - Namespace.is_active == True, + exclude_code_wikis( + db.query(Kind) + .join(Namespace, Kind.namespace == Namespace.name) + .filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Namespace.level == GroupLevel.organization.value, + Namespace.is_active == True, + ) ) .order_by(Kind.updated_at.desc()) .all() diff --git a/backend/init_data/skills/wiki_submit/wiki_submit.js b/backend/init_data/skills/wiki_submit/wiki_submit.js index 160b38c6fa..8d27abdab1 100644 --- a/backend/init_data/skills/wiki_submit/wiki_submit.js +++ b/backend/init_data/skills/wiki_submit/wiki_submit.js @@ -157,8 +157,16 @@ function makeRequest(url, options, body) { * @returns {Promise} */ async function readPage(endpoint, token, generationId, pagePath) { - // The read endpoint sits beside the write one under /generations. - const base = endpoint.replace(/\/generations\/contents\/?$/, '') + // The read endpoint sits beside the write one under /generations. A custom + // endpoint that does not end in that suffix would make this a no-op and build a + // wrong URL, whose 404 would then read as "page does not exist". + const suffix = /\/generations\/contents\/?$/ + if (!suffix.test(endpoint)) { + console.error(`Error: cannot derive the read URL from endpoint '${endpoint}'.`) + console.error("It must end in '/generations/contents'.") + process.exit(1) + } + const base = endpoint.replace(suffix, '') const url = `${base}/generations/${generationId}/pages?path=${encodeURIComponent(pagePath)}` return makeRequest( @@ -293,7 +301,9 @@ async function cmdRead(args) { // A page that does not exist yet is an answer, not a failure: in an incremental // run it means this page is new. Reported on stderr so it cannot be mistaken // for content when stdout is redirected to a file. - if (/404|no page at/i.test(result.message || '')) { + // The backend's phrase, not a bare status code: any other 404 means something + // is misconfigured, and calling that "a new page" hides it. + if (/has no page at/i.test(result.message || '')) { console.error(`Page '${args.path}' does not exist yet`) return 0 } diff --git a/backend/tests/services/knowledge/code_wiki/test_generation.py b/backend/tests/services/knowledge/code_wiki/test_generation.py index 7c9d5d3539..8bf1aa1a37 100644 --- a/backend/tests/services/knowledge/code_wiki/test_generation.py +++ b/backend/tests/services/knowledge/code_wiki/test_generation.py @@ -347,3 +347,42 @@ def test_the_run_mode_reason_is_kept_for_troubleshooting( ) assert started.generation.ext["runModeReason"] + + +def test_a_wiki_running_on_increments_forever_is_eventually_rebuilt( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + """decide_run_mode has had this branch from the start and was never given the + inputs, so it defaulted to "no drift yet" and the rebuild never came however + long the wiki ran on increments.""" + from app.models.wiki import WikiGenerationType + from app.services.knowledge.code_wiki.run_mode import DEFAULT_POLICY + + _publish_a_first_wiki(test_db, knowledge_base, test_user, effects) + + for index in range(DEFAULT_POLICY.max_incrementals_since_full + 1): + done = WikiGeneration( + project_id=0, + kind_id=knowledge_base.id, + user_id=test_user.id, + task_id=0, + team_id=0, + generation_type=WikiGenerationType.INCREMENTAL, + source_snapshot={"commit": f"c{index}"}, + status=WikiGenerationStatus.COMPLETED, + completed_at=NOW + timedelta(minutes=index + 1), + ) + test_db.add(done) + test_db.flush() + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + now=NOW + timedelta(days=1), + ) + + assert RunMode(started.decision.mode) is RunMode.FULL + assert "incremental" in started.decision.reason.lower() diff --git a/backend/tests/services/knowledge/code_wiki/test_mermaid_check.py b/backend/tests/services/knowledge/code_wiki/test_mermaid_check.py index 8e3a966306..1f49266a28 100644 --- a/backend/tests/services/knowledge/code_wiki/test_mermaid_check.py +++ b/backend/tests/services/knowledge/code_wiki/test_mermaid_check.py @@ -192,3 +192,19 @@ def test_declarations_the_pinned_mermaid_supports_are_not_reported(declaration: sent back to the agent as a broken diagram — a wasted round spent "fixing" one that renders.""" assert check_mermaid_blocks(_fence(f"{declaration}\n A --> B")) == [] + + +def test_frontmatter_does_not_hide_the_diagram_type(): + """Mermaid allows `--- ... ---` before the type, and several of the types the + pinned version added use it. Read as the diagram, the opening `---` becomes the + type and something that renders is reported as broken.""" + body = "---\nconfig:\n theme: dark\n---\nradar-beta\n axis a, b" + + assert check_mermaid_blocks(_fence(body)) == [] + + +def test_an_unterminated_dash_line_is_not_treated_as_frontmatter(): + """Otherwise the whole diagram would be swallowed and nothing checked.""" + warnings = check_mermaid_blocks(_fence("---\nflowchat TD")) + + assert warnings diff --git a/backend/tests/services/knowledge/code_wiki/test_read_access.py b/backend/tests/services/knowledge/code_wiki/test_read_access.py index a7d3572a57..f1f7bf29a7 100644 --- a/backend/tests/services/knowledge/code_wiki/test_read_access.py +++ b/backend/tests/services/knowledge/code_wiki/test_read_access.py @@ -124,3 +124,89 @@ def test_a_wiki_with_no_repository_is_refused( test_db.flush() assert not may_read_code_wiki(test_db, test_user, code_wiki) + + +# --- judging many wikis at once --------------------------------------------- + + +class CountingCache: + """Counts how many times the repository cache is actually read.""" + + def __init__(self, entries): + self.entries = entries + self.reads = 0 + + def patch(self): + async def reader(user, git_domain): + self.reads += 1 + return self.entries + + return patch( + "app.services.knowledge.code_wiki.read_access.provider_for", + return_value=type( + "P", (), {"_get_all_repositories_from_cache": staticmethod(reader)} + )(), + ) + + +def _wiki(test_db: Session, name: str, project: str) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name=name, + namespace="default", + user_id=99999, + json={ + "spec": { + "name": name, + "kbType": "code_wiki", + "source": {**SOURCE, "projectName": project}, + } + }, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def test_many_wikis_on_one_host_cost_one_cache_read(test_db: Session, test_user: User): + """Judging each separately opens a fresh event loop and issues a fresh Redis + read every time, on a listing that grows with the number of repositories.""" + from app.services.knowledge.code_wiki.read_access import readable_wiki_ids + + wikis = [_wiki(test_db, f"kb-{index}", f"org/repo-{index}") for index in range(5)] + cache = CountingCache([{"full_name": f"org/repo-{index}"} for index in range(5)]) + + with cache.patch(): + readable = readable_wiki_ids(test_db, test_user, wikis) + + assert readable == {wiki.id for wiki in wikis} + assert cache.reads == 1 + + +def test_only_the_wikis_the_cache_could_not_answer_reach_the_provider( + test_db: Session, test_user: User +): + """A warm cache leaves nothing for the live check, which is why it runs last.""" + from app.services.knowledge.code_wiki.read_access import readable_wiki_ids + + known = _wiki(test_db, "known", "org/known") + unknown = _wiki(test_db, "unknown", "org/unknown") + cache = CountingCache([{"full_name": "org/known"}]) + asked: list[str] = [] + + def record(db, user_id, source): + asked.append(source.project_name) + raise SourceAccessDenied("no access") + + with ( + cache.patch(), + patch( + "app.services.knowledge.code_wiki.source.assert_user_can_read_source", + side_effect=record, + ), + ): + readable = readable_wiki_ids(test_db, test_user, [known, unknown]) + + assert readable == {known.id} + assert asked == ["org/unknown"] diff --git a/backend/tests/services/knowledge/code_wiki/test_run_mode.py b/backend/tests/services/knowledge/code_wiki/test_run_mode.py index f0f07ea63f..051b2f4792 100644 --- a/backend/tests/services/knowledge/code_wiki/test_run_mode.py +++ b/backend/tests/services/knowledge/code_wiki/test_run_mode.py @@ -94,44 +94,6 @@ def test_dependency_manifest_change_forces_a_rebuild(): assert "manifest" in decision.reason -def test_new_top_level_module_forces_a_rebuild(): - decision = decide_run_mode( - head_commit=HEAD, - last_commit=PREVIOUS, - changed_paths=[ChangedPath("gateway/main.go", "A")], - previous_top_level_dirs=frozenset({"backend", "frontend"}), - current_top_level_dirs=frozenset({"backend", "frontend", "gateway"}), - ) - - assert decision.mode is RunMode.FULL - assert "gateway" in decision.reason - - -def test_removed_top_level_module_forces_a_rebuild(): - decision = decide_run_mode( - head_commit=HEAD, - last_commit=PREVIOUS, - changed_paths=[ChangedPath("legacy/app.py", "D")], - previous_top_level_dirs=frozenset({"backend", "legacy"}), - current_top_level_dirs=frozenset({"backend"}), - ) - - assert decision.mode is RunMode.FULL - assert "legacy" in decision.reason - - -def test_unknown_top_level_dirs_skip_that_check(): - decision = decide_run_mode( - head_commit=HEAD, - last_commit=PREVIOUS, - changed_paths=_edits(2), - previous_top_level_dirs=None, - current_top_level_dirs=frozenset({"backend"}), - ) - - assert decision.mode is RunMode.INCREMENTAL - - def test_too_many_changed_files_forces_a_rebuild(): decision = decide_run_mode( head_commit=HEAD, @@ -196,24 +158,6 @@ def test_skip_takes_precedence_over_a_due_periodic_rebuild(): assert decision.mode is RunMode.SKIP -def test_a_removed_module_is_caught_even_when_the_diff_looks_empty(): - """The diff may be filtered to documented file types while the tree is not. - - Removing a module of protos or assets can then show up only in the directory sets, - and skipping first would orphan its pages with nothing left to catch it. - """ - decision = decide_run_mode( - head_commit=HEAD, - last_commit=PREVIOUS, - changed_paths=[], - previous_top_level_dirs=frozenset({"backend", "protos"}), - current_top_level_dirs=frozenset({"backend"}), - ) - - assert decision.mode is RunMode.FULL - assert "protos" in decision.reason - - def test_many_added_or_removed_files_force_a_rebuild_sooner_than_edits(): """Files appearing and disappearing reshape the wiki more than edits do.""" moves = [ChangedPath(f"src/new_{i}.py", "A") for i in range(16)] From 9367b99cf876e1b7a040d5886f2fe39fd1e14741 Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Tue, 4 Aug 2026 12:34:02 +0800 Subject: [PATCH 07/14] feat(knowledge): a reader for code wikis, and a list that is one request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old wiki reader was a parallel stack: its own markdown renderer, its own copy button, its own LaTeX block, and its own Mermaid component with no error handling at all — beside a knowledge base viewer that already had all of it, including a Mermaid fallback to the raw source. This builds on the shared one. Three regions. The wiki's structure on the left, the page in the middle, its own outline on the right. The middle switches between reading and a conversation rather than splitting: they never need to be visible at once, and going back returns to the page still scrolled where it was, which opening the conversation on its own route would lose. The navigation is assembled server-side. Hierarchy is in the page paths and order is on the knowledge base, so a client merging them would be a second place for the tree to be wrong — and it would have to page through documents to do it. A section that holds pages but has none of its own comes back as a node with no document, because the publish gate allows that and the reader has to render it: a heading that expands but cannot be opened. The outline is new; nothing in the repository did this. It skips fenced blocks, where `# ` is a comment rather than a heading, and gives repeated titles distinct ids, or every entry sharing a title would scroll to the first one. The active entry is the section being read rather than the one just scrolled past. The list is one request. The old one fanned out a generations call per project — twenty projects meant twenty requests and two hundred records — to find two fields per card, which now come from the knowledge base itself. Creating asks for a repository and a name, and nothing else: a code wiki belongs to the wiki account and its repository decides who may read it, so asking anyone to place it would be asking a question whose answer is ignored. The old wiki stack goes with it rather than being left for PR5b: once the code tab stopped using it, every binding it fed was dead, and lint was right to refuse them. Also adds a seed script. Everything after the agent — gate, projection, attachment storage, index queue, reader — can be exercised without a model, because the agent's only interface is the write API, and the script plays it. Co-Authored-By: Claude Opus 5 --- backend/app/api/endpoints/knowledge.py | 52 ++++- backend/app/schemas/knowledge.py | 22 ++ .../knowledge/code_wiki/navigation.py | 119 ++++++++++ backend/scripts/seed_code_wiki.py | 154 ++++++++++++ .../knowledge/code_wiki/test_navigation.py | 170 ++++++++++++++ frontend/src/apis/code-wiki.ts | 61 +++++ .../code-wiki/[knowledgeBaseId]/page.tsx | 79 +++++++ frontend/src/app/(tasks)/knowledge/page.tsx | 128 ++-------- .../code-wiki/CodeWikiCreateDialog.tsx | 133 +++++++++++ .../knowledge/code-wiki/CodeWikiList.tsx | 122 ++++++++++ .../knowledge/code-wiki/CodeWikiReader.tsx | 221 ++++++++++++++++++ .../knowledge/code-wiki/PageOutline.tsx | 132 +++++++++++ .../knowledge/code-wiki/WikiNavigation.tsx | 152 ++++++++++++ .../knowledge/code-wiki/WikiPageContent.tsx | 96 ++++++++ .../knowledge/code-wiki/useCodeWikis.ts | 49 ++++ frontend/src/i18n/locales/en/knowledge.json | 34 +++ .../src/i18n/locales/zh-CN/knowledge.json | 34 +++ frontend/src/types/code-wiki.ts | 77 ++++++ 18 files changed, 1727 insertions(+), 108 deletions(-) create mode 100644 backend/app/services/knowledge/code_wiki/navigation.py create mode 100644 backend/scripts/seed_code_wiki.py create mode 100644 backend/tests/services/knowledge/code_wiki/test_navigation.py create mode 100644 frontend/src/apis/code-wiki.ts create mode 100644 frontend/src/app/(tasks)/knowledge/code-wiki/[knowledgeBaseId]/page.tsx create mode 100644 frontend/src/features/knowledge/code-wiki/CodeWikiCreateDialog.tsx create mode 100644 frontend/src/features/knowledge/code-wiki/CodeWikiList.tsx create mode 100644 frontend/src/features/knowledge/code-wiki/CodeWikiReader.tsx create mode 100644 frontend/src/features/knowledge/code-wiki/PageOutline.tsx create mode 100644 frontend/src/features/knowledge/code-wiki/WikiNavigation.tsx create mode 100644 frontend/src/features/knowledge/code-wiki/WikiPageContent.tsx create mode 100644 frontend/src/features/knowledge/code-wiki/useCodeWikis.ts create mode 100644 frontend/src/types/code-wiki.ts diff --git a/backend/app/api/endpoints/knowledge.py b/backend/app/api/endpoints/knowledge.py index ee8e6f1a85..ae5dcebf10 100644 --- a/backend/app/api/endpoints/knowledge.py +++ b/backend/app/api/endpoints/knowledge.py @@ -47,6 +47,8 @@ CodeWikiCreate, CodeWikiListItem, CodeWikiListResponse, + CodeWikiPageNode, + CodeWikiPageTree, CodeWikiRunCreate, CodeWikiRunResponse, DocumentContentUpdate, @@ -80,11 +82,15 @@ knowledge_base_qa_service, ) from app.services.knowledge.code_wiki.generation import GenerationInFlight +from app.services.knowledge.code_wiki.navigation import page_tree from app.services.knowledge.code_wiki.publisher import ( PUBLISHED_AT_KEY, PUBLISHED_COMMIT_KEY, ) -from app.services.knowledge.code_wiki.read_access import readable_wiki_ids +from app.services.knowledge.code_wiki.read_access import ( + may_read_code_wiki, + readable_wiki_ids, +) from app.services.knowledge.code_wiki.registry import ( CODE_WIKI_NAMESPACE, CodeWikiOwnerMissing, @@ -705,6 +711,50 @@ def create_code_wiki( ) from e +@router.get("/{knowledge_base_id}/code-wiki/pages", response_model=CodeWikiPageTree) +@trace_sync("get_code_wiki_pages", "knowledge.api") +def get_code_wiki_pages( + knowledge_base_id: int, + current_user: User = Depends(security.get_current_user), + db: Session = Depends(get_db), +): + """The navigation for a code wiki: every published page, nested and ordered. + + Access is the repository's, not the knowledge base's — a code wiki belongs to the + wiki account, so its ACL grants nobody else anything. + """ + knowledge_base = _readable_code_wiki(db, current_user, knowledge_base_id) + return CodeWikiPageTree( + pages=[_as_page_node(node) for node in page_tree(db, knowledge_base)] + ) + + +def _as_page_node(node) -> CodeWikiPageNode: + return CodeWikiPageNode( + path=node.path, + title=node.title, + document_id=node.document_id, + has_content=node.has_content, + children=[_as_page_node(child) for child in node.children], + ) + + +def _readable_code_wiki(db: Session, user: User, knowledge_base_id: int) -> Kind: + """Load a code wiki the caller may read, or refuse.""" + knowledge_base = KnowledgeService._get_knowledge_base_record(db, knowledge_base_id) + if knowledge_base is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Code wiki not found" + ) + if not may_read_code_wiki(db, user, knowledge_base): + # Indistinguishable from missing on purpose: whether a repository has a wiki + # is itself something only its members should learn. + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Code wiki not found" + ) + return knowledge_base + + @router.post( "/{knowledge_base_id}/code-wiki/generations", response_model=CodeWikiRunResponse, diff --git a/backend/app/schemas/knowledge.py b/backend/app/schemas/knowledge.py index f8212938a1..d610d9bcf5 100644 --- a/backend/app/schemas/knowledge.py +++ b/backend/app/schemas/knowledge.py @@ -366,6 +366,28 @@ class CodeWikiListResponse(BaseModel): total: int +class CodeWikiPageNode(BaseModel): + """One node of the reader's navigation. + + Every node is a page. The hierarchy comes from the page paths and the order from + the knowledge base's recorded order, so the client renders what it is given + rather than reassembling a tree from a paginated document list and a separate + array — two things it would have to keep consistent itself. + """ + + path: str = Field(..., description="Stable page path; identity, not a label") + title: str = Field(..., description="What the page is called") + document_id: int = Field(0, description="0 for a section with no page of its own") + has_content: bool = True + children: List["CodeWikiPageNode"] = Field(default_factory=list) + + +class CodeWikiPageTree(BaseModel): + """The navigation for one code wiki.""" + + pages: List[CodeWikiPageNode] + + class CodeWikiRunCreate(BaseModel): """Request to regenerate a code wiki now. diff --git a/backend/app/services/knowledge/code_wiki/navigation.py b/backend/app/services/knowledge/code_wiki/navigation.py new file mode 100644 index 0000000000..ef2c566065 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/navigation.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Turning published pages into the tree a reader navigates. + +The hierarchy is in the paths and the order is on the knowledge base, so assembling +them belongs here rather than in the client: the client would have to fetch a +paginated document list and an order array and keep the two consistent while it +merged them, which is a second place for the tree to be wrong. + +A section that holds pages but has no page of its own becomes a node with no +document. The publish gate reports that as a warning rather than refusing the +version, so the reader has to render it — as a heading that cannot be opened. +""" + +import logging +from dataclasses import dataclass, field + +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import KnowledgeDocument +from app.services.knowledge.code_wiki.page_path import collation_key +from app.services.knowledge.code_wiki.projection_plan import PAGE_PATH_KEY +from app.services.knowledge.code_wiki.publisher import PAGE_ORDER_KEY +from app.services.knowledge.content_scope import generated_wiki_pages + +logger = logging.getLogger(__name__) + + +@dataclass +class PageNode: + """One entry in the navigation.""" + + path: str + title: str + document_id: int = 0 + children: list["PageNode"] = field(default_factory=list) + + @property + def has_content(self) -> bool: + return self.document_id != 0 + + +def page_tree(db: Session, knowledge_base: Kind) -> list[PageNode]: + """The published pages of ``knowledge_base``, nested and ordered.""" + documents = generated_wiki_pages( + db.query(KnowledgeDocument).filter( + KnowledgeDocument.kind_id == knowledge_base.id + ) + ).all() + + by_key: dict[str, PageNode] = {} + for document in documents: + path = (document.source_config or {}).get(PAGE_PATH_KEY) + if not path: + logger.warning( + "[code_wiki] document %s has no page path; not navigable", document.id + ) + continue + by_key[collation_key(path)] = PageNode( + path=path, title=document.name, document_id=document.id + ) + + _add_missing_sections(by_key) + return _nest(by_key, _declared_order(knowledge_base)) + + +def _add_missing_sections(by_key: dict[str, PageNode]) -> None: + """Invent a node for a section that holds pages but is not one itself. + + Without it the pages under that section would have nowhere to hang and would + surface at the top level, which reads as though they were unrelated. + """ + for key in list(by_key): + path = by_key[key].path + while "/" in path: + path = path.rsplit("/", 1)[0] + section_key = collation_key(path) + if section_key in by_key: + break + by_key[section_key] = PageNode( + path=path, title=path.rsplit("/", 1)[-1], document_id=0 + ) + + +def _declared_order(knowledge_base: Kind) -> dict[str, int]: + spec = (knowledge_base.json or {}).get("spec", {}) + declared = spec.get(PAGE_ORDER_KEY) or [] + return {collation_key(str(path)): index for index, path in enumerate(declared)} + + +def _nest(by_key: dict[str, PageNode], order: dict[str, int]) -> list[PageNode]: + """Attach each node to its parent, and sort every level the same way.""" + roots: list[PageNode] = [] + for key, node in by_key.items(): + parent_key = ( + collation_key(node.path.rsplit("/", 1)[0]) if "/" in node.path else None + ) + parent = by_key.get(parent_key) if parent_key else None + if parent is None: + roots.append(node) + else: + parent.children.append(node) + + def rank(node: PageNode) -> tuple[int, str]: + # Anything the agent did not rank sorts after what it did, then by path so + # the result is at least stable rather than dependent on row order. + key = collation_key(node.path) + return (order.get(key, len(order)), key) + + def sort(nodes: list[PageNode]) -> list[PageNode]: + nodes.sort(key=rank) + for node in nodes: + sort(node.children) + return nodes + + return sort(roots) diff --git a/backend/scripts/seed_code_wiki.py b/backend/scripts/seed_code_wiki.py new file mode 100644 index 0000000000..f8981233c9 --- /dev/null +++ b/backend/scripts/seed_code_wiki.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Fill a code wiki with pages without running an agent. + +Everything after the agent — the publish gate, the projection, attachment storage, +the index queue and the reader — can be exercised without a model or an executor, +because the agent's only interface is the write API. This plays that interface. + +What it does *not* cover: choosing a run mode, cloning the repository, and whether +the model produces anything worth reading. Those need the real thing. + + uv run python scripts/seed_code_wiki.py --kb-id 12 + uv run python scripts/seed_code_wiki.py --kb-id 12 --pages index architecture \\ + architecture/backend +""" + +import argparse +import sys +from datetime import datetime + +from app.db.session import SessionLocal +from app.models.kind import Kind +from app.models.user import User +from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.schemas.wiki import ( + WikiContentSection, + WikiContentSummary, + WikiContentWriteRequest, +) +from app.services.knowledge.code_wiki.generation import start_generation +from app.services.knowledge.code_wiki.registry import wiki_owner +from app.services.wiki_service import WikiService + +DEFAULT_PAGES = ["index", "architecture", "architecture/backend", "guides/setup"] + + +def _body(path: str) -> str: + """Content with a heading tree and a diagram, so the outline and Mermaid show.""" + return f"""# {path} + +Seeded page for `{path}`. + +## What this covers + +Placeholder prose so the outline on the right has something to list. + +## How it fits + +```mermaid +flowchart TD + A[Request] --> B[{path}] + B --> C[(Storage)] +``` + +### A deeper heading + +The outline folds anything below level three, so this is the last level shown. +""" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kb-id", type=int, required=True, help="Code wiki to fill") + parser.add_argument( + "--pages", nargs="*", default=DEFAULT_PAGES, help="Page paths to write" + ) + parser.add_argument( + "--commit", + default="seed" + datetime.now().strftime("%H%M%S"), + help="Commit the seeded version claims to document", + ) + args = parser.parse_args() + + with SessionLocal() as db: + knowledge_base = db.get(Kind, args.kb_id) + if knowledge_base is None or knowledge_base.kind != "KnowledgeBase": + print(f"No knowledge base {args.kb_id}", file=sys.stderr) + return 1 + + requester = db.get(User, knowledge_base.user_id) + if requester is None: + print(f"Knowledge base {args.kb_id} has no owner", file=sys.stderr) + return 1 + owner: User = wiki_owner(db, requester) + started = start_generation( + db, + knowledge_base=knowledge_base, + user=owner, + head_commit=args.commit, + ) + if not started.started: + print(f"No run needed: {started.decision.reason}", file=sys.stderr) + return 1 + + generation: WikiGeneration = started.generation + db.commit() + print(f"generation {generation.id} ({started.decision.mode})") + + # Written through the same API the agent uses, so the path matching, the + # removal channel and the completion handoff are all the real ones. + service = WikiService() + service.save_generation_contents( + db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[ + WikiContentSection( + type="chapter", + title=path.rsplit("/", 1)[-1].replace("-", " ").title(), + content=_body(path), + path=path, + ) + for path in args.pages + ], + ), + ) + + service.save_generation_contents( + db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[], + summary=WikiContentSummary( + status="COMPLETED", + head_commit=args.commit, + structure_order=list(args.pages), + ), + ), + ) + + db.refresh(generation) + published = ( + (knowledge_base.json or {}).get("spec", {}).get("publishedGenerationId") + ) + pages = ( + db.query(WikiContent) + .filter(WikiContent.generation_id == generation.id) + .count() + ) + print( + f"status={generation.status} pages={pages} published={published}", + file=( + sys.stderr + if generation.status != WikiGenerationStatus.COMPLETED + else sys.stdout + ), + ) + return 0 if published == generation.id else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/services/knowledge/code_wiki/test_navigation.py b/backend/tests/services/knowledge/code_wiki/test_navigation.py new file mode 100644 index 0000000000..2103be10b8 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_navigation.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the navigation a reader is given. + +The hierarchy lives in the paths and the order on the knowledge base, and the client +renders what it is handed rather than merging the two itself. So what is pinned here +is that the two arrive already reconciled — including the case neither of them +describes on its own: a section that holds pages but is not one. +""" + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import ContentOrigin, KnowledgeDocument +from app.services.knowledge.code_wiki.navigation import page_tree +from app.services.knowledge.code_wiki.projection_plan import PAGE_PATH_KEY +from app.services.knowledge.code_wiki.publisher import PAGE_ORDER_KEY + +KIND_ID = 771 + + +@pytest.fixture +def knowledge_base(test_db: Session) -> Kind: + kind = Kind( + id=KIND_ID, + kind="KnowledgeBase", + name="kb-nav", + namespace="default", + user_id=1, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def _page(test_db: Session, path: str, title: str) -> KnowledgeDocument: + document = KnowledgeDocument( + kind_id=KIND_ID, + attachment_id=1, + name=title, + file_extension="md", + file_size=1, + user_id=1, + folder_id=0, + origin=ContentOrigin.GENERATED.value, + source_config={PAGE_PATH_KEY: path}, + ) + test_db.add(document) + test_db.flush() + return document + + +def _order(test_db: Session, knowledge_base: Kind, paths: list[str]) -> None: + payload = dict(knowledge_base.json or {}) + spec = dict(payload.get("spec", {})) + spec[PAGE_ORDER_KEY] = paths + payload["spec"] = spec + knowledge_base.json = payload + test_db.flush() + + +def _shape(nodes) -> list: + return [(node.path, _shape(node.children)) for node in nodes] + + +def test_a_page_nests_under_the_page_whose_path_prefixes_it( + test_db: Session, knowledge_base: Kind +): + _page(test_db, "architecture", "Architecture") + _page(test_db, "architecture/backend", "Backend") + + assert _shape(page_tree(test_db, knowledge_base)) == [ + ("architecture", [("architecture/backend", [])]) + ] + + +def test_a_flat_wiki_needs_no_special_case(test_db: Session, knowledge_base: Kind): + """A simple repository has one level, and that has to work as it stands.""" + _page(test_db, "index", "Overview") + _page(test_db, "setup", "Setup") + _order(test_db, knowledge_base, ["index", "setup"]) + + assert _shape(page_tree(test_db, knowledge_base)) == [("index", []), ("setup", [])] + + +def test_the_declared_order_is_what_the_reader_gets( + test_db: Session, knowledge_base: Kind +): + """Alphabetically the API reference precedes the overview, and a wiki read in + that order reads wrong.""" + for path in ("api", "index", "architecture"): + _page(test_db, path, path) + _order(test_db, knowledge_base, ["index", "architecture", "api"]) + + assert [node.path for node in page_tree(test_db, knowledge_base)] == [ + "index", + "architecture", + "api", + ] + + +def test_children_are_ordered_the_same_way_as_the_top_level( + test_db: Session, knowledge_base: Kind +): + _page(test_db, "guide", "Guide") + _page(test_db, "guide/second", "Second") + _page(test_db, "guide/first", "First") + _order(test_db, knowledge_base, ["guide", "guide/first", "guide/second"]) + + (guide,) = page_tree(test_db, knowledge_base) + assert [child.path for child in guide.children] == ["guide/first", "guide/second"] + + +def test_a_section_with_no_page_of_its_own_still_holds_its_pages( + test_db: Session, knowledge_base: Kind +): + """The publish gate allows this, so the reader has to render it. Without the + node its pages would surface at the top level, reading as though unrelated.""" + _page(test_db, "architecture/backend", "Backend") + + tree = page_tree(test_db, knowledge_base) + + assert _shape(tree) == [("architecture", [("architecture/backend", [])])] + assert tree[0].has_content is False + assert tree[0].children[0].has_content is True + + +def test_a_missing_section_several_levels_up_is_filled_in( + test_db: Session, knowledge_base: Kind +): + _page(test_db, "a/b/c", "Deep") + + assert _shape(page_tree(test_db, knowledge_base)) == [ + ("a", [("a/b", [("a/b/c", [])])]) + ] + + +def test_an_unranked_page_follows_the_ranked_ones( + test_db: Session, knowledge_base: Kind +): + """A page added without updating the order must still appear, somewhere + predictable rather than at a position nobody chose.""" + _page(test_db, "index", "Overview") + _page(test_db, "stray", "Stray") + _order(test_db, knowledge_base, ["index"]) + + assert [node.path for node in page_tree(test_db, knowledge_base)] == [ + "index", + "stray", + ] + + +def test_a_document_without_a_page_path_is_not_navigable( + test_db: Session, knowledge_base: Kind +): + """It has no place in a tree built from paths, and guessing one would put it + somewhere the next publish disagrees with.""" + document = _page(test_db, "index", "Overview") + stray = _page(test_db, "other", "Other") + stray.source_config = {} + test_db.flush() + + assert [node.document_id for node in page_tree(test_db, knowledge_base)] == [ + document.id + ] diff --git a/frontend/src/apis/code-wiki.ts b/frontend/src/apis/code-wiki.ts new file mode 100644 index 0000000000..f594d8c885 --- /dev/null +++ b/frontend/src/apis/code-wiki.ts @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +import type { + CodeWikiCreateRequest, + CodeWikiListResponse, + CodeWikiPageTree, + CodeWikiRunResponse, + CodeWikiSummary, +} from '@/types/code-wiki' +import client from './client' + +export const codeWikiApi = { + /** + * Code wikis whose repositories the caller can read. + * + * Separate from the knowledge base list on purpose: a code wiki belongs to the wiki + * account, so it appears in none of that list's scopes. + */ + list: async (params?: { page?: number; limit?: number }): Promise => { + const query = new URLSearchParams() + if (params?.page) query.append('page', String(params.page)) + if (params?.limit) query.append('limit', String(params.limit)) + const suffix = query.toString() + return client.get( + `/knowledge-bases/code-wikis${suffix ? `?${suffix}` : ''}` + ) + }, + + /** + * Bind a repository and create its wiki, or return the one it already has. + * + * One repository has one wiki, so this answers 200 with the existing one rather + * than refusing — the caller wanted that repository's wiki, not the act of + * creating it. + */ + create: async (data: CodeWikiCreateRequest): Promise => + client.post('/knowledge-bases/code-wikis', data), + + /** + * The navigation: every published page, already nested and ordered. + * + * Assembled server-side because the hierarchy is in the page paths and the order + * is on the knowledge base — merging them here would be a second place for the + * tree to be wrong. + */ + pages: async (knowledgeBaseId: number): Promise => + client.get(`/knowledge-bases/${knowledgeBaseId}/code-wiki/pages`), + + /** + * Regenerate now, without waiting for a schedule. + * + * Answers 202 even when nothing was needed; read `started` to tell which happened. + */ + regenerate: async (knowledgeBaseId: number): Promise => + client.post( + `/knowledge-bases/${knowledgeBaseId}/code-wiki/generations`, + {} + ), +} diff --git a/frontend/src/app/(tasks)/knowledge/code-wiki/[knowledgeBaseId]/page.tsx b/frontend/src/app/(tasks)/knowledge/code-wiki/[knowledgeBaseId]/page.tsx new file mode 100644 index 0000000000..556ce229d2 --- /dev/null +++ b/frontend/src/app/(tasks)/knowledge/code-wiki/[knowledgeBaseId]/page.tsx @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useEffect, useState } from 'react' +import { useParams, useRouter } from 'next/navigation' +import { ArrowLeft } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Spinner } from '@/components/ui/spinner' +import TopNavigation from '@/features/layout/TopNavigation' +import UserMenu from '@/features/layout/UserMenu' +import { GithubStarButton } from '@/features/layout/GithubStarButton' +import { CodeWikiReader } from '@/features/knowledge/code-wiki/CodeWikiReader' +import { codeWikiApi } from '@/apis/code-wiki' +import type { CodeWikiSummary } from '@/types/code-wiki' +import '@/app/tasks/tasks.css' +import '@/features/common/scrollbar.css' + +export default function CodeWikiPage() { + const params = useParams() + const router = useRouter() + const knowledgeBaseId = Number(params.knowledgeBaseId) + + const [wiki, setWiki] = useState(null) + const [loading, setLoading] = useState(true) + + useEffect(() => { + // The list is the only place a wiki's summary comes from, and it is the same + // request that decides whether this reader may show anything at all. + let cancelled = false + codeWikiApi + .list() + .then(response => { + if (cancelled) return + setWiki(response.items.find(item => item.id === knowledgeBaseId) ?? null) + }) + .finally(() => { + if (!cancelled) setLoading(false) + }) + return () => { + cancelled = true + } + }, [knowledgeBaseId]) + + return ( +
+ + + + + +
+
+ +
+ + {loading ? ( +
+ +
+ ) : wiki ? ( + + ) : ( +

404

+ )} +
+
+ ) +} diff --git a/frontend/src/app/(tasks)/knowledge/page.tsx b/frontend/src/app/(tasks)/knowledge/page.tsx index 1326493f2c..fae280eafb 100644 --- a/frontend/src/app/(tasks)/knowledge/page.tsx +++ b/frontend/src/app/(tasks)/knowledge/page.tsx @@ -21,12 +21,13 @@ import { GithubStarButton } from '@/features/layout/GithubStarButton' import { ThemeToggle } from '@/features/theme/ThemeToggle' import { useTranslation } from '@/hooks/useTranslation' import { saveLastTab } from '@/utils/userPreferences' -import { useUser } from '@/features/common/UserContext' import { useIsMobile } from '@/features/layout/hooks/useMediaQuery' import { useTaskSession } from '@/features/tasks/session/TaskSession' import { paths } from '@/config/paths' import { Spinner } from '@/components/ui/spinner' -import { useWikiProjects } from '@/features/knowledge/useWikiProjects' +import { CodeWikiCreateDialog } from '@/features/knowledge/code-wiki/CodeWikiCreateDialog' +import { CodeWikiList } from '@/features/knowledge/code-wiki/CodeWikiList' +import { useCodeWikis } from '@/features/knowledge/code-wiki/useCodeWikis' import { SearchBox } from '@/features/knowledge/SearchBox' import { KnowledgeTabs } from '@/features/knowledge/KnowledgeTabs' import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' @@ -35,18 +36,6 @@ import type { KnowledgeTabType } from '@/features/knowledge/KnowledgeTabs' import type { KnowledgeViewState } from '@/features/knowledge/document/components/KnowledgeDocumentPage' import { useKnowledgeTaskSidebar } from '@/features/knowledge/document/hooks/useKnowledgeTaskSidebar' -const WikiProjectList = dynamic(() => import('@/features/knowledge/WikiProjectList'), { - ssr: false, -}) - -const AddRepoModal = dynamic(() => import('@/features/knowledge/AddRepoModal'), { - ssr: false, -}) - -const CancelConfirmDialog = dynamic(() => import('@/features/knowledge/CancelConfirmDialog'), { - ssr: false, -}) - const KnowledgeDocumentPage = dynamic( () => import('@/features/knowledge/document/components/KnowledgeDocumentPage').then(mod => ({ @@ -63,7 +52,6 @@ function KnowledgePageContent() { const { t } = useTranslation() const router = useRouter() const searchParams = useSearchParams() - const { user } = useUser() const { selectTask } = useTaskSession() const isMobile = useIsMobile() const [knowledgeViewState, setKnowledgeViewState] = useState({ @@ -78,32 +66,10 @@ function KnowledgePageContent() { return 'document' // default }, [searchParams]) - // Use shared Hook to manage all state and logic - const { - projects, - loading, - loadingMore, - error, - cancellingIds, - hasMore, - isModalOpen, - formErrors, - isSubmitting, - confirmDialogOpen, - selectedRepo, - // Wiki config state (system-level configuration) - wikiConfig, - loadProjects, - loadMoreProjects, - handleAddRepo, - handleCloseModal, - handleRepoChange, - handleSubmit, - handleCancelClick, - confirmCancelGeneration, - setConfirmDialogOpen, - setPendingCancelProjectId, - } = useWikiProjects() + // The code tab now lists code wikis. useWikiProjects still drives the legacy + // wiki modal below; both go in PR5b, when the old service path is retired. + const codeWikis = useCodeWikis() + const [createWikiOpen, setCreateWikiOpen] = useState(false) // Active knowledge tab - initialized from URL const [activeTab, setActiveTab] = useState(getInitialKnowledgeTab) @@ -167,14 +133,6 @@ function KnowledgePageContent() { [router] ) - const navigateToKnowledgeDetail = (projectId: number) => { - router.push(`/knowledge/project/${projectId}?from=code`) - } - - const navigateToTask = (taskId: number) => { - router.push(`/chat?taskId=${taskId}`) - } - const knowledgeViewSwitcher = activeTab === 'document' && knowledgeViewState.visible ? ( { - // Check if user has any generations for this project - return ( - project.generations && - project.generations.length > 0 && - (project.generations[0].status === 'RUNNING' || - project.generations[0].status === 'COMPLETED' || - project.generations[0].status === 'PENDING' || - project.generations[0].status === 'FAILED' || - project.generations[0].status === 'CANCELLED') - ) - }) - useEffect(() => { saveLastTab('wiki') }, []) - useEffect(() => { - if (!user) return - loadProjects() - }, [user, loadProjects]) - // Handle new task from collapsed sidebar button const handleNewTask = () => { // IMPORTANT: Clear selected task FIRST to ensure UI state is reset immediately @@ -309,21 +247,13 @@ function KnowledgePageContent() { size="md" className="mb-6 max-w-2xl mx-auto" /> - {/* Project list */} - setCreateWikiOpen(true)} + onOpen={wiki => router.push(`/knowledge/code-wiki/${wiki.id}`)} /> )} @@ -336,30 +266,14 @@ function KnowledgePageContent() { )} - {/* Add repository modal */} - {isModalOpen && ( - - )} - {/* Cancel confirm dialog */} - {confirmDialogOpen && ( - { - setConfirmDialogOpen(false) - setPendingCancelProjectId(null) - }} - onConfirm={confirmCancelGeneration} - /> - )} + { + codeWikis.add(wiki) + router.push(`/knowledge/code-wiki/${wiki.id}`) + }} + /> ) } diff --git a/frontend/src/features/knowledge/code-wiki/CodeWikiCreateDialog.tsx b/frontend/src/features/knowledge/code-wiki/CodeWikiCreateDialog.tsx new file mode 100644 index 0000000000..0bd8b8d191 --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/CodeWikiCreateDialog.tsx @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useCallback, useState } from 'react' +import { toast } from 'sonner' +import { Button } from '@/components/ui/button' +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { RepositorySelector } from '@/features/tasks/components/selector' +import { useTranslation } from '@/hooks/useTranslation' +import { codeWikiApi } from '@/apis/code-wiki' +import type { GitRepoInfo } from '@/types/api' +import type { CodeWikiSummary } from '@/types/code-wiki' + +interface CodeWikiCreateDialogProps { + open: boolean + onOpenChange: (open: boolean) => void + onCreated: (wiki: CodeWikiSummary) => void +} + +/** + * Three fields, and everything else defaults. + * + * No namespace: a code wiki belongs to the wiki account and who may read it is + * decided by its repository, so asking anyone to place it would be asking a + * question whose answer is ignored. + */ +export function CodeWikiCreateDialog({ open, onOpenChange, onCreated }: CodeWikiCreateDialogProps) { + const { t } = useTranslation() + const [repo, setRepo] = useState(null) + const [name, setName] = useState('') + const [submitting, setSubmitting] = useState(false) + + const handleRepoChange = useCallback((next: GitRepoInfo | null) => { + setRepo(next) + // Derived, not forced: `wecode-ai/Wegent` becomes `Wegent`, and anyone who + // wants something else can still type it. + setName(current => current || next?.git_repo?.split('/').pop() || '') + }, []) + + const reset = useCallback(() => { + setRepo(null) + setName('') + }, []) + + const handleSubmit = useCallback(async () => { + if (!repo) return + setSubmitting(true) + try { + const sourceType = repo.type === 'gitee' ? 'gitea' : repo.type + const wiki = await codeWikiApi.create({ + name: name.trim() || repo.git_repo.split('/').pop() || repo.git_repo, + source_type: sourceType, + source_url: repo.git_url, + }) + onCreated(wiki) + onOpenChange(false) + reset() + } catch (error) { + toast.error(error instanceof Error ? error.message : String(error)) + } finally { + setSubmitting(false) + } + }, [repo, name, onCreated, onOpenChange, reset]) + + return ( + + + + {t('knowledge:codeWiki.create.title')} + + +
+
+ +
+ +
+
+ +
+ + setName(event.target.value)} + placeholder={t('knowledge:codeWiki.create.namePlaceholder')} + disabled={submitting} + data-testid="code-wiki-create-name" + /> +
+
+ + + + + + + + ) +} diff --git a/frontend/src/features/knowledge/code-wiki/CodeWikiList.tsx b/frontend/src/features/knowledge/code-wiki/CodeWikiList.tsx new file mode 100644 index 0000000000..05717954c8 --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/CodeWikiList.tsx @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useMemo } from 'react' +import { BookOpen, GitBranch, Plus } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Spinner } from '@/components/ui/spinner' +import { useTranslation } from '@/hooks/useTranslation' +import type { CodeWikiSummary } from '@/types/code-wiki' + +interface CodeWikiListProps { + wikis: CodeWikiSummary[] + loading: boolean + error: string | null + searchTerm: string + onCreate: () => void + onOpen: (wiki: CodeWikiSummary) => void +} + +const formatWhen = (value?: string | null): string => { + if (!value) return '' + const parsed = new Date(value) + return Number.isNaN(parsed.getTime()) ? '' : parsed.toLocaleDateString() +} + +export function CodeWikiList({ + wikis, + loading, + error, + searchTerm, + onCreate, + onOpen, +}: CodeWikiListProps) { + const { t } = useTranslation() + + const visible = useMemo(() => { + const needle = searchTerm.trim().toLowerCase() + if (!needle) return wikis + return wikis.filter( + wiki => + wiki.name.toLowerCase().includes(needle) || wiki.project_name.toLowerCase().includes(needle) + ) + }, [wikis, searchTerm]) + + if (loading) { + return ( +
+ +
+ ) + } + + if (error) { + return

{error}

+ } + + return ( +
+
+ +
+ + {visible.length === 0 ? ( +
+ +

{t('knowledge:codeWiki.list.empty')}

+

+ {t('knowledge:codeWiki.list.emptyHint')} +

+
+ ) : ( +
+ {visible.map(wiki => ( + + ))} +
+ )} +
+ ) +} diff --git a/frontend/src/features/knowledge/code-wiki/CodeWikiReader.tsx b/frontend/src/features/knowledge/code-wiki/CodeWikiReader.tsx new file mode 100644 index 0000000000..52fcf8ade9 --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/CodeWikiReader.tsx @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useCallback, useEffect, useMemo, useState } from 'react' +import { ArrowLeft, RefreshCw } from 'lucide-react' +import { toast } from 'sonner' +import { Button } from '@/components/ui/button' +import { Spinner } from '@/components/ui/spinner' +import { ChatArea } from '@/features/tasks/components/chat' +import { useTeamContext } from '@/contexts/TeamContext' +import { useTranslation } from '@/hooks/useTranslation' +import { codeWikiApi } from '@/apis/code-wiki' +import type { CodeWikiPageNode, CodeWikiSummary } from '@/types/code-wiki' +import { PageOutline } from './PageOutline' +import { WikiNavigation } from './WikiNavigation' +import { WikiPageContent } from './WikiPageContent' + +interface CodeWikiReaderProps { + wiki: CodeWikiSummary +} + +/** Depth-first, so "the first page" means the first one the reader would see. */ +const firstReadable = (nodes: CodeWikiPageNode[]): CodeWikiPageNode | null => { + for (const node of nodes) { + if (node.has_content) return node + const child = firstReadable(node.children) + if (child) return child + } + return null +} + +const findByPath = (nodes: CodeWikiPageNode[], path: string): CodeWikiPageNode | null => { + for (const node of nodes) { + if (node.path === path) return node + const found = findByPath(node.children, path) + if (found) return found + } + return null +} + +/** + * Three regions: the wiki's structure, the page, and the page's own outline. + * + * The middle switches between reading and a conversation rather than splitting. + * They never need to be visible at once — asking something replaces the page with + * the exchange, and going back returns to the page still scrolled where it was, + * which is what opening the conversation on its own route would lose. + */ +export function CodeWikiReader({ wiki }: CodeWikiReaderProps) { + const { t } = useTranslation() + const { teams, isTeamsLoading, refreshTeams } = useTeamContext() + + const [pages, setPages] = useState([]) + const [loading, setLoading] = useState(true) + const [activePath, setActivePath] = useState('') + const [markdown, setMarkdown] = useState('') + const [mode, setMode] = useState<'read' | 'chat'>('read') + const [regenerating, setRegenerating] = useState(false) + const [scrollHost, setScrollHost] = useState(null) + + useEffect(() => { + let cancelled = false + setLoading(true) + codeWikiApi + .pages(wiki.id) + .then(response => { + if (cancelled) return + setPages(response.pages) + const first = firstReadable(response.pages) + if (first) setActivePath(first.path) + }) + .catch(error => { + if (!cancelled) toast.error(error instanceof Error ? error.message : String(error)) + }) + .finally(() => { + if (!cancelled) setLoading(false) + }) + return () => { + cancelled = true + } + }, [wiki.id]) + + const activePage = useMemo( + () => (activePath ? findByPath(pages, activePath) : null), + [pages, activePath] + ) + + const handleRegenerate = useCallback(async () => { + setRegenerating(true) + try { + const result = await codeWikiApi.regenerate(wiki.id) + // "Nothing to do" is the answer the caller asked for, not a failure: the + // repository has not moved since the published version. + toast.success( + result.started + ? t('knowledge:codeWiki.reader.started') + : t('knowledge:codeWiki.reader.upToDate') + ) + } catch (error) { + toast.error(error instanceof Error ? error.message : String(error)) + } finally { + setRegenerating(false) + } + }, [wiki.id, t]) + + const knowledgeTeams = useMemo( + () => teams.filter(team => team.bind_mode?.includes('chat') ?? true), + [teams] + ) + + if (loading) { + return ( +
+ +
+ ) + } + + if (pages.length === 0) { + return ( +
+

{t('knowledge:codeWiki.reader.empty')}

+

{t('knowledge:codeWiki.reader.emptyHint')}

+ +
+ ) + } + + return ( +
+
+ { + setActivePath(node.path) + setMode('read') + }} + /> +
+ +
+
+ {mode === 'chat' && ( + + )} + + {activePage?.title ?? wiki.name} + + +
+ + {mode === 'read' ? ( +
+
+ +
+ +
+ ) : ( +
+ +
+ )} +
+ + {mode === 'read' && } +
+ ) +} diff --git a/frontend/src/features/knowledge/code-wiki/PageOutline.tsx b/frontend/src/features/knowledge/code-wiki/PageOutline.tsx new file mode 100644 index 0000000000..b2f93cfaa3 --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/PageOutline.tsx @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useEffect, useMemo, useRef, useState } from 'react' +import { useTranslation } from '@/hooks/useTranslation' + +interface Heading { + id: string + text: string + /** 2 or 3; deeper levels are folded into 3 so the rail stays narrow. */ + level: number +} + +interface PageOutlineProps { + /** Raw markdown of the page being read. */ + content: string + /** The element headings are rendered into, watched for scroll position. */ + scrollContainer: HTMLElement | null +} + +/** Fenced blocks are skipped: `# ` inside one is a comment, not a heading. */ +const collectHeadings = (markdown: string): Heading[] => { + const headings: Heading[] = [] + const seen = new Map() + let inFence = false + + for (const line of markdown.split('\n')) { + if (line.trimStart().startsWith('```')) { + inFence = !inFence + continue + } + if (inFence) continue + + const match = /^(#{2,6})\s+(.+?)\s*$/.exec(line) + if (!match) continue + + const text = match[2].replace(/[*_`]/g, '').trim() + if (!text) continue + + // Two sections can share a title, and an id that repeats would scroll to the + // first one from every entry. + const base = text.toLowerCase().replace(/[^\w一-龥]+/g, '-') + const count = seen.get(base) ?? 0 + seen.set(base, count + 1) + + headings.push({ + id: count === 0 ? base : `${base}-${count}`, + text, + level: Math.min(match[1].length, 3), + }) + } + + return headings +} + +/** + * The current page's headings, as a rail beside the content. + * + * Separate from the wiki navigation on the left, which moves between pages: this + * moves within one. A generated page runs long enough that reaching a section by + * scrolling is the slow way to do it. + */ +export function PageOutline({ content, scrollContainer }: PageOutlineProps) { + const { t } = useTranslation() + const headings = useMemo(() => collectHeadings(content), [content]) + const [activeId, setActiveId] = useState('') + const observer = useRef(null) + + useEffect(() => { + if (!scrollContainer || headings.length === 0) return + + // rootMargin pulls the trigger line to the top quarter of the viewport, so the + // highlighted entry is the section being read rather than the one just scrolled + // past. + observer.current = new IntersectionObserver( + entries => { + const visible = entries + .filter(entry => entry.isIntersecting) + .sort((a, b) => a.boundingClientRect.top - b.boundingClientRect.top) + if (visible[0]) setActiveId(visible[0].target.id) + }, + { root: scrollContainer, rootMargin: '0px 0px -75% 0px', threshold: 0 } + ) + + for (const heading of headings) { + const element = scrollContainer.querySelector(`#${CSS.escape(heading.id)}`) + if (element) observer.current.observe(element) + } + + return () => observer.current?.disconnect() + }, [headings, scrollContainer]) + + if (headings.length === 0) return null + + const jumpTo = (id: string) => { + const element = scrollContainer?.querySelector(`#${CSS.escape(id)}`) + element?.scrollIntoView({ behavior: 'smooth', block: 'start' }) + } + + return ( + + ) +} diff --git a/frontend/src/features/knowledge/code-wiki/WikiNavigation.tsx b/frontend/src/features/knowledge/code-wiki/WikiNavigation.tsx new file mode 100644 index 0000000000..51bf152b27 --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/WikiNavigation.tsx @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useEffect, useState } from 'react' +import { ChevronDown, ChevronRight } from 'lucide-react' +import { useTranslation } from '@/hooks/useTranslation' +import type { CodeWikiPageNode } from '@/types/code-wiki' + +interface WikiNavigationProps { + pages: CodeWikiPageNode[] + activePath: string + onSelect: (node: CodeWikiPageNode) => void +} + +/** Every ancestor of the active page, so opening a deep page reveals where it sits. */ +const ancestorsOf = (path: string): string[] => { + const parts = path.split('/') + return parts.slice(0, -1).map((_, index) => parts.slice(0, index + 1).join('/')) +} + +interface NodeProps { + node: CodeWikiPageNode + depth: number + activePath: string + expanded: Set + onToggle: (path: string) => void + onSelect: (node: CodeWikiPageNode) => void +} + +function NavigationNode({ node, depth, activePath, expanded, onToggle, onSelect }: NodeProps) { + const { t } = useTranslation() + const hasChildren = node.children.length > 0 + const isOpen = expanded.has(node.path) + const isActive = node.path === activePath + + return ( +
  • +
    + {hasChildren ? ( + + ) : ( + + )} + + +
    + + {hasChildren && isOpen && ( +
      + {node.children.map(child => ( + + ))} +
    + )} +
    • + ) +} + +/** + * The wiki's own structure, derived server-side from page paths. + * + * A read-only relative of the knowledge base folder tree: no upload, rename or + * delete, because everything here is a projection the next publish rewrites. + */ +export function WikiNavigation({ pages, activePath, onSelect }: WikiNavigationProps) { + const [expanded, setExpanded] = useState>(new Set()) + + useEffect(() => { + // Follows the active page rather than replacing the set, so a section the + // reader opened by hand stays open when they move elsewhere. + if (!activePath) return + setExpanded(current => { + const next = new Set(current) + for (const ancestor of ancestorsOf(activePath)) next.add(ancestor) + return next + }) + }, [activePath]) + + const toggle = (path: string) => + setExpanded(current => { + const next = new Set(current) + if (next.has(path)) next.delete(path) + else next.add(path) + return next + }) + + return ( + + ) +} diff --git a/frontend/src/features/knowledge/code-wiki/WikiPageContent.tsx b/frontend/src/features/knowledge/code-wiki/WikiPageContent.tsx new file mode 100644 index 0000000000..0941c2adda --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/WikiPageContent.tsx @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useEffect, useState } from 'react' +import dynamic from 'next/dynamic' +import { Spinner } from '@/components/ui/spinner' +import { useTheme } from '@/features/theme/ThemeProvider' +import { useTranslation } from '@/hooks/useTranslation' +import { getDocumentContent } from '@/apis/knowledge' +import type { CodeWikiPageNode } from '@/types/code-wiki' + +const EnhancedMarkdown = dynamic(() => import('@/components/common/EnhancedMarkdown'), { + ssr: false, +}) + +interface WikiPageContentProps { + page: CodeWikiPageNode | null + onContentChange: (markdown: string) => void +} + +/** + * One page's body. + * + * Rendered through `EnhancedMarkdown`, the same component the knowledge base + * document viewer uses — which already handles Mermaid with a fallback to the raw + * source when a diagram will not render, and resolves wiki links. The old wiki + * reader had its own parallel implementation of all of it, including a Mermaid + * component with no error handling at all. + */ +export function WikiPageContent({ page, onContentChange }: WikiPageContentProps) { + const { t } = useTranslation() + const { theme } = useTheme() + const [markdown, setMarkdown] = useState('') + const [loading, setLoading] = useState(false) + + useEffect(() => { + if (!page?.document_id) { + setMarkdown('') + onContentChange('') + return + } + + let cancelled = false + setLoading(true) + getDocumentContent(page.document_id, 0, 1) + .then(response => { + if (cancelled) return + const body = response.content ?? '' + setMarkdown(body) + onContentChange(body) + }) + .catch(() => { + if (!cancelled) { + setMarkdown('') + onContentChange('') + } + }) + .finally(() => { + if (!cancelled) setLoading(false) + }) + + return () => { + cancelled = true + } + }, [page?.document_id, onContentChange]) + + if (!page) return null + + if (!page.has_content) { + return ( +

    + {t('knowledge:codeWiki.reader.sectionOnly')} +

    + ) + } + + if (loading) { + return ( +
    + +
    + ) + } + + return ( +
    + +
    + ) +} diff --git a/frontend/src/features/knowledge/code-wiki/useCodeWikis.ts b/frontend/src/features/knowledge/code-wiki/useCodeWikis.ts new file mode 100644 index 0000000000..a52d8195ff --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/useCodeWikis.ts @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useCallback, useEffect, useState } from 'react' +import { codeWikiApi } from '@/apis/code-wiki' +import type { CodeWikiSummary } from '@/types/code-wiki' + +/** + * The code wikis the signed-in user can read. + * + * One request. The list carries the repository, the last publish and its commit + * straight from the knowledge base, so nothing here fans out per wiki — which is + * what the old wiki list did, one generations call per project. + */ +export function useCodeWikis() { + const [wikis, setWikis] = useState([]) + const [loading, setLoading] = useState(true) + const [error, setError] = useState(null) + + const load = useCallback(async () => { + setLoading(true) + setError(null) + try { + const response = await codeWikiApi.list() + setWikis(response.items) + } catch (caught) { + setError(caught instanceof Error ? caught.message : String(caught)) + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + void load() + }, [load]) + + const add = useCallback((wiki: CodeWikiSummary) => { + // Creating a wiki for a repository that already has one returns the existing + // wiki, so this must not append a duplicate. + setWikis(current => + current.some(existing => existing.id === wiki.id) ? current : [wiki, ...current] + ) + }, []) + + return { wikis, loading, error, reload: load, add } +} diff --git a/frontend/src/i18n/locales/en/knowledge.json b/frontend/src/i18n/locales/en/knowledge.json index 06bf870f97..4f254f8ab9 100644 --- a/frontend/src/i18n/locales/en/knowledge.json +++ b/frontend/src/i18n/locales/en/knowledge.json @@ -1050,5 +1050,39 @@ "description": "You do not have permission to access this knowledge base. If you have any questions, please contact the knowledge base creator or administrator.", "backButton": "Back to Knowledge Base List" } + }, + "codeWiki": { + "outline": { + "title": "On this page", + "label": "Page outline" + }, + "list": { + "empty": "No code wikis yet", + "emptyHint": "Create one from a repository you can read.", + "neverGenerated": "Not generated yet", + "lastGenerated": "Updated {{when}}", + "atCommit": "at {{commit}}", + "pages_one": "{{count}} page", + "pages_other": "{{count}} pages" + }, + "create": { + "title": "New code wiki", + "repository": "Repository", + "name": "Name", + "namePlaceholder": "Defaults to the repository name", + "submit": "Create", + "noBoundModel": "The code wiki agent has no model bound, so generation cannot run." + }, + "reader": { + "regenerate": "Regenerate", + "regenerating": "Generating…", + "upToDate": "Already up to date with the repository", + "started": "Generation started", + "back": "Back to the page", + "askPlaceholder": "Ask about this wiki", + "sectionOnly": "This section has no page of its own", + "empty": "This wiki has no pages yet", + "emptyHint": "Generate it to see its documentation here." + } } } diff --git a/frontend/src/i18n/locales/zh-CN/knowledge.json b/frontend/src/i18n/locales/zh-CN/knowledge.json index 5bf97d688c..b9ce32c742 100644 --- a/frontend/src/i18n/locales/zh-CN/knowledge.json +++ b/frontend/src/i18n/locales/zh-CN/knowledge.json @@ -1050,5 +1050,39 @@ "description": "您没有该知识库详细信息的访问权限。如有疑问，请联系知识库创建者或管理员。", "backButton": "返回知识库列表" } + }, + "codeWiki": { + "outline": { + "title": "本页目录", + "label": "页面目录" + }, + "list": { + "empty": "还没有代码 Wiki", + "emptyHint": "从一个你有读权限的仓库创建。", + "neverGenerated": "尚未生成", + "lastGenerated": "更新于 {{when}}", + "atCommit": "commit {{commit}}", + "pages_one": "{{count}} 个页面", + "pages_other": "{{count}} 个页面" + }, + "create": { + "title": "新建代码 Wiki", + "repository": "仓库", + "name": "名称", + "namePlaceholder": "默认取仓库名", + "submit": "创建", + "noBoundModel": "代码 Wiki 智能体未绑定模型，无法生成。" + }, + "reader": { + "regenerate": "重新生成", + "regenerating": "生成中…", + "upToDate": "仓库自上次生成后没有变化", + "started": "已开始生成", + "back": "返回文档", + "askPlaceholder": "就这个 wiki 提问", + "sectionOnly": "该章节没有自己的页面", + "empty": "这个 wiki 还没有页面", + "emptyHint": "生成一次即可在此查看文档。" + } } } diff --git a/frontend/src/types/code-wiki.ts b/frontend/src/types/code-wiki.ts new file mode 100644 index 0000000000..61b6960d05 --- /dev/null +++ b/frontend/src/types/code-wiki.ts @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * A code wiki is a knowledge base an agent writes from a source repository. + * + * It is listed and read through its own endpoints rather than the general knowledge + * base ones: it belongs to the wiki account rather than to anyone who asks for it, so + * it matches none of that list's scopes, and who may read it is decided by who may + * read its repository. + */ +export interface CodeWikiSummary { + id: number + name: string + description?: string | null + /** Repository the wiki documents, e.g. `wecode-ai/Wegent`. */ + project_name: string + source_url: string + /** When the live version was published; null when nothing has been. */ + last_published_at?: string | null + /** Commit the live version documents. */ + last_published_commit: string + document_count: number + created_at: string + updated_at: string +} + +export interface CodeWikiListResponse { + items: CodeWikiSummary[] + total: number +} + +export interface CodeWikiCreateRequest { + name: string + description?: string + source_type: 'github' | 'gitlab' | 'gitea' + source_url: string +} + +/** + * What happened when a wiki was asked to regenerate. + * + * `started: false` is a success, not a failure: the repository had not changed since + * the published version, so there was nothing to build. + */ +export interface CodeWikiRunResponse { + started: boolean + /** `full`, `incremental` or `skip`. */ + mode: string + reason: string + generation_id: number + task_id: number +} + +/** + * One node of the reader's navigation. + * + * Every node is a page. The tree is derived from page paths, and a section that holds + * pages but has no page of its own renders as a heading that cannot be opened — + * allowed, and reported by the publish gate as a warning. + */ +export interface CodeWikiPageNode { + /** Stable path, e.g. `architecture/backend`. Identity, not a display value. */ + path: string + /** What the page is called; the document's name. */ + title: string + /** 0 for a section that holds pages but has none of its own. */ + document_id: number + /** False for such a section: a heading that cannot be opened. */ + has_content: boolean + children: CodeWikiPageNode[] +} + +export interface CodeWikiPageTree { + pages: CodeWikiPageNode[] +} From cbba7dc320bbfa70ec858d7333a32adfc3cdaa91 Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Tue, 4 Aug 2026 12:38:00 +0800 Subject: [PATCH 08/14] docs(knowledge): document the code wiki fields on a knowledge base The YAML reference described retrieval and summary configuration but not the fields this work added, so a knowledge base bound to a repository was not documented at all. Records them alongside the invariants that make them readable: which one decides the live version, what decides who may read, and why there is only ever one wiki per repository. Co-Authored-By: Claude Opus 5 --- .../en/wegent/reference/yaml-specification.md | 34 +++++++++++++++++++ .../zh/wegent/reference/yaml-specification.md | 26 ++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/docs/en/wegent/reference/yaml-specification.md b/docs/en/wegent/reference/yaml-specification.md index 6567f85038..9fcff4e8d9 100644 --- a/docs/en/wegent/reference/yaml-specification.md +++ b/docs/en/wegent/reference/yaml-specification.md @@ -452,6 +452,40 @@ KnowledgeBase is used to manage document knowledge bases, retrieval configuratio --- +### Code wiki fields + +A knowledge base with `spec.kbType = "code_wiki"` is generated by an agent from a +source repository. Every field below is written by the system and should not be +edited by hand. + +| Field | Type | Description | +| ---------------------------- | ------- | -------------------------------------------------------------------- | +| `spec.kbType` | string | `notebook`, `classic` or `code_wiki`. **Cannot move into or out of `code_wiki`** | +| `spec.source.sourceType` | string | `github`, `gitlab` or `gitea` | +| `spec.source.sourceUrl` | string | Repository URL, derived at creation with any credentials stripped | +| `spec.source.projectName` | string | `owner/repo` | +| `spec.source.sourceDomain` | string | Repository host | +| `spec.publishedGenerationId` | integer | **Sole authority for which version is live**; 0 when nothing is | +| `spec.lastPublishedAt` | string | When that version was published, written with the pointer | +| `spec.lastPublishedCommit` | string | Commit the live version documents | +| `spec.pageOrder` | array | Page paths in display order; hierarchy comes from the paths, order only from here | +| `spec.pendingIndexCleanup` | array | Vector store references owed for removed pages, retried at the next publish | + +**Invariants:** + +- **`publishedGenerationId` is the only authority.** Never infer the live version + from the newest completed generation: one can finish and still be refused by the + publish gate. +- **Ownership is fixed to the wiki account** (`WIKI_DEFAULT_USER_ID`) in the + `default` namespace. Who may read is decided by **repository access**, not by + knowledge base ACLs. +- **One repository has one wiki**, enforced by the UNIQUE on + `wiki_projects.source_url`. +- Generated pages carry `origin = generated`; the projection only adds and removes + those, so user content is never touched. + +--- + ## 🤝 Collaboration Collaboration defines the interaction patterns and workflows between Bots in a Team. diff --git a/docs/zh/wegent/reference/yaml-specification.md b/docs/zh/wegent/reference/yaml-specification.md index 45f0ea995a..ad32d411be 100644 --- a/docs/zh/wegent/reference/yaml-specification.md +++ b/docs/zh/wegent/reference/yaml-specification.md @@ -452,6 +452,32 @@ KnowledgeBase 用于管理文档知识库、检索配置和摘要能力。 --- +### 代码 Wiki 专属字段 + +`spec.kbType = "code_wiki"` 的知识库由智能体从一个源仓库生成，下列字段**全部由系统写入**，不应手工编辑。 + +| 字段 | 类型 | 说明 | +| --------------------------- | ------- | -------------------------------------------------------------------- | +| `spec.kbType` | string | `notebook`、`classic` 或 `code_wiki`。**创建后不能进出 `code_wiki`** | +| `spec.source.sourceType` | string | `github`、`gitlab` 或 `gitea` | +| `spec.source.sourceUrl` | string | 仓库地址，创建时从 URL 推导并剥除凭据 | +| `spec.source.projectName` | string | `owner/repo` | +| `spec.source.sourceDomain` | string | 仓库主机 | +| `spec.publishedGenerationId`| integer | **当前生效版本的唯一权威**，0 表示尚未发布 | +| `spec.lastPublishedAt` | string | 上次发布时间，与指针同事务写入 | +| `spec.lastPublishedCommit` | string | 生效版本所记录的 commit | +| `spec.pageOrder` | array | 页面路径的展示顺序；层级从路径推导，顺序只在这里 | +| `spec.pendingIndexCleanup` | array | 已删页面中尚未清理的向量库引用，下次发布时重试 | + +**几个不变量：** + +- **`publishedGenerationId` 是唯一权威** —— 不要从「最新完成的生成」推断生效版本：一次生成可以完成后被发布闸门拒绝 +- **归属固定在 wiki 账号**（`WIKI_DEFAULT_USER_ID`）、namespace 固定 `default` —— 谁能读由**仓库权限**决定，不由知识库 ACL 决定 +- **一个仓库一份 wiki**，由 `wiki_projects.source_url` 的唯一约束保证 +- 生成页面的 `origin` 为 `generated`；投影只增删这一类，用户内容永不受影响 + +--- + ## 🤝 Collaboration Collaboration 定义了团队中 Bot 之间的交互模式和工作流程。 From 651b1088bd17339fa6d3c12e8f5e6d62c7073530 Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Tue, 4 Aug 2026 16:30:15 +0800 Subject: [PATCH 09/14] fix(backend): decrypt stored git tokens in every repository provider Listing repositories returned an empty list for any user whose token was bound through the UI. The ciphertext was passed to the provider API as if it were the credential, which answers 401, and two layers of `continue` turned that into `200 []` -- indistinguishable from owning no repositories. The decryption used to happen once, when the session user was loaded: get_current_user called user_service.get_user_by_name, whose last line is `return self.decrypt_user_git_info(user)`. #2185 replaced those three call sites with a plain query so that authentication would not depend on Git crypto configuration -- correct in itself, and likely forced by #2110 making GIT_TOKEN_AES_IV mandatory two days earlier, since decrypting during login would otherwise reject every session on a deployment without the IV. But the decryption was removed rather than moved, and all five providers had been relying on it. Decrypt at the provider boundary instead, in each _get_git_infos, which is the single place entries are built. The placeholder '***' and empty strings are passed through untouched: the first marks a credential a deployment overlay substitutes at call time, the second is what callers test to raise "not configured". gitee.validate_token got the same treatment; it was the only provider that did not decrypt even there. Report the domains that drop out of an aggregated result, separating a refused credential from an unreachable host. The helper lives on the base class rather than in five copies -- five copies of _get_git_infos is how one omission became five. gerrit already logged this and is unchanged. Existing provider tests could not have caught any of it: their fixtures use plaintext tokens, which pass through decrypt_token untouched. The new tests encrypt, and are parametrized over all five providers so a sixth cannot be added with the omission intact. Co-Authored-By: Claude Opus 5 --- backend/app/repository/gerrit_provider.py | 2 +- backend/app/repository/gitea_provider.py | 8 +- backend/app/repository/gitee_provider.py | 17 +- backend/app/repository/github_provider.py | 10 +- backend/app/repository/gitlab_provider.py | 21 ++- .../interfaces/repository_provider.py | 23 +++ .../tests/repository/test_gitlab_provider.py | 174 ++++++++++++++++++ .../test_provider_token_resolution.py | 113 ++++++++++++ 8 files changed, 347 insertions(+), 21 deletions(-) create mode 100644 backend/tests/repository/test_gitlab_provider.py create mode 100644 backend/tests/repository/test_provider_token_resolution.py diff --git a/backend/app/repository/gerrit_provider.py b/backend/app/repository/gerrit_provider.py index 665938a798..668e334bc4 100644 --- a/backend/app/repository/gerrit_provider.py +++ b/backend/app/repository/gerrit_provider.py @@ -64,7 +64,7 @@ def _get_git_infos( entries.append( { "git_domain": info.get("git_domain", ""), - "git_token": info.get("git_token", ""), + "git_token": self.decrypt_token(info.get("git_token", "")), "user_name": info.get("user_name", ""), "type": info.get("type", ""), "auth_type": info.get("auth_type", "digest"), diff --git a/backend/app/repository/gitea_provider.py b/backend/app/repository/gitea_provider.py index fb29fce451..176dc716bb 100644 --- a/backend/app/repository/gitea_provider.py +++ b/backend/app/repository/gitea_provider.py @@ -66,7 +66,7 @@ def _get_git_infos( entries.append( { "git_domain": info.get("git_domain", ""), - "git_token": info.get("git_token", ""), + "git_token": self.decrypt_token(info.get("git_token", "")), "type": info.get("type", ""), "user_name": info.get("user_name", ""), } @@ -227,7 +227,8 @@ async def get_repositories( for repo in mapped_repos ] ) - except requests.exceptions.RequestException: + except requests.exceptions.RequestException as e: + self._log_domain_failure("list repositories", git_domain, e) continue return all_repos @@ -573,7 +574,8 @@ async def search_repositories( for r in filtered_repos ] ) - except requests.exceptions.RequestException: + except requests.exceptions.RequestException as e: + self._log_domain_failure("search repositories", git_domain, e) continue return all_results diff --git a/backend/app/repository/gitee_provider.py b/backend/app/repository/gitee_provider.py index 70b4e2d078..18f27022e9 100644 --- a/backend/app/repository/gitee_provider.py +++ b/backend/app/repository/gitee_provider.py @@ -61,7 +61,7 @@ def _get_git_infos( entries.append( { "git_domain": info.get("git_domain", ""), - "git_token": info.get("git_token", ""), + "git_token": self.decrypt_token(info.get("git_token", "")), "type": info.get("type", ""), } ) @@ -195,8 +195,8 @@ async def get_repositories( for repo in repos ] ) - except requests.exceptions.RequestException: - # skip failed domain, continue others + except requests.exceptions.RequestException as e: + self._log_domain_failure("list repositories", git_domain, e) continue return all_repos @@ -317,9 +317,14 @@ def validate_token(self, token: str, git_domain: str = None) -> Dict[str, Any]: # Use custom domain if provided, otherwise use default api_base_url = self._get_api_base_url(git_domain) + # Re-validating an already stored credential hands this the ciphertext, the + # same way the other providers are given it. Decrypting here keeps that case + # from reading as an invalid token. + decrypt_token = self.decrypt_token(token) + try: response = requests.get( - f"{api_base_url}/user", params={"access_token": token} + f"{api_base_url}/user", params={"access_token": decrypt_token} ) if response.status_code == 401: @@ -559,8 +564,8 @@ async def search_repositories( for r in filtered_repos ] ) - except requests.exceptions.RequestException: - # skip this domain on error + except requests.exceptions.RequestException as e: + self._log_domain_failure("search repositories", git_domain, e) continue return all_results diff --git a/backend/app/repository/github_provider.py b/backend/app/repository/github_provider.py index 8356ae06b5..8cbe7efeda 100644 --- a/backend/app/repository/github_provider.py +++ b/backend/app/repository/github_provider.py @@ -71,7 +71,7 @@ def _get_git_infos( entries.append( { "git_domain": info.get("git_domain", ""), - "git_token": info.get("git_token", ""), + "git_token": self.decrypt_token(info.get("git_token", "")), "type": info.get("type", ""), } ) @@ -204,8 +204,8 @@ async def get_repositories( for repo in repos ] ) - except requests.exceptions.RequestException: - # skip failed domain, continue others + except requests.exceptions.RequestException as e: + self._log_domain_failure("list repositories", git_domain, e) continue return all_repos @@ -580,8 +580,8 @@ async def search_repositories( for r in filtered_repos ] ) - except requests.exceptions.RequestException: - # skip this domain on error + except requests.exceptions.RequestException as e: + self._log_domain_failure("search repositories", git_domain, e) continue return all_results diff --git a/backend/app/repository/gitlab_provider.py b/backend/app/repository/gitlab_provider.py index c58e7cb3e2..5a0f440882 100644 --- a/backend/app/repository/gitlab_provider.py +++ b/backend/app/repository/gitlab_provider.py @@ -49,7 +49,11 @@ def _get_git_infos( git_domain: Optional domain to filter a specific GitLab entry Returns: - List of dictionaries containing git_domain, git_token, type + List of dictionaries containing git_domain, git_token, type. The token is + decrypted here, at the one place entries are built, so every caller below + holds something usable. Tokens are stored encrypted, and passing the + ciphertext to GitLab reads as an authentication failure rather than as a + configuration error. Raises: HTTPException: Raised when GitLab information is not configured @@ -65,7 +69,7 @@ def _get_git_infos( entries.append( { "git_domain": info.get("git_domain", ""), - "git_token": info.get("git_token", ""), + "git_token": self.decrypt_token(info.get("git_token", "")), "type": info.get("type", ""), } ) @@ -292,8 +296,11 @@ async def get_repositories( self._fetch_all_repositories_async(user, git_token, git_domain) ) - except requests.exceptions.RequestException: - # skip failed domain, continue others + except requests.exceptions.RequestException as e: + # Skip the failed domain and keep the others, but say so: silently + # returning fewer repositories is indistinguishable from the user + # owning none, which hides an expired or rejected token. + self._log_domain_failure("list repositories", git_domain, e) continue return all_repos @@ -632,8 +639,10 @@ async def search_repositories( for r in filtered_repos ] ) - except requests.exceptions.RequestException: - # skip this domain on error + except requests.exceptions.RequestException as e: + # Same reasoning as get_repositories: an empty search result and a + # rejected token must not look alike in the logs. + self._log_domain_failure("search repositories", git_domain, e) continue return all_results diff --git a/backend/app/repository/interfaces/repository_provider.py b/backend/app/repository/interfaces/repository_provider.py index 957eb34e65..825df21e0d 100644 --- a/backend/app/repository/interfaces/repository_provider.py +++ b/backend/app/repository/interfaces/repository_provider.py @@ -6,6 +6,7 @@ Repository provider interface, defining methods related to code repositories """ +import logging from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional @@ -69,6 +70,28 @@ def decrypt_token(self, token: str) -> str: def _is_token_encrypted(self, token: str) -> bool: return is_token_encrypted(token) + def _log_domain_failure( + self, action: str, git_domain: str, error: Exception + ) -> None: + """Report a domain that dropped out of an aggregated result. + + Providers gather repositories across every domain a user configured and skip + the ones that fail, so a rejected credential otherwise reaches the caller as + a short list rather than as an error. The status is worth separating out: + 401 means the token was refused, which is a different problem for the user + than the host being unreachable. + """ + response = getattr(error, "response", None) + status = getattr(response, "status_code", None) + logging.getLogger(type(self).__module__).warning( + "Could not %s from %s domain %s (status=%s): %s", + action, + getattr(self, "type", "git"), + git_domain or "unknown", + status if status is not None else "none", + error, + ) + @abstractmethod def validate_token(self, token: str) -> Dict[str, Any]: """ diff --git a/backend/tests/repository/test_gitlab_provider.py b/backend/tests/repository/test_gitlab_provider.py new file mode 100644 index 0000000000..475efa75ac --- /dev/null +++ b/backend/tests/repository/test_gitlab_provider.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for how GitLabProvider resolves the credential it calls GitLab with. + +Tokens are stored encrypted on the user, and nothing between the session user and +the provider decrypts them. So what is pinned here is that the provider does it +itself, at the single place entries are built, and that a domain whose credential +is refused says so rather than dropping out of the result in silence. +""" + +from unittest.mock import AsyncMock, Mock, patch + +import pytest +import requests + +from app.repository.gitlab_provider import GitLabProvider +from shared.utils.crypto import encrypt_git_token + +PLAIN_TOKEN = "glpat-000000000000000000" +DOMAIN = "gitlab.example.com" + + +@pytest.fixture(autouse=True) +def _crypto_env(monkeypatch): + monkeypatch.setenv("GIT_TOKEN_AES_KEY", "12345678901234567890123456789012") + monkeypatch.setenv("GIT_TOKEN_AES_IV", "1234567890123456") + # The module caches the key on first use, so a test that runs after one which + # left a different key would otherwise read the stale one. + import shared.utils.crypto as crypto + + monkeypatch.setattr(crypto, "_aes_key", None) + monkeypatch.setattr(crypto, "_aes_iv", None) + + +def _user(token: str) -> Mock: + user = Mock() + user.id = 1 + user.user_name = "testuser" + user.git_info = [{"type": "gitlab", "git_domain": DOMAIN, "git_token": token}] + return user + + +@pytest.fixture +def provider() -> GitLabProvider: + return GitLabProvider() + + +@pytest.fixture +def _no_cache(): + """Keep the result-caching side of get_repositories out of the way. + + ``set`` is awaited, so it has to be an async double; ``generate_full_cache_key`` + is not. + """ + cache = Mock(set=AsyncMock(), generate_full_cache_key=Mock(return_value="key")) + with patch("app.repository.gitlab_provider.cache_manager", cache): + yield cache + + +@pytest.mark.unit +class TestTokenResolution: + def test_a_stored_token_is_decrypted_before_it_is_used(self, provider): + """The ciphertext is not a credential. Sent as one it yields a 401, which + reads as a bad token rather than as the provider never decrypting it.""" + user = _user(encrypt_git_token(PLAIN_TOKEN)) + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == PLAIN_TOKEN + + def test_a_plaintext_token_is_left_alone(self, provider): + """Deployments exist that put a usable token straight into git_info.""" + user = _user(PLAIN_TOKEN) + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == PLAIN_TOKEN + + def test_the_placeholder_is_left_alone(self, provider): + """'***' means the token lives elsewhere and is substituted before the call. + Decrypting it would destroy the marker the substitution looks for.""" + user = _user("***") + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == "***" + + @pytest.mark.asyncio + async def test_the_decrypted_token_is_what_reaches_gitlab( + self, provider, _no_cache + ): + """Covers the whole path rather than the helper alone: a caller between + _get_git_infos and the request could still pass the raw entry through.""" + user = _user(encrypt_git_token(PLAIN_TOKEN)) + response = Mock(status_code=200) + response.json.return_value = [] + + with ( + patch.object( + provider, "_get_all_repositories_from_cache", return_value=None + ), + patch.object( + provider, "_make_request_with_auth_retry", return_value=response + ) as request, + ): + await provider.get_repositories(user, page=1, limit=10) + + assert request.call_args.kwargs["token"] == PLAIN_TOKEN + + +@pytest.mark.unit +class TestFailedDomainsAreReported: + @pytest.mark.asyncio + async def test_a_refused_domain_is_logged_rather_than_silently_skipped( + self, provider, caplog + ): + """An empty list and a rejected credential must not look alike. This is the + signal that turns 'this user has no repositories' back into a diagnosis.""" + user = _user(encrypt_git_token(PLAIN_TOKEN)) + refused = requests.exceptions.HTTPError("401 Unauthorized") + refused.response = Mock(status_code=401) + + with ( + patch.object( + provider, "_get_all_repositories_from_cache", return_value=None + ), + patch.object( + provider, "_make_request_with_auth_retry", side_effect=refused + ), + caplog.at_level("WARNING"), + ): + result = await provider.get_repositories(user, page=1, limit=10) + + assert result == [] + assert DOMAIN in caplog.text + assert "401" in caplog.text + + @pytest.mark.asyncio + async def test_one_bad_domain_does_not_lose_a_good_one(self, provider, _no_cache): + """Reporting the failure must not turn a partial result into no result.""" + user = _user(encrypt_git_token(PLAIN_TOKEN)) + user.git_info.append( + { + "type": "gitlab", + "git_domain": "gitlab.other.com", + "git_token": PLAIN_TOKEN, + } + ) + good = Mock(status_code=200) + good.json.return_value = [ + { + "id": 7, + "name": "repo", + "path_with_namespace": "group/repo", + "http_url_to_repo": "https://gitlab.other.com/group/repo.git", + "visibility": "private", + } + ] + refused = requests.exceptions.HTTPError("401 Unauthorized") + refused.response = Mock(status_code=401) + + with ( + patch.object( + provider, "_get_all_repositories_from_cache", return_value=None + ), + patch.object( + provider, "_make_request_with_auth_retry", side_effect=[refused, good] + ), + ): + result = await provider.get_repositories(user, page=1, limit=10) + + assert [repo["full_name"] for repo in result] == ["group/repo"] diff --git a/backend/tests/repository/test_provider_token_resolution.py b/backend/tests/repository/test_provider_token_resolution.py new file mode 100644 index 0000000000..1754960646 --- /dev/null +++ b/backend/tests/repository/test_provider_token_resolution.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Every provider must decrypt the stored credential before it calls out. + +Tokens are written to ``users.git_info`` encrypted, and nothing between the session +user and a provider decrypts them: ``get_current_user`` deliberately does not, so +that an unrelated endpoint never depends on Git crypto configuration. Each provider +therefore has to do it itself, and each one builds its entries in its own copy of +``_get_git_infos`` — which is exactly how the same omission ended up in all five. + +This is parametrized rather than written per provider so a sixth one cannot be added +with the omission intact. +""" + +import pytest + +from app.repository.gerrit_provider import GerritProvider +from app.repository.gitea_provider import GiteaProvider +from app.repository.gitee_provider import GiteeProvider +from app.repository.github_provider import GitHubProvider +from app.repository.gitlab_provider import GitLabProvider +from shared.utils.crypto import encrypt_git_token + +PLAIN_TOKEN = "glpat-000000000000000000" + +PROVIDERS = [ + pytest.param(GitHubProvider, "github", id="github"), + pytest.param(GitLabProvider, "gitlab", id="gitlab"), + pytest.param(GiteaProvider, "gitea", id="gitea"), + pytest.param(GiteeProvider, "gitee", id="gitee"), + pytest.param(GerritProvider, "gerrit", id="gerrit"), +] + + +@pytest.fixture(autouse=True) +def _crypto_env(monkeypatch): + monkeypatch.setenv("GIT_TOKEN_AES_KEY", "12345678901234567890123456789012") + monkeypatch.setenv("GIT_TOKEN_AES_IV", "1234567890123456") + import shared.utils.crypto as crypto + + monkeypatch.setattr(crypto, "_aes_key", None) + monkeypatch.setattr(crypto, "_aes_iv", None) + + +def _user(provider_type: str, token: str): + from unittest.mock import Mock + + user = Mock() + user.id = 1 + user.user_name = "testuser" + user.git_info = [ + { + "type": provider_type, + "git_domain": f"{provider_type}.example.com", + "git_token": token, + # Gerrit refuses an entry without one; harmless to the others. + "user_name": "testuser", + } + ] + return user + + +@pytest.mark.unit +@pytest.mark.parametrize("provider_class,provider_type", PROVIDERS) +def test_a_stored_token_is_decrypted(provider_class, provider_type): + """Sent as-is the ciphertext yields a 401, which reads as a bad credential + rather than as the provider never having decrypted it.""" + provider = provider_class() + user = _user(provider_type, encrypt_git_token(PLAIN_TOKEN)) + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == PLAIN_TOKEN + + +@pytest.mark.unit +@pytest.mark.parametrize("provider_class,provider_type", PROVIDERS) +def test_a_plaintext_token_is_left_alone(provider_class, provider_type): + """Deployments exist that put a usable token straight into git_info.""" + provider = provider_class() + user = _user(provider_type, PLAIN_TOKEN) + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == PLAIN_TOKEN + + +@pytest.mark.unit +@pytest.mark.parametrize("provider_class,provider_type", PROVIDERS) +def test_the_placeholder_survives(provider_class, provider_type): + """'***' means the real token is substituted in at call time by a deployment + overlay. Rewriting it would destroy the marker that substitution looks for.""" + provider = provider_class() + user = _user(provider_type, "***") + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == "***" + + +@pytest.mark.unit +@pytest.mark.parametrize("provider_class,provider_type", PROVIDERS) +def test_an_empty_token_stays_empty(provider_class, provider_type): + """Callers test the token for emptiness to raise "not configured". Turning it + into anything else would send them past that check.""" + provider = provider_class() + user = _user(provider_type, "") + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == "" From e24bf182bf2790f4177f5902dac8833f1cea0506 Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Tue, 4 Aug 2026 20:22:32 +0800 Subject: [PATCH 10/14] refactor(knowledge): own code wikis by their creator A code wiki was filed under a configured wiki account so that a repository would have one wiki outliving whoever asked for it. The knowledge-base ACL then granted nobody else access, which made repository permission the only authorisation -- and that check existed only in the endpoints written for it. Every path that reused an existing one fell back to the ACL and refused everybody: triggering a run, reading page content, chat citation, and the MCP tool. Public repositories were refused too, since the check is membership-based and never looks at visibility. Make it an ordinary knowledge base instead. The creator owns it, the ordinary ACL decides who reads it, and all four of those paths work without being taught anything. The repository is consulted once, when the wiki is created; sharing it afterwards is the creator's decision, the same as for any other knowledge base holding private material. A repository may now have several wikis, one per person who built one: a wiki created by A is invisible to B under A's ACL, so refusing B one of their own would take it away on a first-come basis. wiki_projects therefore holds one row per (repository, wiki) and the UNIQUE moves to that pair -- still a database constraint rather than a check-then-insert, and it lets COUNT(*) answer "how many wikis exist for this repository" without reading a JSON field. Triggering a run requires write access to the repository rather than merely being able to read the wiki: a run rewrites every page, so a wiki shared with a reader would otherwise let them spend a generation on somebody else's knowledge base. Creating a wiki now starts its first run, since a new wiki that sits empty until somebody finds the regenerate button is not a flow anyone would guess; a run that cannot start is logged rather than raised, because the knowledge base is already committed. read_access.py and exclude_code_wikis go with it. The second is worth naming: hiding code wikis from the general listing would also hide them from chat and the MCP tool, where being citable is the point. Spec section 6 is rewritten, including why this was reversed -- the rule was never written as a checkable invariant and had no list of enforcement points, so missing one was a matter of time rather than of attention. Co-Authored-By: Claude Opus 5 --- ...a7b8_allow_several_wikis_per_repository.py | 54 ++++ backend/app/api/endpoints/knowledge.py | 210 ++++++------ backend/app/api/endpoints/wiki.py | 9 +- backend/app/models/wiki.py | 21 +- backend/app/schemas/knowledge.py | 8 + .../knowledge/code_wiki/read_access.py | 164 ---------- .../services/knowledge/code_wiki/registry.py | 139 ++++---- .../services/knowledge/code_wiki/runner.py | 20 +- .../services/knowledge/code_wiki/source.py | 27 ++ .../app/services/knowledge/content_scope.py | 32 +- .../services/knowledge/knowledge_service.py | 163 +++++----- backend/tests/api/test_knowledge_code_wiki.py | 299 +++++++++++------- .../knowledge/code_wiki/test_read_access.py | 212 ------------- .../services/knowledge/test_content_scope.py | 204 ------------ 14 files changed, 529 insertions(+), 1033 deletions(-) create mode 100644 backend/alembic/versions/20260804_c3d4e5f6a7b8_allow_several_wikis_per_repository.py delete mode 100644 backend/app/services/knowledge/code_wiki/read_access.py delete mode 100644 backend/tests/services/knowledge/code_wiki/test_read_access.py delete mode 100644 backend/tests/services/knowledge/test_content_scope.py diff --git a/backend/alembic/versions/20260804_c3d4e5f6a7b8_allow_several_wikis_per_repository.py b/backend/alembic/versions/20260804_c3d4e5f6a7b8_allow_several_wikis_per_repository.py new file mode 100644 index 0000000000..a8aadcb5e4 --- /dev/null +++ b/backend/alembic/versions/20260804_c3d4e5f6a7b8_allow_several_wikis_per_repository.py @@ -0,0 +1,54 @@ +"""allow several wikis per repository + +Revision ID: c3d4e5f6a7b8 +Revises: 2b5791acc5fa + +A code wiki belongs to whoever created it, so a wiki built by one person is invisible +to everyone else under the ordinary knowledge-base ACL. "One repository, one wiki" +therefore stopped being a saving and became a way to take a wiki away from the second +person to ask for one. + +``wiki_projects`` accordingly holds one row per ``(repository, wiki)`` rather than one +per repository. The UNIQUE moves from ``source_url`` alone to the pair, which is still +a database constraint rather than a check-then-insert: it settles two requests racing +for the same pair, and it lets ``COUNT(*) WHERE source_url = ?`` answer "how many +wikis already exist for this repository" without reading a JSON field. + +Legacy wiki rows carry ``kind_id = 0`` and stay at most one per repository, which the +same constraint gives for free. +""" + +from collections.abc import Sequence +from typing import Union + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "c3d4e5f6a7b8" +down_revision: Union[str, Sequence[str], None] = "2b5791acc5fa" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# MySQL names the constraint after the column it was declared on. +OLD_UNIQUE = "source_url" +NEW_UNIQUE = "uq_wiki_projects_source_url_kind_id" + + +def upgrade() -> None: + """Upgrade schema.""" + # Created before the old one is dropped: between the two statements the table is + # covered by both rather than by neither, so a concurrent insert cannot slip a + # duplicate pair in through the gap. + op.create_unique_constraint(NEW_UNIQUE, "wiki_projects", ["source_url", "kind_id"]) + op.drop_constraint(OLD_UNIQUE, "wiki_projects", type_="unique") + + +def downgrade() -> None: + """Downgrade schema. + + Fails if any repository has more than one wiki, which is correct: silently + discarding one of them would destroy a generated knowledge base. Delete the + surplus wikis first if this has to be reversed. + """ + op.create_unique_constraint(OLD_UNIQUE, "wiki_projects", ["source_url"]) + op.drop_constraint(NEW_UNIQUE, "wiki_projects", type_="unique") diff --git a/backend/app/api/endpoints/knowledge.py b/backend/app/api/endpoints/knowledge.py index ae5dcebf10..c855fc3614 100644 --- a/backend/app/api/endpoints/knowledge.py +++ b/backend/app/api/endpoints/knowledge.py @@ -87,17 +87,10 @@ PUBLISHED_AT_KEY, PUBLISHED_COMMIT_KEY, ) -from app.services.knowledge.code_wiki.read_access import ( - may_read_code_wiki, - readable_wiki_ids, -) from app.services.knowledge.code_wiki.registry import ( CODE_WIKI_NAMESPACE, - CodeWikiOwnerMissing, claim_repository, existing_wiki_id, - project_for, - wiki_owner, ) from app.services.knowledge.code_wiki.run_mode import ChangedPath from app.services.knowledge.code_wiki.runner import CodeWikiRunError, start_run @@ -105,8 +98,8 @@ SourceAccessDenied, SourceRepository, assert_user_can_read_source, + assert_user_can_write_source, ) -from app.services.knowledge.content_scope import only_code_wikis from app.services.knowledge.orchestrator import ( DEFAULT_KNOWLEDGE_LIST_LIMIT, MAX_DOCUMENT_READ_LIMIT, @@ -523,53 +516,6 @@ def create_knowledge_base( ) -def _assert_the_wiki_account_can_clone( - db: Session, owner: User, requester: User, source: SourceRepository -) -> None: - """Check the account that will actually clone, not just the one asking. - - The requester's access was verified above, but the repository is cloned with the - wiki account's credentials. Without this the mismatch surfaces much later as a - failed generation, and nothing in that failure says the account simply needs - adding to the repository. - """ - if owner.id == requester.id: - return - try: - assert_user_can_read_source(db, owner.id, source) - except SourceAccessDenied as e: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=( - f"The wiki account cannot read '{source.project_name}', so the " - f"repository could be checked out by you but not by the account " - f"that generates the wiki. Give '{owner.user_name}' read access to " - f"the repository, or set WIKI_DEFAULT_USER_ID=0 to generate with " - f"the requester's own credentials. ({e})" - ), - ) from e - - -def _code_wiki_response(db: Session, knowledge_base_id: int) -> KnowledgeBaseResponse: - """Render a code wiki without a knowledge-base ACL check. - - The wiki belongs to the wiki account, so the requester never passes that check — - including for the wiki they just asked to have built. Authorisation for this - endpoint is the repository gate above, which they have already passed. - """ - kind = KnowledgeService._get_knowledge_base_record(db, knowledge_base_id) - if kind is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Code wiki not found" - ) - # Counted rather than left to default to zero: this path returns a wiki that - # already exists and normally holds pages, and reporting none of them would - # disagree with what the listing says about the same wiki. - return KnowledgeBaseResponse.from_kind( - kind, KnowledgeService.get_document_count(db, kind.id) - ) - - @router.get("/code-wikis", response_model=CodeWikiListResponse) @trace_sync("list_code_wikis", "knowledge.api") def list_code_wikis( @@ -578,27 +524,20 @@ def list_code_wikis( current_user: User = Depends(security.get_current_user), db: Session = Depends(get_db), ): - """Code wikis the caller may read. + """Code wikis the caller may read, judged by the ordinary knowledge-base ACL. - Separate from the general knowledge base list rather than a filter on it: a code - wiki belongs to the wiki account, so it matches none of that list's scopes, and - who may read one is decided by the repository instead. + A code wiki belongs to whoever created it, so this is the same visibility every + other knowledge base has. The endpoint stays separate only because its list items + carry repository fields; it grants nothing the general list would not. """ - wikis = ( - only_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, # noqa: E712 - ) + visible = [ + kind + for kind in KnowledgeService.list_knowledge_bases( + db, current_user.id, scope=ResourceScope.ALL ) - .order_by(Kind.updated_at.desc()) - .all() - ) - # Access is judged before paging, because a page of rows the caller cannot read - # would come back short and the total would be wrong. The judgement itself is - # one cache read per host, not one per wiki. - readable = readable_wiki_ids(db, current_user, wikis) - visible = [kind for kind in wikis if kind.id in readable] + if (kind.json or {}).get("spec", {}).get("kbType") + == KnowledgeBaseType.CODE_WIKI.value + ] window = visible[(page - 1) * limit : page * limit] # One grouped query for the whole page rather than a COUNT per wiki. @@ -656,50 +595,48 @@ def create_code_wiki( except SourceAccessDenied as e: raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) from e - existing_id = existing_wiki_id(db, source) + existing_id = existing_wiki_id(db, source, owner_id=current_user.id) if existing_id: response.status_code = status.HTTP_200_OK - return _code_wiki_response(db, existing_id) + return KnowledgeBaseResponse.from_kind( + KnowledgeService._get_knowledge_base_record(db, existing_id), + KnowledgeService.get_document_count(db, existing_id), + ) try: - owner = wiki_owner(db, current_user) - _assert_the_wiki_account_can_clone(db, owner, current_user, source) - project = project_for(db, source) result = knowledge_orchestrator.create_knowledge_base( db=db, - user=owner, + user=current_user, name=data.name, description=data.description, - namespace=CODE_WIKI_NAMESPACE, + namespace=data.namespace or CODE_WIKI_NAMESPACE, kb_type=KnowledgeBaseType.CODE_WIKI.value, source=source, ) - claim_repository(db, project, result.id) + claim_repository(db, source, result.id) db.commit() add_span_event( "knowledge.code_wiki.created", { "kb_id": str(result.id), "project_name": source.project_name, - "requested_by": str(current_user.id), - "owner_id": str(owner.id), + "owner_id": str(current_user.id), }, ) + _start_the_first_run(db, current_user, result.id) return result - except CodeWikiOwnerMissing as e: - db.rollback() - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) - ) from e except IntegrityError as e: - # Two requests for one repository, both past the check above. The UNIQUE on - # wiki_projects.source_url is what settles it; the loser returns the winner's - # wiki rather than an error, since that is what it asked for. + # The same caller asking twice at once. UNIQUE (source_url, kind_id) settles + # it at the database rather than in a check-then-insert window; the loser + # returns the winner's wiki, since that is what it asked for. db.rollback() - settled = existing_wiki_id(db, source) + settled = existing_wiki_id(db, source, owner_id=current_user.id) if settled: response.status_code = status.HTTP_200_OK - return _code_wiki_response(db, settled) + return KnowledgeBaseResponse.from_kind( + KnowledgeService._get_knowledge_base_record(db, settled), + KnowledgeService.get_document_count(db, settled), + ) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=f"Knowledge base with name '{data.name}' already exists", @@ -740,21 +677,69 @@ def _as_page_node(node) -> CodeWikiPageNode: def _readable_code_wiki(db: Session, user: User, knowledge_base_id: int) -> Kind: - """Load a code wiki the caller may read, or refuse.""" - knowledge_base = KnowledgeService._get_knowledge_base_record(db, knowledge_base_id) - if knowledge_base is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Code wiki not found" - ) - if not may_read_code_wiki(db, user, knowledge_base): - # Indistinguishable from missing on purpose: whether a repository has a wiki - # is itself something only its members should learn. + """Load a code wiki the caller may read, or refuse. + + The ordinary knowledge-base ACL decides, the same as for any other knowledge + base. Refusal is 404 rather than 403 so that the reply does not confirm the + existence of a wiki the caller cannot see. + """ + knowledge_base, has_access = KnowledgeService.get_knowledge_base( + db=db, knowledge_base_id=knowledge_base_id, user_id=user.id + ) + if knowledge_base is None or not has_access: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Code wiki not found" ) return knowledge_base +def _start_the_first_run(db: Session, user: User, knowledge_base_id: int) -> None: + """Begin generating the wiki that was just created. + + Without this a new wiki is empty until somebody finds the regenerate button, + which is not a flow anyone would guess. Failures are logged rather than raised: + the knowledge base is already committed and returning an error for it would say + the creation failed when it did not. The reader shows the empty state, and the + same button still starts a run. + """ + knowledge_base = KnowledgeService._get_knowledge_base_record(db, knowledge_base_id) + if knowledge_base is None: # pragma: no cover - just committed + return + try: + start_run(db, knowledge_base=knowledge_base, user=user) + except (CodeWikiRunError, GenerationInFlight) as e: + logger.warning( + "[code_wiki] first run not started for kb %s: %s", knowledge_base_id, e + ) + except Exception: # pragma: no cover - defensive + db.rollback() + logger.exception( + "[code_wiki] first run not started for kb %s", knowledge_base_id + ) + + +def _assert_caller_may_regenerate( + db: Session, user: User, knowledge_base: Kind +) -> None: + """Refuse a caller who may read the wiki but not change its repository. + + A knowledge base that is not a code wiki, or one with no repository recorded, + is left to ``start_run`` to reject: it already says which of the two it is, and + answering "you lack write access" for something with nothing to write to would + be a worse account of the refusal. + """ + spec = (knowledge_base.json or {}).get("spec", {}) + if spec.get("kbType") != KnowledgeBaseType.CODE_WIKI.value: + return + source = SourceRepository.from_spec(spec.get("source")) + if source is None or not source.project_name: + return + try: + assert_user_can_write_source(db, user.id, source) + except SourceAccessDenied as e: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) from e + + @router.post( "/{knowledge_base_id}/code-wiki/generations", response_model=CodeWikiRunResponse, @@ -769,26 +754,17 @@ def start_code_wiki_run( ): """Regenerate a code wiki now, without waiting for its schedule. - Managing the knowledge base is required rather than merely reading it: a run - rewrites every page in it, so this is closer to replacing its content than to - viewing it. + **Write access to the repository is required**, not merely the ability to read + the wiki. A run rewrites every page, so it is closer to changing the repository + than to reading its documentation — and a wiki shared with a reader would + otherwise let them spend a generation on somebody else's knowledge base. Answers 202 even when no run was needed. "The repository has not changed since the published version" is a successful outcome, not a failure, and the response says which it was. """ - knowledge_base = KnowledgeService._get_knowledge_base_record(db, knowledge_base_id) - if knowledge_base is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Knowledge base not found" - ) - if not KnowledgeService.can_manage_knowledge_base( - db, knowledge_base_id, current_user.id - ): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="You do not have permission to regenerate this code wiki", - ) + knowledge_base = _readable_code_wiki(db, current_user, knowledge_base_id) + _assert_caller_may_regenerate(db, current_user, knowledge_base) try: started = start_run( diff --git a/backend/app/api/endpoints/wiki.py b/backend/app/api/endpoints/wiki.py index 380d49f90a..b676dc44d0 100644 --- a/backend/app/api/endpoints/wiki.py +++ b/backend/app/api/endpoints/wiki.py @@ -502,9 +502,12 @@ def get_wiki_config( bound_model_name = None if default_team_name: - # Determine which user_id to use for team lookup - # If DEFAULT_USER_ID is set (> 0), use it; otherwise use current user - lookup_user_id = default_user_id if default_user_id > 0 else current_user.id + # A code wiki runs as its own owner, so the team has to be resolved for the + # caller: answering for the legacy wiki account would report a bound model + # that the run about to start does not have. + lookup_user_id = ( + current_user.id if code_wiki or default_user_id <= 0 else default_user_id + ) # Find team by name and namespace team = team_kinds_service.get_team_by_name_and_namespace( diff --git a/backend/app/models/wiki.py b/backend/app/models/wiki.py index 3fe2a5a703..cb825d33da 100644 --- a/backend/app/models/wiki.py +++ b/backend/app/models/wiki.py @@ -17,6 +17,7 @@ Integer, String, Text, + UniqueConstraint, ) from sqlalchemy.sql import func @@ -33,29 +34,35 @@ class WikiProject(WikiBase): project_name = Column(String(200), nullable=False, index=True) project_type = Column(String(50), nullable=False, default="git", index=True) source_type = Column(String(50), nullable=False, default="github", index=True) - source_url = Column(String(500), nullable=False, unique=True) + source_url = Column(String(500), nullable=False) source_id = Column(String(100), nullable=True) source_domain = Column(String(100), nullable=True) description = Column(Text) ext = Column(JSON, comment="Project extension data") - # The code wiki built from this repository, or 0 for a legacy project that has - # none. Recorded here rather than on the knowledge base because `source_url` - # above is UNIQUE: that constraint is what makes "one repository, one wiki" hold - # under two people creating at the same moment, which a check-then-insert on a - # JSON field could not. + # The code wiki this row registers, or 0 for a legacy wiki project. One row per + # (repository, wiki): a code wiki belongs to its creator, so a repository may + # have several, one per person who built one. kind_id = Column( Integer, nullable=False, default=0, server_default="0", index=True, - comment="Code wiki knowledge base built from this repository; 0 = none", + comment="Code wiki knowledge base built from this repository; 0 = legacy", ) is_active = Column(Boolean, nullable=False, default=True) created_at = Column(DateTime, default=func.now()) updated_at = Column(DateTime, default=func.now(), onupdate=func.now()) __table_args__ = ( + # The pair rather than the URL alone. It is what settles two requests racing + # for the same (repository, wiki), which a check against a JSON field on the + # knowledge base could not — that leaves a window between read and insert + # exactly where it matters. Legacy rows carry kind_id = 0 and are therefore + # still limited to one per repository. + UniqueConstraint( + "source_url", "kind_id", name="uq_wiki_projects_source_url_kind_id" + ), { "sqlite_autoincrement": True, "mysql_engine": "InnoDB", diff --git a/backend/app/schemas/knowledge.py b/backend/app/schemas/knowledge.py index d610d9bcf5..eda939e0d1 100644 --- a/backend/app/schemas/knowledge.py +++ b/backend/app/schemas/knowledge.py @@ -307,6 +307,14 @@ class CodeWikiCreate(BaseModel): name: str = Field(..., min_length=1, max_length=100) description: Optional[str] = Field(None, max_length=500) + namespace: Optional[str] = Field( + None, + max_length=100, + description=( + "Where to file the wiki, as for any other knowledge base. Defaults to " + "the creator's personal namespace." + ), + ) source_type: Literal["github", "gitlab", "gitea"] = Field( ..., description=( diff --git a/backend/app/services/knowledge/code_wiki/read_access.py b/backend/app/services/knowledge/code_wiki/read_access.py deleted file mode 100644 index 59569fc2f3..0000000000 --- a/backend/app/services/knowledge/code_wiki/read_access.py +++ /dev/null @@ -1,164 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Weibo, Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Who may read a code wiki. - -A code wiki belongs to the wiki account, so knowledge-base ACLs grant nobody else -anything. What decides access instead is the repository: **if you can read the code, -you can read its documentation**, which is the only rule that does not need a second -place to keep in sync with the repository's own membership. - -The direction matters, and it is why this works where syncing members would not. -Syncing runs git → wegent and needs every repository member mapped back to a wegent -account: a many-to-many lookup against credentials stored in a JSON column, which -silently omits everyone who has never configured a token. Asking at read time runs -wegent → git and uses the signed-in user's own credentials, so there is no mapping at -all — the identity is theirs by construction, and someone with no token is correctly -unable to demonstrate anything. - -The answer usually costs nothing: the provider layer already keeps each user's -accessible repositories in Redis for two hours, built once behind a lock because it -is expensive. This reads that. Only a cold cache falls through to asking about the -one repository, which is the same call the creation gate makes. - -Two hours is also how long someone removed from a repository keeps access. -""" - -import logging -from typing import Any - -from sqlalchemy.orm import Session - -from app.models.kind import Kind -from app.models.user import User -from app.services.knowledge.code_wiki.source import SourceRepository, provider_for -from app.services.knowledge.knowledge_service import _run_async_in_new_loop - -logger = logging.getLogger(__name__) - - -def may_read_code_wiki(db: Session, user: User, knowledge_base: Kind) -> bool: - """Whether ``user`` may read this code wiki, judged by its repository.""" - spec = (knowledge_base.json or {}).get("spec", {}) - source = SourceRepository.from_spec(spec.get("source")) - if source is None or not source.project_name: - # No repository to judge by. Refusing is the safe direction: a wiki nobody - # can attribute to a repository is one nobody should inherit access to. - logger.warning( - "[code_wiki] kb %s has no source repository; read refused", - knowledge_base.id, - ) - return False - - if source.project_name.casefold() in _cached_repository_names(user, source): - return True - - return _repository_is_readable_now(db, user, source) - - -def _cached_repository_names(user: User, source: SourceRepository) -> frozenset[str]: - """The repositories this user can reach on this host, as the cache has them. - - Keyed by host rather than by repository, so judging many wikis on one host costs - one read. Returns an empty set for a cold cache, which the caller distinguishes - from "not a member" by falling through to the live check. - """ - provider = provider_for(source.source_type) - reader = getattr(provider, "_get_all_repositories_from_cache", None) - if reader is None: - return frozenset() - - try: - cached = _run_async_in_new_loop(reader(user, source.source_domain)) - except Exception as exc: - logger.debug( - "[code_wiki] could not read the repository cache for user %s: %s", - user.id, - exc, - ) - return frozenset() - - return frozenset( - _full_name(entry).casefold() for entry in (cached or []) if _full_name(entry) - ) - - -def _full_name(entry: Any) -> str: - if isinstance(entry, dict): - return str(entry.get("full_name") or entry.get("name") or "") - return "" - - -def _repository_is_readable_now( - db: Session, user: User, source: SourceRepository -) -> bool: - """Ask the provider about this one repository. - - Reached when the cache is cold, which is a user who has not opened the repository - picker yet. Building the whole list here would make a first read pay for every - repository they can see; asking about one is the same call the creation gate - makes and is bounded. - """ - from app.services.knowledge.code_wiki.source import ( - SourceAccessDenied, - assert_user_can_read_source, - ) - - try: - assert_user_can_read_source(db, user.id, source) - return True - except SourceAccessDenied: - return False - except Exception as exc: - # An unreachable provider must not become an open door. - logger.warning( - "[code_wiki] could not check %s for user %s: %s", - source.project_name, - user.id, - exc, - ) - return False - - -def readable_wiki_ids(db: Session, user: User, knowledge_bases: list[Kind]) -> set[int]: - """Which of these code wikis ``user`` may read. - - The cache is read once per host, not once per wiki. Judging each separately would - open a fresh event loop and issue a fresh Redis read every time, and on a cold - cache would put one live provider call per wiki on the request thread — the cost - of a listing growing with the number of documented repositories. - """ - sources: dict[int, SourceRepository] = {} - for knowledge_base in knowledge_bases: - source = SourceRepository.from_spec( - (knowledge_base.json or {}).get("spec", {}).get("source") - ) - if source is None or not source.project_name: - logger.warning( - "[code_wiki] kb %s has no source repository; read refused", - knowledge_base.id, - ) - continue - sources[knowledge_base.id] = source - - caches: dict[tuple[str, str], frozenset[str]] = {} - readable: set[int] = set() - unresolved: dict[int, SourceRepository] = {} - - for kind_id, source in sources.items(): - host = (source.source_type, source.source_domain) - if host not in caches: - caches[host] = _cached_repository_names(user, source) - if source.project_name.casefold() in caches[host]: - readable.add(kind_id) - else: - unresolved[kind_id] = source - - # Only what the cache could not answer. A warm cache leaves this empty, which is - # the common case; a cold one pays per repository, which is why it is last. - for kind_id, source in unresolved.items(): - if _repository_is_readable_now(db, user, source): - readable.add(kind_id) - - return readable diff --git a/backend/app/services/knowledge/code_wiki/registry.py b/backend/app/services/knowledge/code_wiki/registry.py index 1b30f84320..419a725b92 100644 --- a/backend/app/services/knowledge/code_wiki/registry.py +++ b/backend/app/services/knowledge/code_wiki/registry.py @@ -2,27 +2,21 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Which repository a code wiki documents, and who owns the result. - -**One repository, one wiki.** Two people asking for the same repository should get -the same wiki, not two of them — two full generations, two indexes, two sets of -citations, for one repository's worth of documentation. The claim is registered on -``wiki_projects``, whose ``source_url`` is already UNIQUE, so the database refuses the -second one even when both requests are in flight at once. A check against a JSON field -on the knowledge base could not: it would leave a window between the read and the -insert exactly wide enough for the case worth preventing. - -**A code wiki belongs to the configured wiki account, not to whoever asked for it.** -The account is the one whose Git credentials clone the repository, so it is the -identity the wiki actually depends on. Attributing it to the requester instead would -make a repository's documentation disappear when one person leaves, and would let two -colleagues each own a private copy of the same thing. - -The consequence is deliberate and load-bearing: knowledge-base ACLs no longer grant -anyone but that account access, so **who may read a code wiki is decided by who may -read its repository**. That check is not an extra convenience here — it is the only -one, which is why it has a fallback rather than being allowed to fail closed on a -cold cache. +"""Which repository a code wiki documents. + +A code wiki is an ordinary knowledge base owned by whoever created it, so the +ordinary ACL decides who may read it. The repository is consulted once, when the +wiki is created, and never tracked afterwards. + +**A repository may have several wikis.** One per person who built one: since a wiki +created by A is invisible to B under A's ACL, refusing B a wiki of their own would +take it away on a first-come basis. ``wiki_projects`` therefore holds one row per +``(repository, wiki)`` rather than one per repository, and the composite UNIQUE is +what settles two requests racing for the same pair — a check against a JSON field on +the knowledge base could not, leaving a window between the read and the insert +exactly wide enough for the case worth preventing. + +Rows with ``kind_id = 0`` belong to the legacy wiki, at most one per repository. """ import logging @@ -30,61 +24,23 @@ from sqlalchemy.orm import Session -from app.core.wiki_config import wiki_settings -from app.models.user import User from app.models.wiki import WikiProject from app.services.knowledge.code_wiki.source import SourceRepository logger = logging.getLogger(__name__) -# Code wikis are not filed under anybody's team or organization: the repository -# decides who may read them, so the namespace would be a second, weaker answer to a -# question already settled. +# Where a code wiki lands when the caller expressed no preference. Unlike every +# other decision here this one is a plain default, not a constraint: the creator may +# file a wiki in any namespace they could file a knowledge base in. CODE_WIKI_NAMESPACE = "default" -class CodeWikiOwnerMissing(RuntimeError): - """Raised when the configured wiki account does not exist.""" - - -def wiki_owner(db: Session, requester: User) -> User: - """The account a code wiki is created under. - - Falls back to the requester only when no wiki account is configured - (``WIKI_DEFAULT_USER_ID=0``), which is the single-user setup where the two are - the same thing anyway. - """ - configured_id = wiki_settings.DEFAULT_USER_ID - if configured_id <= 0: - return requester - - # Queried directly: user_service.get_user_by_id raises a 404 for a missing user, - # which would surface as "not found" on a request that found everything it asked - # for and is really a misconfiguration. - owner = db.query(User).filter(User.id == configured_id).first() - if owner is None: - raise CodeWikiOwnerMissing( - f"Configured wiki account {configured_id} does not exist. " - "Check WIKI_DEFAULT_USER_ID." - ) - return owner - +def claim_repository(db: Session, source: SourceRepository, kind_id: int) -> None: + """Register that ``kind_id`` is a wiki of this repository. -def project_for(db: Session, source: SourceRepository) -> WikiProject: - """The registry row for a repository, created if this is the first time. - - Shared with the legacy wiki deliberately — one row per repository, whichever - feature asked first. The two never mix beyond that, because a version line is - keyed by ``wiki_generations.kind_id`` and legacy rows carry ``0``. + Flushed rather than committed: the caller owns the transaction, so a failure + later in the same request must take this row with it. """ - existing = ( - db.query(WikiProject) - .filter(WikiProject.source_url == source.source_url) - .first() - ) - if existing is not None: - return existing - project = WikiProject( project_name=source.project_name, project_type="git", @@ -93,32 +49,53 @@ def project_for(db: Session, source: SourceRepository) -> WikiProject: source_domain=source.source_domain, description="", ext={}, - kind_id=0, + kind_id=kind_id, is_active=True, ) db.add(project) db.flush() logger.info( - "[code_wiki] registered repository %s as project %s", + "[code_wiki] registered repository %s as project %s for kb %s", source.project_name, project.id, + kind_id, ) - return project -def existing_wiki_id(db: Session, source: SourceRepository) -> Optional[int]: - """The code wiki already built from this repository, if there is one.""" - project = ( - db.query(WikiProject) - .filter(WikiProject.source_url == source.source_url) +def existing_wiki_id( + db: Session, source: SourceRepository, *, owner_id: int +) -> Optional[int]: + """This caller's own wiki of this repository, if they already have one. + + Scoped to the caller because somebody else's wiki is not an answer to "give me a + wiki of this repository" — under their ACL the caller may not even be able to + read it. + """ + from app.models.kind import Kind + + row = ( + db.query(WikiProject.kind_id) + .join(Kind, Kind.id == WikiProject.kind_id) + .filter( + WikiProject.source_url == source.source_url, + WikiProject.kind_id > 0, + Kind.user_id == owner_id, + Kind.is_active.is_(True), + ) .first() ) - if project is None or not project.kind_id: - return None - return project.kind_id + return row[0] if row else None -def claim_repository(db: Session, project: WikiProject, kind_id: int) -> None: - """Record that this repository's wiki is ``kind_id``.""" - project.kind_id = kind_id - db.flush() +def wiki_count_for(db: Session, source_url: str) -> int: + """How many code wikis exist for this repository, whoever owns them. + + Shown before creating one so that somebody can ask for a share instead of paying + for a second generation. Counted across all owners on purpose — the point is that + the work has already been done, not that the caller can see it. + """ + return ( + db.query(WikiProject) + .filter(WikiProject.source_url == source_url, WikiProject.kind_id > 0) + .count() + ) diff --git a/backend/app/services/knowledge/code_wiki/runner.py b/backend/app/services/knowledge/code_wiki/runner.py index 24e30207b3..cbcfd9f67b 100644 --- a/backend/app/services/knowledge/code_wiki/runner.py +++ b/backend/app/services/knowledge/code_wiki/runner.py @@ -279,23 +279,17 @@ def _knowledge_base_of(db: Session, generation: WikiGeneration) -> Optional[Kind def _resolve_execution_context(db: Session, user: User) -> tuple[Kind, User]: """Find the team that runs code wikis, and the user it runs as. - Both come from configuration rather than from the request. A wiki is generated by - the system on the repository's behalf, so letting the caller choose the team would - let them choose the prompt and the tools the agent gets. + The run executes as the knowledge base's owner, using their Git credentials to + clone. An expired token then fails that owner's own wiki, which is attributable; + a shared account's expiry would fail everybody's at once. + + The team, by contrast, comes from configuration rather than from the request: it + carries the prompt and the tools the agent gets, so letting the caller choose it + would let them choose those. """ from app.services.adapters.team_kinds import team_kinds_service - from app.services.user import user_service task_user = user - if wiki_settings.DEFAULT_USER_ID > 0: - configured = user_service.get_user_by_id(db, wiki_settings.DEFAULT_USER_ID) - if configured is None: - raise CodeWikiRunError( - f"Configured wiki user {wiki_settings.DEFAULT_USER_ID} does not exist. " - "Check WIKI_DEFAULT_USER_ID." - ) - task_user = configured - team_name = wiki_settings.CODE_WIKI_TEAM_NAME if not team_name: raise CodeWikiRunError( diff --git a/backend/app/services/knowledge/code_wiki/source.py b/backend/app/services/knowledge/code_wiki/source.py index 069f9210cc..71e95f0768 100644 --- a/backend/app/services/knowledge/code_wiki/source.py +++ b/backend/app/services/knowledge/code_wiki/source.py @@ -172,6 +172,12 @@ def provider_for(source_type: str): return None +# Where "may write" starts, on the scale both providers are mapped onto: GitLab's +# Developer is 30, and GitHub's "push" is mapped to the same number. Anything below +# it can read the repository but not change it. +WRITE_ACCESS_LEVEL = 30 + + def _check_access(provider, source_type: str, token: str, source: SourceRepository): """Ask the provider whether the token can read the repository. @@ -259,3 +265,24 @@ def assert_user_can_read_source( source.project_name, ) return result + + +def assert_user_can_write_source( + db: Session, user_id: int, source: SourceRepository +) -> Dict[str, Any]: + """Verify a user may change the repository, or raise ``SourceAccessDenied``. + + Triggering a run rewrites every page of the wiki, so it is closer to changing + the repository than to reading it. Read access is checked first and separately, + so that somebody who cannot see the repository at all is told that rather than + being told their permissions are merely insufficient. + """ + result = assert_user_can_read_source(db, user_id, source) + + level = result.get("access_level") + if not isinstance(level, int) or level < WRITE_ACCESS_LEVEL: + raise SourceAccessDenied( + f"You have read access to '{source.project_name}' but not write access, " + "which is what regenerating its wiki requires." + ) + return result diff --git a/backend/app/services/knowledge/content_scope.py b/backend/app/services/knowledge/content_scope.py index 6bcf29728b..7a6ed3519e 100644 --- a/backend/app/services/knowledge/content_scope.py +++ b/backend/app/services/knowledge/content_scope.py @@ -79,31 +79,15 @@ def code_targets(query: Query) -> Query: return query.filter(KnowledgeDocument.source_type == CODE_TARGET_SOURCE_TYPE) -def exclude_code_wikis(query: Query) -> Query: - """Keep code wikis out of a ``Kind`` query listing knowledge bases. - - The default for every general listing. A code wiki is a projection an agent - rewrites on a schedule, so it belongs in the reader built for that, not among - knowledge bases a person fills by hand — and an agent shown one through the MCP - tool may well try to write into it, where the next publish silently deletes - whatever it added. - - Relying on ownership to hide them instead would be incidental rather than - stated: it holds only while the wiki account is somebody else, and stops holding - for that account, for an administrator, or if that decision is ever revisited. - """ - kb_type = Kind.json["spec"]["kbType"].as_string() - # NULL has to be spelled out: a knowledge base predating kbType compares NULL - # against the literal, which is neither true nor false, and the row would vanish - # from every listing. Rendered through SQLAlchemy's JSON accessor rather than - # json_unquote, which only exists on MySQL. - return query.filter( - or_(kb_type.is_(None), kb_type != KnowledgeBaseType.CODE_WIKI.value) - ) - - def only_code_wikis(query: Query) -> Query: - """The other half: a ``Kind`` query restricted to code wikis.""" + """A ``Kind`` query restricted to code wikis. + + For listings that render repository fields, not for access control: a code wiki + is visible exactly as far as its own ACL says, the same as any other knowledge + base. There is deliberately no ``exclude_code_wikis`` counterpart — hiding them + from the general listing would also hide them from chat and the MCP tool, where + being citable is the point. + """ return query.filter( Kind.json["spec"]["kbType"].as_string() == KnowledgeBaseType.CODE_WIKI.value ) diff --git a/backend/app/services/knowledge/knowledge_service.py b/backend/app/services/knowledge/knowledge_service.py index 18149def9c..66544ce21e 100644 --- a/backend/app/services/knowledge/knowledge_service.py +++ b/backend/app/services/knowledge/knowledge_service.py @@ -60,7 +60,7 @@ get_user_groups, get_view_role_in_group, ) -from app.services.knowledge.content_scope import exclude_code_wikis, wiki_pages +from app.services.knowledge.content_scope import wiki_pages from app.services.knowledge.folder_policy import assert_document_can_be_placed_in_folder from app.services.knowledge.knowledge_access_policy import ( can_directly_access_knowledge_base as evaluate_direct_knowledge_base_access, @@ -512,14 +512,16 @@ def list_knowledge_bases( # Single query to get personal and explicitly shared knowledge bases # Personal: user_id matches and namespace is "default" # Shared: id is in shared_kb_ids - all_kbs = exclude_code_wikis( - db.query(Kind).filter( + all_kbs = ( + db.query(Kind) + .filter( Kind.kind == "KnowledgeBase", Kind.is_active == True, ((Kind.user_id == user_id) & (Kind.namespace == "default")) | (Kind.id.in_(shared_kb_ids) if shared_kb_ids else False), ) - ).all() + .all() + ) # Separate into personal and shared for sorting personal = [ @@ -546,12 +548,11 @@ def list_knowledge_bases( # KBs belonging to this group (native group KBs only) # Entity-authorized KBs are shown in personal shared_with_me instead group_kbs = ( - exclude_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.namespace == group_name, - Kind.is_active == True, - ) + db.query(Kind) + .filter( + Kind.kind == "KnowledgeBase", + Kind.namespace == group_name, + Kind.is_active == True, ) .order_by(Kind.updated_at.desc()) .all() @@ -562,15 +563,13 @@ def list_knowledge_bases( # Organization knowledge bases are visible to all users # Query knowledge bases in namespaces with level='organization' organization_kbs = ( - exclude_code_wikis( - db.query(Kind) - .join(Namespace, Kind.namespace == Namespace.name) - .filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Namespace.level == GroupLevel.organization.value, - Namespace.is_active == True, - ) + db.query(Kind) + .join(Namespace, Kind.namespace == Namespace.name) + .filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Namespace.level == GroupLevel.organization.value, + Namespace.is_active == True, ) .order_by(Kind.updated_at.desc()) .all() @@ -624,11 +623,9 @@ def list_knowledge_bases( # Team: namespace is in accessible_groups # Organization: namespace has level='organization' # Shared: id is in shared_kb_ids - query = exclude_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - ) + query = db.query(Kind).filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, ) conditions = [(Kind.user_id == user_id) & (Kind.namespace == "default")] @@ -2011,13 +2008,12 @@ def get_accessible_knowledge( # Get personal knowledge bases (created by user) personal_kbs = ( - exclude_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.user_id == user_id, - Kind.namespace == "default", - Kind.is_active == True, - ) + db.query(Kind) + .filter( + Kind.kind == "KnowledgeBase", + Kind.user_id == user_id, + Kind.namespace == "default", + Kind.is_active == True, ) .order_by(Kind.updated_at.desc()) .all() @@ -2056,13 +2052,12 @@ def get_accessible_knowledge( ) - included_personal_ids if extra_shared_ids: shared_kbs = ( - exclude_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.id.in_(extra_shared_ids), - Kind.is_active == True, - Kind.user_id != user_id, - ) + db.query(Kind) + .filter( + Kind.kind == "KnowledgeBase", + Kind.id.in_(extra_shared_ids), + Kind.is_active == True, + Kind.user_id != user_id, ) .order_by(Kind.updated_at.desc()) .all() @@ -2102,12 +2097,11 @@ def get_accessible_knowledge( # Get knowledge bases in this group group_kbs = ( - exclude_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.namespace == group_name, - Kind.is_active == True, - ) + db.query(Kind) + .filter( + Kind.kind == "KnowledgeBase", + Kind.namespace == group_name, + Kind.is_active == True, ) .order_by(Kind.updated_at.desc()) .all() @@ -2220,13 +2214,12 @@ def get_personal_knowledge_bases_grouped( # Get KBs created by user (personal knowledge bases, namespace=default) created_kbs = ( - exclude_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Kind.namespace == "default", - Kind.user_id == user_id, - ) + db.query(Kind) + .filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Kind.namespace == "default", + Kind.user_id == user_id, ) .order_by(Kind.updated_at.desc()) .all() @@ -2254,14 +2247,13 @@ def get_personal_knowledge_bases_grouped( shared_kbs = [] if shared_kb_ids: shared_kbs = ( - exclude_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Kind.id.in_(shared_kb_ids), - Kind.namespace == "default", - Kind.user_id != user_id, # Exclude KBs created by current user - ) + db.query(Kind) + .filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Kind.id.in_(shared_kb_ids), + Kind.namespace == "default", + Kind.user_id != user_id, # Exclude KBs created by current user ) .order_by(Kind.updated_at.desc()) .all() @@ -2687,13 +2679,12 @@ def get_all_knowledge_bases_grouped( # 1. Get personal knowledge bases created by user (single query) personal_created = ( - exclude_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Kind.namespace == "default", - Kind.user_id == user_id, - ) + db.query(Kind) + .filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Kind.namespace == "default", + Kind.user_id == user_id, ) .order_by(Kind.updated_at.desc()) .all() @@ -2717,13 +2708,12 @@ def get_all_knowledge_bases_grouped( shared_kbs: list[Kind] = [] if shared_kb_ids: shared_kbs = ( - exclude_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Kind.id.in_(shared_kb_ids), - Kind.user_id != user_id, - ) + db.query(Kind) + .filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Kind.id.in_(shared_kb_ids), + Kind.user_id != user_id, ) .order_by(Kind.updated_at.desc()) .all() @@ -2797,12 +2787,11 @@ def get_all_knowledge_bases_grouped( group_kbs: list[Kind] = [] if accessible_groups: group_kbs = ( - exclude_code_wikis( - db.query(Kind).filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Kind.namespace.in_(accessible_groups), - ) + db.query(Kind) + .filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Kind.namespace.in_(accessible_groups), ) .order_by(Kind.updated_at.desc()) .all() @@ -2847,15 +2836,13 @@ def get_all_knowledge_bases_grouped( # 5. Get organization knowledge bases (single query) org_kbs = ( - exclude_code_wikis( - db.query(Kind) - .join(Namespace, Kind.namespace == Namespace.name) - .filter( - Kind.kind == "KnowledgeBase", - Kind.is_active == True, - Namespace.level == GroupLevel.organization.value, - Namespace.is_active == True, - ) + db.query(Kind) + .join(Namespace, Kind.namespace == Namespace.name) + .filter( + Kind.kind == "KnowledgeBase", + Kind.is_active == True, + Namespace.level == GroupLevel.organization.value, + Namespace.is_active == True, ) .order_by(Kind.updated_at.desc()) .all() diff --git a/backend/tests/api/test_knowledge_code_wiki.py b/backend/tests/api/test_knowledge_code_wiki.py index c5627d481e..7d92896c8c 100644 --- a/backend/tests/api/test_knowledge_code_wiki.py +++ b/backend/tests/api/test_knowledge_code_wiki.py @@ -144,10 +144,21 @@ def _run_url(knowledge_base_id: int) -> str: return f"/api/knowledge-bases/{knowledge_base_id}/code-wiki/generations" +@pytest.fixture +def caller_can_write(): + """Grant repository write access for tests that are about something else.""" + with patch( + "app.api.endpoints.knowledge.assert_user_can_write_source", + return_value={"has_access": True, "access_level": 30}, + ): + yield + + def test_a_run_can_be_triggered_without_waiting_for_a_schedule( test_client: TestClient, auth_headers: dict[str, str], kind_services_use_test_db, + caller_can_write, ): kb_id = _create_wiki(test_client, auth_headers) @@ -173,6 +184,7 @@ def test_a_run_that_was_not_needed_is_a_success_not_a_failure( test_client: TestClient, auth_headers: dict[str, str], kind_services_use_test_db, + caller_can_write, ): """ "Nothing changed" is the answer the caller asked for, not an error.""" kb_id = _create_wiki(test_client, auth_headers) @@ -197,6 +209,7 @@ def test_a_second_run_while_one_is_live_is_a_conflict( test_client: TestClient, auth_headers: dict[str, str], kind_services_use_test_db, + caller_can_write, ): from app.services.knowledge.code_wiki.generation import GenerationInFlight @@ -245,87 +258,121 @@ def test_triggering_a_run_requires_authentication(test_client: TestClient): # --- who owns a code wiki, and how many there are --------------------------- -@pytest.fixture -def wiki_account(test_db: Session) -> User: - """A wiki account that is not the requester, which is the real deployment.""" - from app.core.security import get_password_hash - - account = User( - user_name="wiki-bot", - password_hash=get_password_hash("irrelevant"), - email="wiki-bot@example.com", - is_active=True, - git_info=None, - ) - test_db.add(account) - test_db.commit() - return account +# --- ownership and listing -------------------------------------------------- -def _point_wiki_account_at(monkeypatch: pytest.MonkeyPatch, user_id: int) -> None: - from app.core.wiki_config import wiki_settings - - monkeypatch.setattr(wiki_settings, "DEFAULT_USER_ID", user_id) +LIST_URL = "/api/knowledge-bases/code-wikis" -def test_a_code_wiki_belongs_to_the_wiki_account_not_the_requester( +def test_a_code_wiki_belongs_to_whoever_created_it( test_client: TestClient, auth_headers: dict[str, str], test_db: Session, test_user: User, - wiki_account: User, - monkeypatch: pytest.MonkeyPatch, kind_services_use_test_db, ): - """The account whose credentials clone the repository is the one the wiki - depends on. Attributing it to the requester would make a repository's - documentation disappear when one person leaves.""" + """Ownership is what makes the ordinary ACL apply. Filing it under a shared wiki + account instead left the knowledge base visible to nobody, which forced a second + authorisation rule that only the newly written endpoints ever consulted.""" from app.models.kind import Kind - _point_wiki_account_at(monkeypatch, wiki_account.id) - kb_id = _create_wiki(test_client, auth_headers) kind = test_db.get(Kind, kb_id) - assert kind.user_id == wiki_account.id - assert kind.user_id != test_user.id + assert kind.user_id == test_user.id -def test_asking_for_a_repository_that_already_has_a_wiki_returns_it( +def test_the_creator_can_read_the_wiki_they_just_created( test_client: TestClient, auth_headers: dict[str, str], - wiki_account: User, - monkeypatch: pytest.MonkeyPatch, kind_services_use_test_db, ): - """Two full generations and two indexes for one repository's documentation is - the thing worth preventing. 200 rather than 201 says which happened.""" - _point_wiki_account_at(monkeypatch, wiki_account.id) - first_id = _create_wiki(test_client, auth_headers) + """The failure this pins is not hypothetical: under the previous owner the + creator was refused their own wiki's navigation immediately after creating it.""" + kb_id = _create_wiki(test_client, auth_headers) - with patch( - "app.api.endpoints.knowledge.assert_user_can_read_source", - return_value={"has_access": True}, - ): - again = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + response = test_client.get( + f"/api/knowledge-bases/{kb_id}/code-wiki/pages", headers=auth_headers + ) - assert again.status_code == 200, again.text - assert again.json()["id"] == first_id + assert response.status_code == 200, response.text + + +def test_a_code_wiki_appears_in_the_general_knowledge_base_list( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """It is a knowledge base. Excluding it from the general list would also take it + out of chat citation and the MCP tool, where being citable is the point.""" + kb_id = _create_wiki(test_client, auth_headers) + + grouped = test_client.get( + "/api/knowledge-bases/all-grouped", headers=auth_headers + ).json() + + listed = [ + kb["id"] + for bucket in ( + grouped["personal"]["created_by_me"], + grouped["personal"]["shared_with_me"], + ) + for kb in bucket + ] + assert kb_id in listed + + +def test_the_list_shows_a_wiki_the_caller_owns( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + kb_id = _create_wiki(test_client, auth_headers) + + response = test_client.get(LIST_URL, headers=auth_headers) + assert response.status_code == 200, response.text + body = response.json() + assert [item["id"] for item in body["items"]] == [kb_id] + assert body["items"][0]["project_name"] == "wecode-ai/Wegent" -def test_the_repository_is_registered_so_the_database_can_refuse_a_second_wiki( + +def test_the_list_hides_another_user_s_wiki( test_client: TestClient, auth_headers: dict[str, str], test_db: Session, - wiki_account: User, - monkeypatch: pytest.MonkeyPatch, kind_services_use_test_db, ): - """source_url is UNIQUE, which is what holds when two requests race — a check - against a JSON field would leave a window exactly where it matters.""" + """Nothing special to code wikis — this is the knowledge-base ACL doing its job, + and the point of the change is that it is the only rule in play.""" + from app.core.security import create_access_token, get_password_hash + + _create_wiki(test_client, auth_headers) + stranger = User( + user_name="stranger", + password_hash=get_password_hash("irrelevant"), + email="stranger@example.com", + is_active=True, + ) + test_db.add(stranger) + test_db.commit() + token = create_access_token(data={"sub": stranger.user_name}) + + response = test_client.get(LIST_URL, headers={"Authorization": f"Bearer {token}"}) + + assert response.json() == {"items": [], "total": 0} + + +def test_the_repository_is_registered_against_the_wiki_that_documents_it( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """One row per (repository, wiki). The composite UNIQUE is what settles two + requests racing for the same pair, which a check on a JSON field could not.""" from app.models.wiki import WikiProject - _point_wiki_account_at(monkeypatch, wiki_account.id) kb_id = _create_wiki(test_client, auth_headers) project = test_db.query(WikiProject).one() @@ -333,121 +380,133 @@ def test_the_repository_is_registered_so_the_database_can_refuse_a_second_wiki( assert project.source_url == "https://github.com/wecode-ai/Wegent.git" -def test_a_missing_wiki_account_is_reported_rather_than_silently_reassigned( +def test_asking_twice_for_the_same_repository_returns_the_caller_s_own_wiki( test_client: TestClient, auth_headers: dict[str, str], - monkeypatch: pytest.MonkeyPatch, kind_services_use_test_db, ): - """Falling back to the requester would create a wiki owned by someone the - operator did not choose, with credentials that may not reach the repository.""" - _point_wiki_account_at(monkeypatch, 987654) + """Answering with the existing one, 200 rather than 201, says which happened.""" + first_id = _create_wiki(test_client, auth_headers) with patch( "app.api.endpoints.knowledge.assert_user_can_read_source", return_value={"has_access": True}, ): - response = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + again = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + + assert again.status_code == 200, again.text + assert again.json()["id"] == first_id + + +def test_another_user_may_build_their_own_wiki_of_the_same_repository( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """The first wiki is invisible to the second caller under its owner's ACL, so + refusing them one of their own would take it away on a first-come basis.""" + from app.core.security import create_access_token, get_password_hash - assert response.status_code == 500 - assert "WIKI_DEFAULT_USER_ID" in response.json()["detail"] + first_id = _create_wiki(test_client, auth_headers) + other = User( + user_name="colleague", + password_hash=get_password_hash("irrelevant"), + email="colleague@example.com", + is_active=True, + ) + test_db.add(other) + test_db.commit() + token = create_access_token(data={"sub": other.user_name}) + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + response = test_client.post( + CREATE_URL, json=PAYLOAD, headers={"Authorization": f"Bearer {token}"} + ) -# --- listing ---------------------------------------------------------------- + assert response.status_code == 201, response.text + assert response.json()["id"] != first_id -LIST_URL = "/api/knowledge-bases/code-wikis" +# --- who may trigger a run -------------------------------------------------- -def test_the_list_shows_a_wiki_whose_repository_the_caller_can_read( +def test_regenerating_requires_write_access_to_the_repository( test_client: TestClient, auth_headers: dict[str, str], - wiki_account: User, - monkeypatch: pytest.MonkeyPatch, kind_services_use_test_db, ): - """The wiki belongs to the wiki account, so nothing in the knowledge-base ACL - would show it — the repository is what does.""" - _point_wiki_account_at(monkeypatch, wiki_account.id) + """A run rewrites every page, so reading the wiki is not enough. Without this a + wiki shared with a reader lets them spend a generation on somebody else's + knowledge base.""" kb_id = _create_wiki(test_client, auth_headers) - with patch("app.api.endpoints.knowledge.readable_wiki_ids", return_value={kb_id}): - response = test_client.get(LIST_URL, headers=auth_headers) + with patch( + "app.api.endpoints.knowledge.assert_user_can_write_source", + side_effect=SourceAccessDenied("read access but not write access"), + ): + response = test_client.post(_run_url(kb_id), json={}, headers=auth_headers) - assert response.status_code == 200, response.text - body = response.json() - assert body["total"] == 1 - assert body["items"][0]["project_name"] == "wecode-ai/Wegent" + assert response.status_code == 403 + assert "write access" in response.json()["detail"] -def test_the_list_hides_a_wiki_whose_repository_the_caller_cannot_read( +def test_a_reader_of_the_repository_cannot_regenerate( test_client: TestClient, auth_headers: dict[str, str], - wiki_account: User, - monkeypatch: pytest.MonkeyPatch, kind_services_use_test_db, ): - _point_wiki_account_at(monkeypatch, wiki_account.id) - _create_wiki(test_client, auth_headers) + """The threshold, not just the presence of a gate: read access reports an access + level below Developer, and that has to be refused rather than rounded up.""" + from app.services.knowledge.code_wiki import source as source_module - with patch("app.api.endpoints.knowledge.readable_wiki_ids", return_value=set()): - response = test_client.get(LIST_URL, headers=auth_headers) + kb_id = _create_wiki(test_client, auth_headers) - assert response.json() == {"items": [], "total": 0} + with patch.object( + source_module, + "assert_user_can_read_source", + return_value={"has_access": True, "access_level": 10}, + ): + response = test_client.post(_run_url(kb_id), json={}, headers=auth_headers) + + assert response.status_code == 403 -def test_a_code_wiki_stays_out_of_the_general_knowledge_base_list( +def test_creating_a_wiki_starts_its_first_run( test_client: TestClient, auth_headers: dict[str, str], - wiki_account: User, - monkeypatch: pytest.MonkeyPatch, kind_services_use_test_db, ): - """Stated by the scope rather than left to ownership: relying on the account not - matching holds only until an administrator looks, or the account itself does.""" - _point_wiki_account_at(monkeypatch, wiki_account.id) - kb_id = _create_wiki(test_client, auth_headers) + """Otherwise a new wiki sits empty until somebody finds the regenerate button, + which is not a flow anyone would guess.""" + with patch("app.api.endpoints.knowledge.start_run") as start: + _create_wiki(test_client, auth_headers) - grouped = test_client.get( - "/api/knowledge-bases/all-grouped", headers=auth_headers - ).json() - - listed = [ - kb["id"] - for bucket in ( - grouped["personal"]["created_by_me"], - grouped["personal"]["shared_with_me"], - ) - for kb in bucket - ] - assert kb_id not in listed + assert start.call_count == 1 -def test_creation_is_refused_when_the_wiki_account_cannot_clone( +def test_a_first_run_that_cannot_start_still_leaves_the_wiki_created( test_client: TestClient, auth_headers: dict[str, str], - test_user: User, - wiki_account: User, - monkeypatch: pytest.MonkeyPatch, kind_services_use_test_db, ): - """The requester's access is not the one that matters at clone time. Without - this the mismatch surfaces as a failed generation, and nothing in that failure - says the account just needs adding to the repository.""" - _point_wiki_account_at(monkeypatch, wiki_account.id) - - def only_the_requester(db, user_id, source): - if user_id == test_user.id: - return {"has_access": True} - raise SourceAccessDenied("not a member") + """The knowledge base is already committed. Reporting the creation as failed + would be untrue, and the reader's own button still starts a run.""" + from app.services.knowledge.code_wiki.runner import CodeWikiRunError with patch( - "app.api.endpoints.knowledge.assert_user_can_read_source", - side_effect=only_the_requester, + "app.api.endpoints.knowledge.start_run", + side_effect=CodeWikiRunError("no team configured"), ): - response = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) - - assert response.status_code == 403 - detail = response.json()["detail"] - assert "wiki-bot" in detail - assert "WIKI_DEFAULT_USER_ID=0" in detail + kb_id = _create_wiki(test_client, auth_headers) + + assert kb_id > 0 + assert ( + test_client.get( + f"/api/knowledge-bases/{kb_id}/code-wiki/pages", headers=auth_headers + ).status_code + == 200 + ) diff --git a/backend/tests/services/knowledge/code_wiki/test_read_access.py b/backend/tests/services/knowledge/code_wiki/test_read_access.py deleted file mode 100644 index f1f7bf29a7..0000000000 --- a/backend/tests/services/knowledge/code_wiki/test_read_access.py +++ /dev/null @@ -1,212 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Weibo, Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for deciding who may read a code wiki. - -A code wiki belongs to the wiki account, so knowledge-base ACLs grant nobody else -anything and this is the only check standing between a reader and the wiki. That -makes both directions worth pinning: it has to let repository members in without a -round trip per read, and it has to stay shut when the answer cannot be obtained. -""" - -from unittest.mock import patch - -import pytest -from sqlalchemy.orm import Session - -from app.models.kind import Kind -from app.models.user import User -from app.services.knowledge.code_wiki.read_access import may_read_code_wiki -from app.services.knowledge.code_wiki.source import SourceAccessDenied - -SOURCE = { - "sourceType": "github", - "sourceUrl": "https://github.com/wecode-ai/Wegent.git", - "sourceDomain": "github.com", - "projectName": "wecode-ai/Wegent", -} - - -@pytest.fixture -def code_wiki(test_db: Session, test_user: User) -> Kind: - kind = Kind( - kind="KnowledgeBase", - name="kb-read-access", - namespace="default", - user_id=99999, # the wiki account, not the reader - json={"spec": {"name": "wiki", "kbType": "code_wiki", "source": SOURCE}}, - is_active=True, - ) - test_db.add(kind) - test_db.flush() - return kind - - -def _with_cache(entries): - """Stand in for the repository list the provider layer keeps in Redis.""" - - async def reader(user, git_domain): - return entries - - # staticmethod: attached to a class it would bind and receive self, and the - # resulting TypeError would be swallowed as "no cache" — the test would pass for - # the wrong reason on the refusal cases and fail here. - return patch( - "app.services.knowledge.code_wiki.read_access.provider_for", - return_value=type( - "P", (), {"_get_all_repositories_from_cache": staticmethod(reader)} - )(), - ) - - -def _live_check(allowed: bool): - if allowed: - return patch( - "app.services.knowledge.code_wiki.source.assert_user_can_read_source", - return_value={"has_access": True}, - ) - return patch( - "app.services.knowledge.code_wiki.source.assert_user_can_read_source", - side_effect=SourceAccessDenied("no access"), - ) - - -def test_a_repository_in_the_users_cache_grants_access( - test_db: Session, test_user: User, code_wiki: Kind -): - """The common path, and it costs one Redis read rather than a call per view.""" - with _with_cache([{"full_name": "wecode-ai/Wegent"}]): - assert may_read_code_wiki(test_db, test_user, code_wiki) - - -def test_the_match_ignores_case(test_db: Session, test_user: User, code_wiki: Kind): - with _with_cache([{"full_name": "WeCode-AI/Wegent"}]): - assert may_read_code_wiki(test_db, test_user, code_wiki) - - -def test_a_cold_cache_falls_through_to_asking_about_the_one_repository( - test_db: Session, test_user: User, code_wiki: Kind -): - """A user who has never opened the repository picker must not be locked out; - building their whole list here would make a first read pay for every repository - they can see.""" - with _with_cache(None), _live_check(allowed=True): - assert may_read_code_wiki(test_db, test_user, code_wiki) - - -def test_a_repository_the_user_cannot_read_is_refused( - test_db: Session, test_user: User, code_wiki: Kind -): - with _with_cache([{"full_name": "someone-else/other"}]), _live_check(allowed=False): - assert not may_read_code_wiki(test_db, test_user, code_wiki) - - -def test_an_unreachable_provider_does_not_become_an_open_door( - test_db: Session, test_user: User, code_wiki: Kind -): - with ( - _with_cache(None), - patch( - "app.services.knowledge.code_wiki.source.assert_user_can_read_source", - side_effect=RuntimeError("provider is down"), - ), - ): - assert not may_read_code_wiki(test_db, test_user, code_wiki) - - -def test_a_wiki_with_no_repository_is_refused( - test_db: Session, test_user: User, code_wiki: Kind -): - """Nothing to judge by. A wiki nobody can attribute to a repository is one - nobody should inherit access to.""" - code_wiki.json = {"spec": {"name": "wiki", "kbType": "code_wiki"}} - test_db.flush() - - assert not may_read_code_wiki(test_db, test_user, code_wiki) - - -# --- judging many wikis at once --------------------------------------------- - - -class CountingCache: - """Counts how many times the repository cache is actually read.""" - - def __init__(self, entries): - self.entries = entries - self.reads = 0 - - def patch(self): - async def reader(user, git_domain): - self.reads += 1 - return self.entries - - return patch( - "app.services.knowledge.code_wiki.read_access.provider_for", - return_value=type( - "P", (), {"_get_all_repositories_from_cache": staticmethod(reader)} - )(), - ) - - -def _wiki(test_db: Session, name: str, project: str) -> Kind: - kind = Kind( - kind="KnowledgeBase", - name=name, - namespace="default", - user_id=99999, - json={ - "spec": { - "name": name, - "kbType": "code_wiki", - "source": {**SOURCE, "projectName": project}, - } - }, - is_active=True, - ) - test_db.add(kind) - test_db.flush() - return kind - - -def test_many_wikis_on_one_host_cost_one_cache_read(test_db: Session, test_user: User): - """Judging each separately opens a fresh event loop and issues a fresh Redis - read every time, on a listing that grows with the number of repositories.""" - from app.services.knowledge.code_wiki.read_access import readable_wiki_ids - - wikis = [_wiki(test_db, f"kb-{index}", f"org/repo-{index}") for index in range(5)] - cache = CountingCache([{"full_name": f"org/repo-{index}"} for index in range(5)]) - - with cache.patch(): - readable = readable_wiki_ids(test_db, test_user, wikis) - - assert readable == {wiki.id for wiki in wikis} - assert cache.reads == 1 - - -def test_only_the_wikis_the_cache_could_not_answer_reach_the_provider( - test_db: Session, test_user: User -): - """A warm cache leaves nothing for the live check, which is why it runs last.""" - from app.services.knowledge.code_wiki.read_access import readable_wiki_ids - - known = _wiki(test_db, "known", "org/known") - unknown = _wiki(test_db, "unknown", "org/unknown") - cache = CountingCache([{"full_name": "org/known"}]) - asked: list[str] = [] - - def record(db, user_id, source): - asked.append(source.project_name) - raise SourceAccessDenied("no access") - - with ( - cache.patch(), - patch( - "app.services.knowledge.code_wiki.source.assert_user_can_read_source", - side_effect=record, - ), - ): - readable = readable_wiki_ids(test_db, test_user, [known, unknown]) - - assert readable == {known.id} - assert asked == ["org/unknown"] diff --git a/backend/tests/services/knowledge/test_content_scope.py b/backend/tests/services/knowledge/test_content_scope.py deleted file mode 100644 index 99a9a99bdd..0000000000 --- a/backend/tests/services/knowledge/test_content_scope.py +++ /dev/null @@ -1,204 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Weibo, Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for knowledge content query scopes and content ownership defaults.""" - -import pytest -from sqlalchemy.orm import Session - -from app.models.kind import Kind -from app.models.knowledge import ContentOrigin, KnowledgeDocument, KnowledgeFolder -from app.services.knowledge.content_scope import ( - CODE_TARGET_SOURCE_TYPE, - NO_FOLDER, - code_targets, - exclude_code_wikis, - generated_folders, - generated_wiki_pages, - only_code_wikis, - wiki_pages, -) - -KIND_ID = 4242 - - -def _add_document( - db: Session, - name: str, - *, - origin: str, - source_type: str = "text", - folder_id: int = 0, -) -> KnowledgeDocument: - document = KnowledgeDocument( - kind_id=KIND_ID, - name=name, - file_extension="md", - user_id=1, - origin=origin, - source_type=source_type, - folder_id=folder_id, - ) - db.add(document) - db.flush() - return document - - -def _add_folder(db: Session, name: str, *, origin: str) -> KnowledgeFolder: - folder = KnowledgeFolder(kind_id=KIND_ID, parent_id=0, name=name, origin=origin) - db.add(folder) - db.flush() - return folder - - -@pytest.fixture -def content(test_db: Session) -> dict[str, KnowledgeDocument]: - """A knowledge base holding every combination of ownership and target kind.""" - items = { - "generated_page": _add_document( - test_db, "architecture", origin=ContentOrigin.GENERATED.value - ), - "user_page": _add_document( - test_db, "our-gotchas", origin=ContentOrigin.USER.value - ), - "code_target": _add_document( - test_db, - "src/foo/bar.py", - origin=ContentOrigin.GENERATED.value, - source_type=CODE_TARGET_SOURCE_TYPE, - folder_id=NO_FOLDER, - ), - } - return items - - -def _names(query) -> set[str]: - return {document.name for document in query.all()} - - -def test_wiki_pages_excludes_code_targets(test_db: Session, content): - result = wiki_pages( - test_db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == KIND_ID) - ) - - assert _names(result) == {"architecture", "our-gotchas"} - - -def test_generated_wiki_pages_excludes_user_content_and_code_targets( - test_db: Session, content -): - result = generated_wiki_pages( - test_db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == KIND_ID) - ) - - assert _names(result) == {"architecture"} - - -def test_code_targets_returns_only_indexed_source_files(test_db: Session, content): - result = code_targets( - test_db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == KIND_ID) - ) - - assert _names(result) == {"src/foo/bar.py"} - - -def test_generated_folders_excludes_user_folders(test_db: Session): - _add_folder(test_db, "architecture", origin=ContentOrigin.GENERATED.value) - _add_folder(test_db, "notes", origin=ContentOrigin.USER.value) - - result = generated_folders( - test_db.query(KnowledgeFolder).filter(KnowledgeFolder.kind_id == KIND_ID) - ) - - assert {folder.name for folder in result.all()} == {"architecture"} - - -def test_content_defaults_to_user_owned(test_db: Session): - """Unmarked content must be treated as user-owned. - - Mislabelling generated content as user-owned only stops automatic cleanup, while - the reverse would let generation delete a person's documents. - """ - document = KnowledgeDocument( - kind_id=KIND_ID, name="uploaded", file_extension="pdf", user_id=1 - ) - folder = KnowledgeFolder(kind_id=KIND_ID, parent_id=0, name="uploads") - test_db.add_all([document, folder]) - test_db.flush() - - assert document.origin == ContentOrigin.USER.value - assert folder.origin == ContentOrigin.USER.value - - -def test_listing_documents_never_returns_code_targets(test_db: Session): - """The scope must be wired into the listing, not merely available to it. - - An earlier revision defined these scopes but left every production query - unfiltered, which reads as protection while providing none. - """ - _add_document(test_db, "architecture", origin=ContentOrigin.GENERATED.value) - _add_document( - test_db, - "src/main.py", - origin=ContentOrigin.GENERATED.value, - source_type=CODE_TARGET_SOURCE_TYPE, - folder_id=NO_FOLDER, - ) - - listed = wiki_pages( - test_db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == KIND_ID) - ).all() - - assert [document.name for document in listed] == ["architecture"] - - -# --- keeping code wikis out of general listings ----------------------------- - - -def _kb(db: Session, name: str, kb_type: str | None) -> Kind: - spec: dict = {"name": name} - if kb_type is not None: - spec["kbType"] = kb_type - kind = Kind( - kind="KnowledgeBase", - name=name, - namespace="default", - user_id=1, - json={"spec": spec}, - is_active=True, - ) - db.add(kind) - db.flush() - return kind - - -def test_a_knowledge_base_predating_kb_type_is_still_listed(test_db: Session): - """NULL compared against a literal is neither true nor false, so spelling it out - is what keeps every knowledge base created before kbType existed from vanishing - out of every listing at once.""" - legacy = _kb(test_db, "legacy", None) - - listed = exclude_code_wikis(test_db.query(Kind)).all() - - assert legacy in listed - - -def test_code_wikis_are_kept_out_of_general_listings(test_db: Session): - """An agent shown one through the MCP tool may write into it, and the next - publish deletes whatever it added.""" - notebook = _kb(test_db, "notes", "notebook") - wiki = _kb(test_db, "wegent-wiki", "code_wiki") - - listed = exclude_code_wikis(test_db.query(Kind)).all() - - assert notebook in listed - assert wiki not in listed - - -def test_the_dedicated_scope_returns_only_code_wikis(test_db: Session): - _kb(test_db, "notes", "notebook") - _kb(test_db, "legacy", None) - wiki = _kb(test_db, "wegent-wiki", "code_wiki") - - assert only_code_wikis(test_db.query(Kind)).all() == [wiki] From 5c949c460af690eb8a8e336ef5b63cf6bc720984 Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Tue, 4 Aug 2026 20:48:59 +0800 Subject: [PATCH 11/14] feat(knowledge): let a public repository have a wiki Creating a code wiki refused anyone without a token for the host, and the access check both providers use is membership-based and never looks at visibility. A repository anyone can read in a browser therefore could not be documented -- a wiki more closed than its own source. Two paths reach it now. Without a credential the repository is described anonymously, and being public is itself read access. With a credential that does not reach it, the refusal falls through to the same probe: "not a member" is also what both providers answer for a public repository nobody has joined, so a stale or unrelated token would otherwise make a world-readable repository undocumentable. A successful public probe is positive evidence obtained from the provider we just failed to reach with a credential, so this does not turn an unreachable provider into an open door: if it is down, the probe fails too and the refusal stands. The anonymous grant reports read access and no more, so it can never satisfy the gate on regenerating. Absent and private are reported identically throughout. GitHub answers 404 rather than 403 for a private repository an anonymous caller cannot see, and that is the behaviour to preserve: "not readable" is the whole answer a caller is entitled to. POST /code-wikis/resolve answers the three questions the create form would otherwise ask separately -- may I read it, what is its default branch, what is it called. The branch matters most: listing branches has no anonymous path, and taking the default from here means one need not be opened up. It also reports how many wikis already document the repository, so somebody can ask for a share instead of paying for a second generation. Anonymous GitHub requests are capped at 60 an hour per address, so results are cached for ten minutes, keyed by whether a credential was used -- the same repository legitimately answers differently with and without one. An unreadable result is deliberately not cached: a repository about to be granted to the caller should not stay unreadable because they asked one moment early. Co-Authored-By: Claude Opus 5 --- backend/app/api/endpoints/knowledge.py | 41 +++++ backend/app/repository/gitea_provider.py | 32 ++++ backend/app/repository/github_provider.py | 37 ++++ backend/app/repository/gitlab_provider.py | 43 +++++ backend/app/schemas/knowledge.py | 31 ++++ .../knowledge/code_wiki/resolution.py | 161 ++++++++++++++++++ .../services/knowledge/code_wiki/source.py | 90 ++++++++-- backend/tests/api/test_knowledge_code_wiki.py | 92 ++++++++++ .../knowledge/code_wiki/test_resolution.py | 135 +++++++++++++++ .../knowledge/code_wiki/test_source.py | 136 ++++++++++++++- 10 files changed, 777 insertions(+), 21 deletions(-) create mode 100644 backend/app/services/knowledge/code_wiki/resolution.py create mode 100644 backend/tests/services/knowledge/code_wiki/test_resolution.py diff --git a/backend/app/api/endpoints/knowledge.py b/backend/app/api/endpoints/knowledge.py index c855fc3614..352e1117ac 100644 --- a/backend/app/api/endpoints/knowledge.py +++ b/backend/app/api/endpoints/knowledge.py @@ -49,6 +49,8 @@ CodeWikiListResponse, CodeWikiPageNode, CodeWikiPageTree, + CodeWikiResolveRequest, + CodeWikiResolveResponse, CodeWikiRunCreate, CodeWikiRunResponse, DocumentContentUpdate, @@ -91,7 +93,9 @@ CODE_WIKI_NAMESPACE, claim_repository, existing_wiki_id, + wiki_count_for, ) +from app.services.knowledge.code_wiki.resolution import resolve_repository from app.services.knowledge.code_wiki.run_mode import ChangedPath from app.services.knowledge.code_wiki.runner import CodeWikiRunError, start_run from app.services.knowledge.code_wiki.source import ( @@ -516,6 +520,43 @@ def create_knowledge_base( ) +@router.post("/code-wikis/resolve", response_model=CodeWikiResolveResponse) +@trace_sync("resolve_code_wiki_source", "knowledge.api") +def resolve_code_wiki_source( + data: CodeWikiResolveRequest, + current_user: User = Depends(security.get_current_user), + db: Session = Depends(get_db), +): + """Describe a repository before a wiki is bound to it. + + Registered above ``/{knowledge_base_id}`` routes by being declared here: a + literal path segment must not be reachable only when it fails to parse as an id. + + Answers 200 with ``exists: false`` rather than 404 for a repository the caller + cannot read. This is a form assisting input, not an assertion that something is + missing, and the two cases are indistinguishable on purpose. + """ + try: + source = SourceRepository.from_url(data.source_type, data.source_url) + except SourceAccessDenied as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) + ) from e + + resolved = resolve_repository(db, current_user.id, source) + return CodeWikiResolveResponse( + exists=resolved.exists, + visibility=resolved.visibility, + default_branch=resolved.default_branch, + name=resolved.name, + description=resolved.description, + access=resolved.access, + # Counted even when the caller cannot read the repository: the number says + # the work already exists, not that they may see it. + existing_wiki_count=wiki_count_for(db, source.source_url), + ) + + @router.get("/code-wikis", response_model=CodeWikiListResponse) @trace_sync("list_code_wikis", "knowledge.api") def list_code_wikis( diff --git a/backend/app/repository/gitea_provider.py b/backend/app/repository/gitea_provider.py index 176dc716bb..a9b685f69b 100644 --- a/backend/app/repository/gitea_provider.py +++ b/backend/app/repository/gitea_provider.py @@ -918,3 +918,35 @@ def get_changed_files( for entry in (payload.get("files") or []) if entry.get("filename") ] + + def describe_repository( + self, token: str, git_domain: str, repo_name: str + ) -> Optional[Dict[str, Any]]: + """Metadata a caller needs before binding a repository, or ``None``. + + ``None`` means "not readable with what was supplied"; private and absent are + deliberately not told apart. + """ + api_base_url = self._get_api_base_url(git_domain) + headers = {"Accept": "application/json"} + if token: + headers["Authorization"] = f"token {self.decrypt_token(token)}" + try: + response = requests.get( + f"{api_base_url}/repos/{repo_name}", + headers=headers, + timeout=REPO_STATE_TIMEOUT_SECONDS, + ) + except requests.exceptions.RequestException as e: + self._log_domain_failure("describe repository", git_domain, e) + return None + if response.status_code != 200: + return None + + data = response.json() or {} + return { + "visibility": "private" if data.get("private") else "public", + "default_branch": data.get("default_branch") or "", + "name": data.get("full_name") or repo_name, + "description": data.get("description") or "", + } diff --git a/backend/app/repository/github_provider.py b/backend/app/repository/github_provider.py index 8cbe7efeda..ac1baa42d2 100644 --- a/backend/app/repository/github_provider.py +++ b/backend/app/repository/github_provider.py @@ -1033,3 +1033,40 @@ def get_changed_files( for entry in files if entry.get("filename") ] + + def describe_repository( + self, token: str, git_domain: str, repo_name: str + ) -> Optional[Dict[str, Any]]: + """Metadata a caller needs before binding a repository, or ``None``. + + GitHub answers 404 rather than 403 for a private repository an anonymous + caller cannot see, which is the behaviour to preserve rather than unpick: + "not readable" is the whole answer a caller is entitled to. + + Anonymous requests are rate limited to 60 an hour per address, so callers + are expected to cache this rather than probe per keystroke. + """ + api_base_url = self._get_api_base_url(git_domain) + headers = {"Accept": "application/vnd.github.v3+json"} + if token: + headers["Authorization"] = f"token {self.decrypt_token(token)}" + try: + response = requests.get( + f"{api_base_url}/repos/{repo_name}", + headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + except requests.exceptions.RequestException as e: + self._log_domain_failure("describe repository", git_domain, e) + return None + if response.status_code != 200: + return None + + data = response.json() or {} + return { + "visibility": data.get("visibility") + or ("private" if data.get("private") else "public"), + "default_branch": data.get("default_branch") or "", + "name": data.get("full_name") or repo_name, + "description": data.get("description") or "", + } diff --git a/backend/app/repository/gitlab_provider.py b/backend/app/repository/gitlab_provider.py index 5a0f440882..1e0e6ffb34 100644 --- a/backend/app/repository/gitlab_provider.py +++ b/backend/app/repository/gitlab_provider.py @@ -1038,3 +1038,46 @@ def get_changed_files( status = "M" changed.append({"path": path, "status": status}) return changed + + def describe_repository( + self, token: str, git_domain: str, repo_name: str + ) -> Optional[Dict[str, Any]]: + """Metadata a caller needs before binding a repository, or ``None``. + + ``None`` means "not readable with what was supplied" and deliberately does + not distinguish private from absent: answering that would tell an anonymous + caller which private repositories exist. + + Works without a token, which is the point — a public repository is readable + by anyone, and requiring a credential to see that would make a wiki more + closed than the repository it documents. + """ + api_base_url = self._get_api_base_url(git_domain) + url = f"{api_base_url}/projects/{quote(repo_name, safe='')}" + try: + if token: + response = self._make_request_with_auth_retry( + method="GET", + url=url, + token=token, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + else: + response = requests.get( + url, + headers={"Accept": "application/json"}, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + if response.status_code != 200: + return None + except requests.exceptions.RequestException as e: + self._log_domain_failure("describe repository", git_domain, e) + return None + + data = response.json() or {} + return { + "visibility": data.get("visibility") or "private", + "default_branch": data.get("default_branch") or "", + "name": data.get("path_with_namespace") or repo_name, + "description": data.get("description") or "", + } diff --git a/backend/app/schemas/knowledge.py b/backend/app/schemas/knowledge.py index eda939e0d1..c456d453a5 100644 --- a/backend/app/schemas/knowledge.py +++ b/backend/app/schemas/knowledge.py @@ -431,6 +431,37 @@ class CodeWikiRunResponse(BaseModel): task_id: int = Field(0, description="Task running the agent, when started") +class CodeWikiResolveRequest(BaseModel): + """Ask what is known about a repository before binding a wiki to it.""" + + source_type: Literal["github", "gitlab", "gitea"] + source_url: str = Field(..., min_length=1, max_length=500) + + +class CodeWikiResolveResponse(BaseModel): + """What the create form needs, in one answer rather than three probes. + + ``exists`` false means "not readable with what you have"; private and absent are + deliberately not told apart, since distinguishing them would disclose which + private repositories exist. + """ + + exists: bool + visibility: str = Field("", description="public or private, when readable") + default_branch: str = Field("", description="Saves listing branches to pick one") + name: str = Field("", description="Repository path, offered as the wiki's name") + description: str = Field("", description="Offered as the wiki's description") + access: str = Field("none", description="public, member, or none") + existing_wiki_count: int = Field( + 0, + description=( + "How many code wikis already document this repository, whoever owns " + "them. Shown so a caller can ask for a share instead of paying for a " + "second generation." + ), + ) + + class KnowledgeBaseTypeUpdate(BaseModel): """Schema for updating the default opening view. diff --git a/backend/app/services/knowledge/code_wiki/resolution.py b/backend/app/services/knowledge/code_wiki/resolution.py new file mode 100644 index 0000000000..4988d20015 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/resolution.py @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""What a caller needs to know about a repository before binding a wiki to it. + +One call answers three questions that would otherwise be three probes: + +1. **May this caller read it?** — which decides whether creating a wiki is allowed. +2. **What is its default branch?** — so the create form does not need a branch listing, + which has no anonymous path and would be a second thing to open up. +3. **What is it called, and what is it?** — used to fill in a name and description the + caller left blank. + +Repositories can be read without a credential when they are public, and that case has +to work: the repository selector only lists repositories the caller is a member of, so +a public repository is one they can read in a browser but cannot pick from a list. A +wiki that could not be built for it would be more closed than the thing it documents. + +Anonymous requests to GitHub are limited to 60 an hour per address, so results are +cached. The cache is keyed by whether a credential was used, because the same +repository legitimately answers differently with and without one. +""" + +import logging +from dataclasses import dataclass +from typing import Optional + +from sqlalchemy.orm import Session + +from app.core.cache import cache_manager +from app.services.git_skill.utils import get_user_git_info +from app.services.knowledge.code_wiki.source import ( + SUPPORTED_SOURCE_TYPES, + SourceAccessDenied, + SourceRepository, + provider_for, +) + +logger = logging.getLogger(__name__) + +# Long enough that a form filled in over several minutes costs one request, short +# enough that a repository turned private stops being reported as public within the +# hour. Visibility is a cache here, never a stored fact. +RESOLUTION_CACHE_SECONDS = 600 + + +@dataclass(frozen=True) +class ResolvedRepository: + """A repository the caller may read, described well enough to bind.""" + + exists: bool + visibility: str + default_branch: str + name: str + description: str + access: str # "public" | "member" | "none" + + +UNREADABLE = ResolvedRepository( + exists=False, + visibility="", + default_branch="", + name="", + description="", + access="none", +) + + +def resolve_repository( + db: Session, user_id: int, source: SourceRepository +) -> ResolvedRepository: + """Describe a repository as far as this caller is entitled to see it. + + Never raises for an unreadable repository: "you cannot read this" is an answer + the form displays, not an error. Absent and private are reported identically, + since telling them apart would disclose which private repositories exist. + """ + if source.source_type not in SUPPORTED_SOURCE_TYPES: + raise SourceAccessDenied( + f"Unsupported repository type '{source.source_type}'. " + f"Supported types: {', '.join(SUPPORTED_SOURCE_TYPES)}" + ) + + provider = provider_for(source.source_type) + if provider is None: # pragma: no cover - guarded by the check above + return UNREADABLE + + git_info = get_user_git_info(user_id=user_id, domain=source.source_domain, db=db) + token = (git_info or {}).get("token") or "" + + cached = _cached(source, has_token=bool(token)) + if cached is not None: + return cached + + described = provider.describe_repository( + token=token, + git_domain=source.source_domain, + repo_name=source.project_name, + ) + if not described: + # Not cached: a repository that is about to be granted to the caller should + # not stay unreadable for ten minutes because they asked one moment early. + return UNREADABLE + + resolved = ResolvedRepository( + exists=True, + visibility=str(described.get("visibility") or "private"), + default_branch=str(described.get("default_branch") or ""), + name=str(described.get("name") or source.project_name), + description=str(described.get("description") or ""), + access="member" if token else "public", + ) + _remember(source, has_token=bool(token), resolved=resolved) + return resolved + + +def _cache_key(source: SourceRepository, *, has_token: bool) -> str: + return ( + f"code_wiki:resolve:{source.source_type}:{source.source_domain}:" + f"{source.project_name}:{'auth' if has_token else 'anon'}" + ) + + +def _cached( + source: SourceRepository, *, has_token: bool +) -> Optional[ResolvedRepository]: + try: + payload = _run(cache_manager.get(_cache_key(source, has_token=has_token))) + except Exception as exc: # pragma: no cover - cache is an optimisation + logger.debug("[code_wiki] resolution cache read failed: %s", exc) + return None + if not isinstance(payload, dict): + return None + return ResolvedRepository(**payload) + + +def _remember( + source: SourceRepository, *, has_token: bool, resolved: ResolvedRepository +) -> None: + try: + _run( + cache_manager.set( + _cache_key(source, has_token=has_token), + resolved.__dict__, + expire=RESOLUTION_CACHE_SECONDS, + ) + ) + except Exception as exc: # pragma: no cover - cache is an optimisation + logger.debug("[code_wiki] resolution cache write failed: %s", exc) + + +def _run(coroutine): + """Await a cache coroutine from this synchronous path. + + The endpoint is synchronous, so there is no running loop to attach to; a fresh + one per call is the same shape the repository providers already use. + """ + import asyncio + + return asyncio.run(coroutine) diff --git a/backend/app/services/knowledge/code_wiki/source.py b/backend/app/services/knowledge/code_wiki/source.py index 71e95f0768..3171bb7d8f 100644 --- a/backend/app/services/knowledge/code_wiki/source.py +++ b/backend/app/services/knowledge/code_wiki/source.py @@ -198,6 +198,61 @@ def _check_access(provider, source_type: str, token: str, source: SourceReposito ) +def _anonymous_read_access(source: SourceRepository) -> Dict[str, Any]: + """Read access granted by the repository being public, or a refusal. + + The access level reported is read and no more: nobody has write access to a + repository they reached without a credential, so this can never satisfy the + write gate. + """ + provider = provider_for(source.source_type) + described = ( + provider.describe_repository( + token="", + git_domain=source.source_domain, + repo_name=source.project_name, + ) + if provider is not None + else None + ) + if not described or described.get("visibility") != "public": + raise SourceAccessDenied( + f"No credentials configured for {source.source_domain}, and " + f"'{source.project_name}' is not readable without one. Add a token for " + "this Git domain, or check the repository address." + ) + + logger.info( + "[code_wiki] user reached public repository %s without a credential", + source.project_name, + ) + return { + "has_access": True, + "access_level": 10, + "access_level_name": "Read", + "visibility": "public", + } + + +def _or_public(source: SourceRepository, refusal: SourceAccessDenied) -> Dict[str, Any]: + """Fall back to reading the repository as anyone would, or raise ``refusal``. + + Raising the original refusal rather than the public probe's own wording keeps + the more specific answer: "your token does not reach this" is more use than + "this is not public". + + Answering a *successful* public probe is not the same as answering an absent + one — it is positive evidence that the repository is world-readable, obtained + from the provider we just failed to reach with a credential. So this does not + turn an unreachable provider into an open door: if the provider is down, the + probe fails too and the refusal stands. + """ + try: + return _anonymous_read_access(source) + except SourceAccessDenied: + raise refusal from None + + def assert_user_can_read_source( db: Session, user_id: int, source: SourceRepository ) -> Dict[str, Any]: @@ -215,10 +270,11 @@ def assert_user_can_read_source( user_id=user_id, domain=source.source_domain, db=db ) if not git_info or not git_info.get("token"): - raise SourceAccessDenied( - f"No credentials configured for {source.source_domain}. " - "Add a token for this Git domain before creating a code wiki." - ) + # No credential is not the same as no access: a public repository is readable + # by anyone, and refusing here would make its wiki more closed than the + # repository. Asked anonymously, so an unreadable one is reported as such + # without disclosing whether it exists. + return _anonymous_read_access(source) # The credential's own type must agree with the declared one. Asking one provider # about a repository hosted by another produces a meaningless answer, so require @@ -238,7 +294,6 @@ def assert_user_can_read_source( try: result = _check_access(provider, source.source_type, git_info["token"], source) except Exception as exc: - # Deny on error: an unreachable provider must not become an open door. logger.warning( "[code_wiki] repository access check failed for %s: %s", source.project_name, @@ -247,15 +302,26 @@ def assert_user_can_read_source( # The provider's own error text is logged above but deliberately kept out of # the message, which reaches the client as a 403 body. It is an external # system's wording about our internal request, not an answer to the caller. - raise SourceAccessDenied( - f"Could not verify access to '{source.project_name}'. " - "Please try again later." - ) from exc + return _or_public( + source, + SourceAccessDenied( + f"Could not verify access to '{source.project_name}'. " + "Please try again later." + ), + ) if not result.get("has_access", False): - raise SourceAccessDenied( - f"You do not have read access to '{source.project_name}'. " - f"{result.get('error', '')}".strip() + # A credential that does not reach this repository is not the last word: the + # membership check both providers use answers "not a member", which is also + # the answer for a public repository the caller has never joined. Falling + # through to the public probe is what stops a stale or unrelated token from + # making a world-readable repository undocumentable. + return _or_public( + source, + SourceAccessDenied( + f"You do not have read access to '{source.project_name}'. " + f"{result.get('error', '')}".strip() + ), ) logger.info( diff --git a/backend/tests/api/test_knowledge_code_wiki.py b/backend/tests/api/test_knowledge_code_wiki.py index 7d92896c8c..ce28d5f6cd 100644 --- a/backend/tests/api/test_knowledge_code_wiki.py +++ b/backend/tests/api/test_knowledge_code_wiki.py @@ -510,3 +510,95 @@ def test_a_first_run_that_cannot_start_still_leaves_the_wiki_created( ).status_code == 200 ) + + +# --- resolving a repository before binding ---------------------------------- + + +RESOLVE_URL = "/api/knowledge-bases/code-wikis/resolve" +RESOLVE_PAYLOAD = { + "source_type": "github", + "source_url": "https://github.com/wecode-ai/Wegent.git", +} + + +def test_resolving_answers_with_what_the_create_form_needs( + test_client: TestClient, auth_headers: dict[str, str] +): + from app.services.knowledge.code_wiki.resolution import ResolvedRepository + + with patch( + "app.api.endpoints.knowledge.resolve_repository", + return_value=ResolvedRepository( + exists=True, + visibility="public", + default_branch="main", + name="wecode-ai/Wegent", + description="An agent operating system", + access="public", + ), + ): + response = test_client.post( + RESOLVE_URL, json=RESOLVE_PAYLOAD, headers=auth_headers + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["default_branch"] == "main" + assert body["access"] == "public" + assert body["name"] == "wecode-ai/Wegent" + + +def test_an_unreadable_repository_resolves_to_200_not_404( + test_client: TestClient, auth_headers: dict[str, str] +): + """This assists a form; it does not assert that something is missing. 404 would + also make private and absent distinguishable, which they must not be.""" + from app.services.knowledge.code_wiki.resolution import UNREADABLE + + with patch( + "app.api.endpoints.knowledge.resolve_repository", return_value=UNREADABLE + ): + response = test_client.post( + RESOLVE_URL, json=RESOLVE_PAYLOAD, headers=auth_headers + ) + + assert response.status_code == 200 + assert response.json()["exists"] is False + + +def test_resolving_reports_how_many_wikis_already_exist( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """Shown so somebody can ask for a share instead of paying for a second + generation. Counted across all owners: the point is that the work exists.""" + from app.services.knowledge.code_wiki.resolution import UNREADABLE + + _create_wiki(test_client, auth_headers) + + with patch( + "app.api.endpoints.knowledge.resolve_repository", return_value=UNREADABLE + ): + response = test_client.post( + RESOLVE_URL, json=RESOLVE_PAYLOAD, headers=auth_headers + ) + + assert response.json()["existing_wiki_count"] == 1 + + +def test_resolving_a_malformed_url_is_a_bad_request( + test_client: TestClient, auth_headers: dict[str, str] +): + response = test_client.post( + RESOLVE_URL, + json={"source_type": "github", "source_url": "not-a-url"}, + headers=auth_headers, + ) + + assert response.status_code == 400 + + +def test_resolving_requires_authentication(test_client: TestClient): + assert test_client.post(RESOLVE_URL, json=RESOLVE_PAYLOAD).status_code == 401 diff --git a/backend/tests/services/knowledge/code_wiki/test_resolution.py b/backend/tests/services/knowledge/code_wiki/test_resolution.py new file mode 100644 index 0000000000..1efcfa5948 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_resolution.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for describing a repository before a wiki is bound to it. + +The case worth pinning is the one the repository selector cannot reach: a public +repository the caller is not a member of. It never appears in the list the selector +draws from, and until this existed it could not be documented at all — a wiki more +closed than the repository it describes. +""" + +from unittest.mock import patch + +import pytest + +from app.services.knowledge.code_wiki import resolution +from app.services.knowledge.code_wiki.resolution import resolve_repository +from app.services.knowledge.code_wiki.source import ( + SourceAccessDenied, + SourceRepository, +) + +PUBLIC = { + "visibility": "public", + "default_branch": "main", + "name": "wecode-ai/Wegent", + "description": "An agent operating system", +} + + +@pytest.fixture(autouse=True) +def _no_cache(monkeypatch): + """The cache is an optimisation; these tests are about what is resolved.""" + monkeypatch.setattr(resolution, "_cached", lambda source, has_token: None) + monkeypatch.setattr( + resolution, "_remember", lambda source, has_token, resolved: None + ) + + +@pytest.fixture +def source() -> SourceRepository: + return SourceRepository.from_url( + "github", "https://github.com/wecode-ai/Wegent.git" + ) + + +def _with(token: str | None, described): + """Patch the two collaborators: whose token, and what the provider answers.""" + provider = type("P", (), {"describe_repository": lambda self, **kw: described})() + return ( + patch.object( + resolution, + "get_user_git_info", + return_value={"token": token} if token else None, + ), + patch.object(resolution, "provider_for", return_value=provider), + ) + + +def test_a_public_repository_resolves_without_any_credential(source): + """The selector only lists repositories the caller belongs to, so this is the + only way a public repository can be documented at all.""" + git_info, provider = _with(None, PUBLIC) + + with git_info, provider: + resolved = resolve_repository(db=None, user_id=1, source=source) + + assert resolved.exists is True + assert resolved.visibility == "public" + assert resolved.access == "public" + assert resolved.default_branch == "main" + + +def test_an_unreadable_repository_is_reported_without_saying_why(source): + """Private and absent must look alike: telling them apart would disclose which + private repositories exist.""" + git_info, provider = _with(None, None) + + with git_info, provider: + resolved = resolve_repository(db=None, user_id=1, source=source) + + assert resolved.exists is False + assert resolved.visibility == "" + assert resolved.access == "none" + + +def test_a_credential_is_used_when_the_caller_has_one(source): + git_info, provider = _with("ghp_token", {**PUBLIC, "visibility": "private"}) + + with git_info, provider: + resolved = resolve_repository(db=None, user_id=1, source=source) + + assert resolved.access == "member" + assert resolved.visibility == "private" + + +def test_the_default_branch_comes_back_so_branches_need_not_be_listed(source): + """Listing branches has no anonymous path; taking the default from here is what + lets the create form work for a public repository.""" + git_info, provider = _with(None, {**PUBLIC, "default_branch": "develop"}) + + with git_info, provider: + assert resolve_repository(db=None, user_id=1, source=source).default_branch == ( + "develop" + ) + + +def test_an_unsupported_type_is_refused_rather_than_probed(source): + other = SourceRepository( + source_type="svn", + source_url="https://example.com/x/y.git", + project_name="x/y", + source_domain="example.com", + ) + + with pytest.raises(SourceAccessDenied): + resolve_repository(db=None, user_id=1, source=other) + + +def test_an_unreadable_result_is_not_cached(source, monkeypatch): + """A repository about to be granted to the caller must not stay unreadable for + the length of the cache because they asked one moment early.""" + remembered = [] + monkeypatch.setattr( + resolution, + "_remember", + lambda source, has_token, resolved: remembered.append(resolved), + ) + git_info, provider = _with(None, None) + + with git_info, provider: + resolve_repository(db=None, user_id=1, source=source) + + assert remembered == [] diff --git a/backend/tests/services/knowledge/code_wiki/test_source.py b/backend/tests/services/knowledge/code_wiki/test_source.py index e8446e2bf6..ad07b7b192 100644 --- a/backend/tests/services/knowledge/code_wiki/test_source.py +++ b/backend/tests/services/knowledge/code_wiki/test_source.py @@ -69,12 +69,49 @@ def test_gitlab_is_identified_by_project_id(): ) -def test_missing_credentials_are_denied(): - with patch( - "app.services.knowledge.code_wiki.source.get_user_git_info", - return_value=None, +def test_a_public_repository_is_readable_without_any_credential(): + """The repository selector only lists repositories the caller belongs to, so + refusing here would leave a publicly readable repository undocumentable — a wiki + more closed than its own source.""" + provider = MagicMock() + provider.describe_repository.return_value = { + "visibility": "public", + "default_branch": "main", + "name": "wecode-ai/Wegent", + "description": "", + } + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value=None, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), ): - with pytest.raises(SourceAccessDenied, match="No credentials configured"): + result = assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + assert result["has_access"] is True + # Read and no more: nobody has write access to a repository they reached without + # a credential, so this can never satisfy the gate on regenerating. + assert result["access_level"] == 10 + + +def test_a_private_repository_without_a_credential_is_denied(): + provider = MagicMock() + provider.describe_repository.return_value = None + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value=None, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="not readable without one"): assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) @@ -238,9 +275,90 @@ def test_a_self_hosted_host_on_an_internal_network_is_still_allowed(url: str): def test_a_credential_record_without_a_usable_token_denies_access(git_info): """Every shape of "configured but unusable" has to deny. A record that exists but carries no token would otherwise reach the provider with ``None``.""" - with patch( - "app.services.knowledge.code_wiki.source.get_user_git_info", - return_value=git_info, + provider = MagicMock() + provider.describe_repository.return_value = None + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value=git_info, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="not readable without one"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_a_public_repository_survives_a_token_that_does_not_reach_it(): + """Both providers answer membership questions, and "not a member" is also the + answer for a public repository nobody has joined. Without this fallback a stale + or unrelated token makes a world-readable repository undocumentable.""" + provider = MagicMock() + provider.check_user_project_access.return_value = { + "has_access": False, + "error": "Not a member", + } + provider.describe_repository.return_value = { + "visibility": "public", + "default_branch": "main", + "name": "wecode-ai/Wegent", + "description": "", + } + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"token": "stale", "type": "github"}, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), ): - with pytest.raises(SourceAccessDenied, match="No credentials configured"): + result = assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + assert result["has_access"] is True + assert result["access_level"] == 10 + + +def test_a_private_repository_a_token_cannot_reach_is_still_denied(): + """The fallback must not become an open door: when the public probe also says + no, the original and more specific refusal is what surfaces.""" + provider = MagicMock() + provider.check_user_project_access.return_value = { + "has_access": False, + "error": "Not a member", + } + provider.describe_repository.return_value = None + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"token": "stale", "type": "github"}, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="do not have read access"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_an_unreachable_provider_is_still_denied(): + """A provider that is down fails the public probe too, so the refusal stands.""" + provider = MagicMock() + provider.check_user_project_access.side_effect = RuntimeError("connection refused") + provider.describe_repository.return_value = None + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"token": "t0ken", "type": "github"}, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="Could not verify access"): assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) From ec7fd12071b0002f7e0cd20d73b07990ed0be075 Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Tue, 4 Aug 2026 21:15:37 +0800 Subject: [PATCH 12/14] feat(knowledge): create a code wiki from the ordinary knowledge base dialog A code wiki is a knowledge base owned by its creator, so it belongs in the same list and the same create dialog as every other one. The separate document/code tabs went with that: they were the visible half of a separation that no longer exists, and the code tab's own list and dialog are removed rather than kept alongside. The dialog asks which kind first, because that decides which fields even apply. A code wiki has no notebook/classic opening view -- it has a reader of its own -- and it does have a repository, a branch and a generation language. Its name is optional: left blank, the server fills in the repository's own. The blank is deliberately not pre-filled in the browser, where it would read as the caller's own input. The repository can be picked or named. Both are needed: the selector's list is membership-scoped on the server, so a public repository the caller can read in a browser never appears in it, and one that could not be documented would leave the wiki more closed than its source. The shared RepositorySelector is embedded rather than extended -- task creation uses it too, and the scoping is not something a component can change. Naming a URL resolves it, which reports whether it is readable, its default branch, and how many wikis already document it. The probe is debounced: anonymous GitHub requests are capped at 60 an hour per address. KnowledgeBaseType gains 'code_wiki', and with it DocumentViewType for the four components that render one of the two document views. They stay narrow on purpose: a code wiki has neither view, so handing one to them should not typecheck. documentViewOf makes each conversion visible. Removing the tabs took the sidebar-expand button that lived in them; the document page has its own (onExpandTree), so the affordance is not lost. Co-Authored-By: Claude Opus 5 --- .../code-wiki/create-kind-selection.test.tsx | 58 ++++ frontend/src/apis/code-wiki.ts | 18 ++ .../[kbName]/[[...docPath]]/page.tsx | 55 +--- frontend/src/app/(tasks)/knowledge/page.tsx | 196 +++---------- .../code-wiki/CodeWikiCreateDialog.tsx | 133 --------- .../knowledge/code-wiki/CodeWikiList.tsx | 122 -------- .../code-wiki/CodeWikiSourceFields.tsx | 265 ++++++++++++++++++ .../knowledge/code-wiki/useCodeWikis.ts | 49 ---- .../components/CreateKnowledgeBaseDialog.tsx | 166 ++++++++--- .../document/components/DocumentList.tsx | 5 +- .../document/components/KnowledgeBaseForm.tsx | 11 +- .../document/hooks/useKnowledgeBaseDialogs.ts | 21 ++ .../document/hooks/useKnowledgeSidebar.ts | 5 +- .../tasks/components/message/MessagesArea.tsx | 3 +- frontend/src/i18n/locales/en/knowledge.json | 21 +- .../src/i18n/locales/zh-CN/knowledge.json | 21 +- frontend/src/types/code-wiki.ts | 37 ++- frontend/src/types/knowledge.ts | 29 +- 18 files changed, 635 insertions(+), 580 deletions(-) create mode 100644 frontend/src/__tests__/features/knowledge/code-wiki/create-kind-selection.test.tsx delete mode 100644 frontend/src/features/knowledge/code-wiki/CodeWikiCreateDialog.tsx delete mode 100644 frontend/src/features/knowledge/code-wiki/CodeWikiList.tsx create mode 100644 frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx delete mode 100644 frontend/src/features/knowledge/code-wiki/useCodeWikis.ts diff --git a/frontend/src/__tests__/features/knowledge/code-wiki/create-kind-selection.test.tsx b/frontend/src/__tests__/features/knowledge/code-wiki/create-kind-selection.test.tsx new file mode 100644 index 0000000000..3ff14c9936 --- /dev/null +++ b/frontend/src/__tests__/features/knowledge/code-wiki/create-kind-selection.test.tsx @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * The create dialog now covers both kinds, so what is pinned here is that choosing + * one does not leave the other's rules in force — most of all the name, which is + * required for a document knowledge base and deliberately optional for a code wiki. + */ + +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { CreateKnowledgeBaseDialog } from '@/features/knowledge/document/components/CreateKnowledgeBaseDialog' + +jest.mock('@/hooks/useTranslation', () => ({ + useTranslation: () => ({ t: (key: string) => key }), +})) + +jest.mock('@/apis/code-wiki', () => ({ + codeWikiApi: { resolve: jest.fn().mockResolvedValue(null) }, +})) + +jest.mock('@/features/tasks/components/selector', () => ({ + RepositorySelector: () =>
    , +})) + +describe('CreateKnowledgeBaseDialog kind selection', () => { + it('starts on documents, where a name is required', async () => { + const onSubmit = jest.fn() + render() + + fireEvent.click(screen.getByTestId('submit-create-kb')) + + await waitFor(() => expect(onSubmit).not.toHaveBeenCalled()) + }) + + it('swaps the opening-view field for repository fields when code is chosen', () => { + render() + + fireEvent.click(screen.getByTestId('create-kb-kind-code')) + + // A code wiki has no notebook/classic choice: it has a reader of its own. + expect(screen.queryByTestId('switch-kb-type')).not.toBeInTheDocument() + expect(screen.getByTestId('repository-selector')).toBeInTheDocument() + expect(screen.getByTestId('code-wiki-language')).toBeInTheDocument() + }) + + it('offers both ways to name a repository', () => { + render() + + fireEvent.click(screen.getByTestId('create-kb-kind-code')) + + // The selector alone is not enough: its list is membership-scoped, so a public + // repository the caller can read in a browser never appears in it. + expect(screen.getByTestId('code-wiki-source-mode-select')).toBeInTheDocument() + fireEvent.click(screen.getByTestId('code-wiki-source-mode-url')) + expect(screen.getByTestId('code-wiki-source-url')).toBeInTheDocument() + }) +}) diff --git a/frontend/src/apis/code-wiki.ts b/frontend/src/apis/code-wiki.ts index f594d8c885..d63a80be9e 100644 --- a/frontend/src/apis/code-wiki.ts +++ b/frontend/src/apis/code-wiki.ts @@ -6,7 +6,9 @@ import type { CodeWikiCreateRequest, CodeWikiListResponse, CodeWikiPageTree, + CodeWikiResolution, CodeWikiRunResponse, + CodeWikiSourceType, CodeWikiSummary, } from '@/types/code-wiki' import client from './client' @@ -28,6 +30,22 @@ export const codeWikiApi = { ) }, + /** + * What is known about a repository, before deciding to bind a wiki to it. + * + * Answers 200 with `exists: false` for one the caller cannot read: this assists + * the form rather than asserting something is missing, and a 404 would also make + * private and absent distinguishable, which they must not be. + */ + resolve: async ( + source_type: CodeWikiSourceType, + source_url: string + ): Promise => + client.post('/knowledge-bases/code-wikis/resolve', { + source_type, + source_url, + }), + /** * Bind a repository and create its wiki, or return the one it already has. * diff --git a/frontend/src/app/(tasks)/knowledge/[namespace]/[kbName]/[[...docPath]]/page.tsx b/frontend/src/app/(tasks)/knowledge/[namespace]/[kbName]/[[...docPath]]/page.tsx index 721be12753..b3ca981c80 100644 --- a/frontend/src/app/(tasks)/knowledge/[namespace]/[kbName]/[[...docPath]]/page.tsx +++ b/frontend/src/app/(tasks)/knowledge/[namespace]/[kbName]/[[...docPath]]/page.tsx @@ -21,7 +21,7 @@ * /knowledge/{namespace}/{kbName}/path/doc.md */ -import { Suspense, useState, useCallback, useEffect } from 'react' +import { Suspense, useState, useEffect } from 'react' import dynamic from 'next/dynamic' import { useParams, useRouter } from 'next/navigation' import { BookOpen, FileText } from 'lucide-react' @@ -42,7 +42,6 @@ import { useTaskSession } from '@/features/tasks/session/TaskSession' import { paths } from '@/config/paths' import { Spinner } from '@/components/ui/spinner' import { useWikiProjects } from '@/features/knowledge/useWikiProjects' -import { KnowledgeTabs } from '@/features/knowledge/KnowledgeTabs' import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' import { useTranslation } from '@/hooks/useTranslation' import type { KnowledgeView } from '@/types/knowledge' @@ -65,9 +64,6 @@ const KnowledgeDocumentPage = dynamic( { ssr: false } ) -// Storage key for knowledge sidebar collapsed state -const KNOWLEDGE_SIDEBAR_COLLAPSED_KEY = 'knowledge-sidebar-collapsed' - function KnowledgeVirtualPageContent() { const params = useParams() const router = useRouter() @@ -107,48 +103,11 @@ function KnowledgeVirtualPageContent() { // Mobile sidebar state const [isMobileSidebarOpen, setIsMobileSidebarOpen] = useState(false) - - // Knowledge sidebar collapsed state (for document tab) - const [isKnowledgeSidebarCollapsed, setIsKnowledgeSidebarCollapsed] = useState(() => { - if (typeof window !== 'undefined') { - return localStorage.getItem(KNOWLEDGE_SIDEBAR_COLLAPSED_KEY) === 'true' - } - return false - }) - const [knowledgeViewState, setKnowledgeViewState] = useState({ visible: false, currentView: 'notebook', }) - // Listen for knowledge sidebar collapse changes from KnowledgeDocumentPageDesktop - useEffect(() => { - const handleCollapseChange = (event: CustomEvent<{ collapsed: boolean }>) => { - setIsKnowledgeSidebarCollapsed(event.detail.collapsed) - } - - window.addEventListener( - 'knowledge-sidebar-collapse-change', - handleCollapseChange as EventListener - ) - - return () => { - window.removeEventListener( - 'knowledge-sidebar-collapse-change', - handleCollapseChange as EventListener - ) - } - }, []) - - // Handle expanding the knowledge sidebar from TopNavigation - const handleExpandKnowledgeSidebar = useCallback(() => { - setIsKnowledgeSidebarCollapsed(false) - localStorage.setItem(KNOWLEDGE_SIDEBAR_COLLAPSED_KEY, 'false') - window.dispatchEvent( - new CustomEvent('knowledge-sidebar-collapse-change', { detail: { collapsed: false } }) - ) - }, []) - useEffect(() => { saveLastTab('wiki') }, []) @@ -228,18 +187,6 @@ function KnowledgeVirtualPageContent() { { - if (tab === 'code') { - router.push('/knowledge?type=code') - } - }} - isKnowledgeSidebarCollapsed={isKnowledgeSidebarCollapsed} - onExpandClick={handleExpandKnowledgeSidebar} - /> - } onMobileSidebarToggle={() => setIsMobileSidebarOpen(true)} isSidebarCollapsed={isTaskSidebarCollapsed} > diff --git a/frontend/src/app/(tasks)/knowledge/page.tsx b/frontend/src/app/(tasks)/knowledge/page.tsx index fae280eafb..64b1c3f878 100644 --- a/frontend/src/app/(tasks)/knowledge/page.tsx +++ b/frontend/src/app/(tasks)/knowledge/page.tsx @@ -4,9 +4,9 @@ 'use client' -import { Suspense, useState, useEffect, useCallback } from 'react' +import { Suspense, useState, useEffect } from 'react' import dynamic from 'next/dynamic' -import { useRouter, useSearchParams } from 'next/navigation' +import { useRouter } from 'next/navigation' import { BookOpen, FileText } from 'lucide-react' import TopNavigation from '@/features/layout/TopNavigation' import { @@ -25,14 +25,8 @@ import { useIsMobile } from '@/features/layout/hooks/useMediaQuery' import { useTaskSession } from '@/features/tasks/session/TaskSession' import { paths } from '@/config/paths' import { Spinner } from '@/components/ui/spinner' -import { CodeWikiCreateDialog } from '@/features/knowledge/code-wiki/CodeWikiCreateDialog' -import { CodeWikiList } from '@/features/knowledge/code-wiki/CodeWikiList' -import { useCodeWikis } from '@/features/knowledge/code-wiki/useCodeWikis' -import { SearchBox } from '@/features/knowledge/SearchBox' -import { KnowledgeTabs } from '@/features/knowledge/KnowledgeTabs' import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' import type { KnowledgeView } from '@/types/knowledge' -import type { KnowledgeTabType } from '@/features/knowledge/KnowledgeTabs' import type { KnowledgeViewState } from '@/features/knowledge/document/components/KnowledgeDocumentPage' import { useKnowledgeTaskSidebar } from '@/features/knowledge/document/hooks/useKnowledgeTaskSidebar' @@ -44,14 +38,10 @@ const KnowledgeDocumentPage = dynamic( { ssr: false } ) -// Storage key for knowledge sidebar collapsed state -const KNOWLEDGE_SIDEBAR_COLLAPSED_KEY = 'knowledge-sidebar-collapsed' - // Main knowledge page content with URL parameter support function KnowledgePageContent() { const { t } = useTranslation() const router = useRouter() - const searchParams = useSearchParams() const { selectTask } = useTaskSession() const isMobile = useIsMobile() const [knowledgeViewState, setKnowledgeViewState] = useState({ @@ -59,118 +49,44 @@ function KnowledgePageContent() { currentView: 'notebook', }) - // Get initial knowledge type tab from URL parameter - const getInitialKnowledgeTab = useCallback((): KnowledgeTabType => { - const type = searchParams.get('type') - if (type === 'code') return 'code' - return 'document' // default - }, [searchParams]) - - // The code tab now lists code wikis. useWikiProjects still drives the legacy - // wiki modal below; both go in PR5b, when the old service path is retired. - const codeWikis = useCodeWikis() - const [createWikiOpen, setCreateWikiOpen] = useState(false) - - // Active knowledge tab - initialized from URL - const [activeTab, setActiveTab] = useState(getInitialKnowledgeTab) - - // Search term for project list - const [mainSearchTerm, setMainSearchTerm] = useState('') - // Mobile sidebar state const [isMobileSidebarOpen, setIsMobileSidebarOpen] = useState(false) - // Knowledge sidebar collapsed state (for document tab) - // This is synced with KnowledgeDocumentPageDesktop via localStorage and custom events - const [isKnowledgeSidebarCollapsed, setIsKnowledgeSidebarCollapsed] = useState(() => { - if (typeof window !== 'undefined') { - return localStorage.getItem(KNOWLEDGE_SIDEBAR_COLLAPSED_KEY) === 'true' - } - return false - }) - - // Listen for knowledge sidebar collapse changes from KnowledgeDocumentPageDesktop - useEffect(() => { - const handleCollapseChange = (event: CustomEvent<{ collapsed: boolean }>) => { - setIsKnowledgeSidebarCollapsed(event.detail.collapsed) - } - - window.addEventListener( - 'knowledge-sidebar-collapse-change', - handleCollapseChange as EventListener - ) - - return () => { - window.removeEventListener( - 'knowledge-sidebar-collapse-change', - handleCollapseChange as EventListener - ) - } - }, []) - - // Handle expanding the knowledge sidebar from TopNavigation - const handleExpandKnowledgeSidebar = useCallback(() => { - setIsKnowledgeSidebarCollapsed(false) - localStorage.setItem(KNOWLEDGE_SIDEBAR_COLLAPSED_KEY, 'false') - // Dispatch event to notify KnowledgeDocumentPageDesktop - window.dispatchEvent( - new CustomEvent('knowledge-sidebar-collapse-change', { detail: { collapsed: false } }) - ) - }, []) - - // Handle knowledge type tab change with URL update - const handleTabChange = useCallback( - (tab: KnowledgeTabType) => { - setActiveTab(tab) - // Update URL - preserve other params like tab and group for document type - if (tab === 'code') { - router.replace('?type=code') - } else { - // For document tab, just set type=document (sub-tab state is managed by KnowledgeDocumentPage) - router.replace('?type=document') - } - }, - [router] - ) - - const knowledgeViewSwitcher = - activeTab === 'document' && knowledgeViewState.visible ? ( - knowledgeViewState.onViewChange?.(value as KnowledgeView)} - className="flex-shrink-0" - > - - - - - {t('knowledge:document.knowledgeBase.typeClassic')} - - - - - - {t('knowledge:document.knowledgeBase.typeNotebook')} - - - - - ) : null + const knowledgeViewSwitcher = knowledgeViewState.visible ? ( + knowledgeViewState.onViewChange?.(value as KnowledgeView)} + className="flex-shrink-0" + > + + + + + {t('knowledge:document.knowledgeBase.typeClassic')} + + + + + + {t('knowledge:document.knowledgeBase.typeNotebook')} + + + + + ) : null const isWorkspaceView = - activeTab === 'document' && - knowledgeViewState.visible && - knowledgeViewState.currentView === 'notebook' + knowledgeViewState.visible && knowledgeViewState.currentView === 'notebook' const { isCollapsed: isTaskSidebarCollapsed, toggle: handleToggleCollapsed } = useKnowledgeTaskSidebar({ isMobile, @@ -221,14 +137,6 @@ function KnowledgePageContent() { - } onMobileSidebarToggle={() => setIsMobileSidebarOpen(true)} isSidebarCollapsed={isTaskSidebarCollapsed} > @@ -237,43 +145,13 @@ function KnowledgePageContent() { {/* Content area based on active tab */} - {activeTab === 'code' && ( -
    - {/* Center search box - using shared component */} - - setCreateWikiOpen(true)} - onOpen={wiki => router.push(`/knowledge/code-wiki/${wiki.id}`)} - /> -
    - )} - {/* Document knowledge - no padding, full height */} - {activeTab === 'document' && ( + {
    - )} + }
    - - { - codeWikis.add(wiki) - router.push(`/knowledge/code-wiki/${wiki.id}`) - }} - /> ) } diff --git a/frontend/src/features/knowledge/code-wiki/CodeWikiCreateDialog.tsx b/frontend/src/features/knowledge/code-wiki/CodeWikiCreateDialog.tsx deleted file mode 100644 index 0bd8b8d191..0000000000 --- a/frontend/src/features/knowledge/code-wiki/CodeWikiCreateDialog.tsx +++ /dev/null @@ -1,133 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Weibo, Inc. -// -// SPDX-License-Identifier: Apache-2.0 - -'use client' - -import { useCallback, useState } from 'react' -import { toast } from 'sonner' -import { Button } from '@/components/ui/button' -import { - Dialog, - DialogContent, - DialogFooter, - DialogHeader, - DialogTitle, -} from '@/components/ui/dialog' -import { Input } from '@/components/ui/input' -import { Label } from '@/components/ui/label' -import { RepositorySelector } from '@/features/tasks/components/selector' -import { useTranslation } from '@/hooks/useTranslation' -import { codeWikiApi } from '@/apis/code-wiki' -import type { GitRepoInfo } from '@/types/api' -import type { CodeWikiSummary } from '@/types/code-wiki' - -interface CodeWikiCreateDialogProps { - open: boolean - onOpenChange: (open: boolean) => void - onCreated: (wiki: CodeWikiSummary) => void -} - -/** - * Three fields, and everything else defaults. - * - * No namespace: a code wiki belongs to the wiki account and who may read it is - * decided by its repository, so asking anyone to place it would be asking a - * question whose answer is ignored. - */ -export function CodeWikiCreateDialog({ open, onOpenChange, onCreated }: CodeWikiCreateDialogProps) { - const { t } = useTranslation() - const [repo, setRepo] = useState(null) - const [name, setName] = useState('') - const [submitting, setSubmitting] = useState(false) - - const handleRepoChange = useCallback((next: GitRepoInfo | null) => { - setRepo(next) - // Derived, not forced: `wecode-ai/Wegent` becomes `Wegent`, and anyone who - // wants something else can still type it. - setName(current => current || next?.git_repo?.split('/').pop() || '') - }, []) - - const reset = useCallback(() => { - setRepo(null) - setName('') - }, []) - - const handleSubmit = useCallback(async () => { - if (!repo) return - setSubmitting(true) - try { - const sourceType = repo.type === 'gitee' ? 'gitea' : repo.type - const wiki = await codeWikiApi.create({ - name: name.trim() || repo.git_repo.split('/').pop() || repo.git_repo, - source_type: sourceType, - source_url: repo.git_url, - }) - onCreated(wiki) - onOpenChange(false) - reset() - } catch (error) { - toast.error(error instanceof Error ? error.message : String(error)) - } finally { - setSubmitting(false) - } - }, [repo, name, onCreated, onOpenChange, reset]) - - return ( - - - - {t('knowledge:codeWiki.create.title')} - - -
    -
    - -
    - -
    -
    - -
    - - setName(event.target.value)} - placeholder={t('knowledge:codeWiki.create.namePlaceholder')} - disabled={submitting} - data-testid="code-wiki-create-name" - /> -
    -
    - - - - - -     -     - ) -} diff --git a/frontend/src/features/knowledge/code-wiki/CodeWikiList.tsx b/frontend/src/features/knowledge/code-wiki/CodeWikiList.tsx deleted file mode 100644 index 05717954c8..0000000000 --- a/frontend/src/features/knowledge/code-wiki/CodeWikiList.tsx +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Weibo, Inc. -// -// SPDX-License-Identifier: Apache-2.0 - -'use client' - -import { useMemo } from 'react' -import { BookOpen, GitBranch, Plus } from 'lucide-react' -import { Button } from '@/components/ui/button' -import { Spinner } from '@/components/ui/spinner' -import { useTranslation } from '@/hooks/useTranslation' -import type { CodeWikiSummary } from '@/types/code-wiki' - -interface CodeWikiListProps { - wikis: CodeWikiSummary[] - loading: boolean - error: string | null - searchTerm: string - onCreate: () => void - onOpen: (wiki: CodeWikiSummary) => void -} - -const formatWhen = (value?: string | null): string => { - if (!value) return '' - const parsed = new Date(value) - return Number.isNaN(parsed.getTime()) ? '' : parsed.toLocaleDateString() -} - -export function CodeWikiList({ - wikis, - loading, - error, - searchTerm, - onCreate, - onOpen, -}: CodeWikiListProps) { - const { t } = useTranslation() - - const visible = useMemo(() => { - const needle = searchTerm.trim().toLowerCase() - if (!needle) return wikis - return wikis.filter( - wiki => - wiki.name.toLowerCase().includes(needle) || wiki.project_name.toLowerCase().includes(needle) - ) - }, [wikis, searchTerm]) - - if (loading) { - return ( -
    - -
    - ) - } - - if (error) { - return

    {error}

    - } - - return ( -
    -
    - -
    - - {visible.length === 0 ? ( -
    - -

    {t('knowledge:codeWiki.list.empty')}

    -

    - {t('knowledge:codeWiki.list.emptyHint')} -

    -
    - ) : ( -
    - {visible.map(wiki => ( - - ))} -
    - )} -
    - ) -} diff --git a/frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx b/frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx new file mode 100644 index 0000000000..08aa300161 --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useCallback, useEffect, useMemo, useState } from 'react' +import { AlertCircle, CheckCircle2, Loader2, Users } from 'lucide-react' +import { Input } from '@/components/ui/input' +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select' +import { RepositorySelector } from '@/features/tasks/components/selector' +import { SimpleConfigRow } from '@/features/settings/components/team-edit/SimpleConfigLayout' +import { useTranslation } from '@/hooks/useTranslation' +import { codeWikiApi } from '@/apis/code-wiki' +import type { GitRepoInfo } from '@/types/api' +import type { CodeWikiResolution, CodeWikiSourceType } from '@/types/code-wiki' + +export interface CodeWikiSource { + source_type: CodeWikiSourceType + source_url: string + language: string + /** Set once resolved; the parent uses it to know the form is usable. */ + resolution: CodeWikiResolution | null +} + +interface CodeWikiSourceFieldsProps { + value: CodeWikiSource + onChange: (next: CodeWikiSource) => void +} + +const SOURCE_TYPES: CodeWikiSourceType[] = ['github', 'gitlab', 'gitea'] + +/** + * Infer which platform hosts a URL, or `null` when it cannot be told. + * + * A self-hosted GitLab or Gitea is not recognisable from its domain, so guessing + * would bind the wiki to a provider that answers meaninglessly about it. Returning + * null makes the caller ask instead. + */ +function inferSourceType(url: string): CodeWikiSourceType | null { + const host = url.match(/^https?:\/\/([^/]+)/)?.[1]?.toLowerCase() + if (!host) return null + if (host === 'github.com') return 'github' + if (host === 'gitlab.com') return 'gitlab' + if (host === 'gitea.com') return 'gitea' + return null +} + +/** + * Where the repository comes from, by picking one or by naming one. + * + * Both are needed. The selector lists only repositories the caller is a member of, + * so a public repository they can read in a browser never appears in it — and a wiki + * that could not be built for one would be more closed than the repository itself. + * + * The shared RepositorySelector is embedded rather than extended: it is used by task + * creation too, and its list is membership-scoped on the server, which is not + * something a component can change. + */ +export function CodeWikiSourceFields({ value, onChange }: CodeWikiSourceFieldsProps) { + const { t } = useTranslation() + const [mode, setMode] = useState<'select' | 'url'>('select') + const [repo, setRepo] = useState(null) + const [urlDraft, setUrlDraft] = useState('') + const [resolving, setResolving] = useState(false) + + const resolution = value.resolution + + const resolve = useCallback( + async (sourceType: CodeWikiSourceType, url: string) => { + if (!url) { + onChange({ ...value, source_url: '', resolution: null }) + return + } + setResolving(true) + try { + const resolved = await codeWikiApi.resolve(sourceType, url) + onChange({ + ...value, + source_type: sourceType, + source_url: url, + resolution: resolved, + }) + } catch { + // A failed probe is not a failed form: the caller may still submit, and the + // create call applies the same check authoritatively. + onChange({ ...value, source_type: sourceType, source_url: url, resolution: null }) + } finally { + setResolving(false) + } + }, + [onChange, value] + ) + + const handleRepoChange = useCallback( + (next: GitRepoInfo | null) => { + setRepo(next) + if (!next) { + onChange({ ...value, source_url: '', resolution: null }) + return + } + const sourceType = (next.type as CodeWikiSourceType) ?? 'github' + void resolve(sourceType, next.git_url) + }, + [onChange, resolve, value] + ) + + // Debounced so that typing a URL costs one probe rather than one per keystroke — + // anonymous GitHub requests are capped at 60 an hour per address. + useEffect(() => { + if (mode !== 'url') return + const inferred = inferSourceType(urlDraft) + const timer = setTimeout(() => { + void resolve(inferred ?? value.source_type, urlDraft.trim()) + }, 600) + return () => clearTimeout(timer) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [urlDraft, mode]) + + const inferred = useMemo(() => inferSourceType(urlDraft), [urlDraft]) + + return ( + <> + + {t('knowledge:codeWiki.create.repository')} * + + } + align="start" + > +
    +
    + {(['select', 'url'] as const).map(option => ( + + ))} +
    + + {mode === 'select' ? ( + + ) : ( +
    + {inferred === null && urlDraft.trim() !== '' && ( + + )} + setUrlDraft(e.target.value)} + placeholder="https://github.com/owner/repo" + data-testid="code-wiki-source-url" + /> +
    + )} + + +
    +     + + + + + + ) +} + +function SourceStatus({ + resolving, + resolution, +}: { + resolving: boolean + resolution: CodeWikiResolution | null +}) { + const { t } = useTranslation() + + if (resolving) { + return ( +

    + + {t('knowledge:codeWiki.create.checking')} +

    + ) + } + if (!resolution) return null + + if (!resolution.exists) { + return ( +

    + + {t('knowledge:codeWiki.create.notReadable')} +

    + ) + } + + return ( +
    +

    + + {resolution.visibility === 'public' + ? t('knowledge:codeWiki.create.publicRepository') + : t('knowledge:codeWiki.create.privateRepository')} + {resolution.default_branch ? ` · ${resolution.default_branch}` : ''} +

    + {resolution.existing_wiki_count > 0 && ( +

    + + {t('knowledge:codeWiki.create.alreadyBuilt', { + count: resolution.existing_wiki_count, + })} +

    + )} +
    + ) +} diff --git a/frontend/src/features/knowledge/code-wiki/useCodeWikis.ts b/frontend/src/features/knowledge/code-wiki/useCodeWikis.ts deleted file mode 100644 index a52d8195ff..0000000000 --- a/frontend/src/features/knowledge/code-wiki/useCodeWikis.ts +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Weibo, Inc. -// -// SPDX-License-Identifier: Apache-2.0 - -'use client' - -import { useCallback, useEffect, useState } from 'react' -import { codeWikiApi } from '@/apis/code-wiki' -import type { CodeWikiSummary } from '@/types/code-wiki' - -/** - * The code wikis the signed-in user can read. - * - * One request. The list carries the repository, the last publish and its commit - * straight from the knowledge base, so nothing here fans out per wiki — which is - * what the old wiki list did, one generations call per project. - */ -export function useCodeWikis() { - const [wikis, setWikis] = useState([]) - const [loading, setLoading] = useState(true) - const [error, setError] = useState(null) - - const load = useCallback(async () => { - setLoading(true) - setError(null) - try { - const response = await codeWikiApi.list() - setWikis(response.items) - } catch (caught) { - setError(caught instanceof Error ? caught.message : String(caught)) - } finally { - setLoading(false) - } - }, []) - - useEffect(() => { - void load() - }, [load]) - - const add = useCallback((wiki: CodeWikiSummary) => { - // Creating a wiki for a repository that already has one returns the existing - // wiki, so this must not append a duplicate. - setWikis(current => - current.some(existing => existing.id === wiki.id) ? current : [wiki, ...current] - ) - }, []) - - return { wikis, loading, error, reload: load, add } -} diff --git a/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx b/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx index 567aec1732..484a9fd963 100644 --- a/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx +++ b/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx @@ -5,7 +5,7 @@ 'use client' import { useState, useEffect } from 'react' -import { BookOpen, Database, User, Building2, Users, FileText } from 'lucide-react' +import { BookOpen, Code2, Database, User, Building2, Users, FileText } from 'lucide-react' import { Dialog, DialogContent, @@ -24,6 +24,10 @@ import { } from '@/components/ui/select' import { useTranslation } from '@/hooks/useTranslation' import { SimpleConfigRow } from '@/features/settings/components/team-edit/SimpleConfigLayout' +import { + CodeWikiSourceFields, + type CodeWikiSource, +} from '@/features/knowledge/code-wiki/CodeWikiSourceFields' import type { DirectAccessRequirement, KnowledgeBaseCreate, @@ -97,6 +101,13 @@ function createDefaultRetrievalConfig(): RetrievalConfigDraft { } } +/** Documents are uploaded and organised; code is generated from a repository. */ +type KnowledgeBaseKind = 'document' | 'code' + +function createEmptySource(): CodeWikiSource { + return { source_type: 'github', source_url: '', language: 'zh', resolution: null } +} + export function CreateKnowledgeBaseDialog({ open, onOpenChange, @@ -118,6 +129,10 @@ export function CreateKnowledgeBaseDialog({ useState('read') // Selected KB type (can be changed by user) const [selectedKbType, setSelectedKbType] = useState(initialKbType) + // Which kind of knowledge base is being created. Chosen first because it decides + // which fields even apply: a code wiki has a repository and no opening view. + const [kind, setKind] = useState('document') + const [source, setSource] = useState(createEmptySource) // Default enable summary for all KB types const [summaryEnabled, setSummaryEnabled] = useState(true) const [summaryModelRef, setSummaryModelRef] = useState(null) @@ -160,6 +175,8 @@ export function CreateKnowledgeBaseDialog({ useEffect(() => { if (open) { setSelectedKbType(initialKbType) + setKind('document') + setSource(createEmptySource()) setSelectedGroupId(defaultGroupId || 'personal') setDirectAccessRequirement('read') } @@ -175,11 +192,18 @@ export function CreateKnowledgeBaseDialog({ setSummaryModelError('') clearMultimodalError() - if (!name.trim()) { + // A code wiki may be left unnamed: the server fills in the repository's own + // name. Pre-filling the box here instead would read as the caller's own input. + if (!name.trim() && kind !== 'code') { setError(t('knowledge:document.knowledgeBase.nameRequired')) return } + if (kind === 'code' && !source.source_url) { + setError(t('knowledge:codeWiki.create.repositoryRequired')) + return + } + if (name.length > 100) { setError(t('knowledge:document.knowledgeBase.nameTooLong')) return @@ -219,13 +243,22 @@ export function CreateKnowledgeBaseDialog({ max_calls_per_conversation: maxCalls, exempt_calls_before_check: exemptCalls, selectedGroupId: showGroupSelector ? selectedGroupId : undefined, - kb_type: selectedKbType, + kb_type: kind === 'code' ? 'code_wiki' : selectedKbType, + ...(kind === 'code' + ? { + source_type: source.source_type, + source_url: source.source_url, + language: source.language, + } + : {}), }) setName('') setDescription('') setDirectAccessRequirement('read') // Reset selectedKbType and keep summaryEnabled as true setSelectedKbType(initialKbType) + setKind('document') + setSource(createEmptySource()) setSummaryEnabled(true) setSummaryModelRef(null) resetMultimodal() @@ -246,6 +279,8 @@ export function CreateKnowledgeBaseDialog({ setDirectAccessRequirement('read') // Reset selectedKbType and keep summaryEnabled as true setSelectedKbType(initialKbType) + setKind('document') + setSource(createEmptySource()) setSummaryEnabled(true) setSummaryModelRef(null) setSummaryModelError('') @@ -279,57 +314,100 @@ export function CreateKnowledgeBaseDialog({
    + {/* Which kind first: it decides which of the fields below even apply. */} +
    + {( + [ + ['document', FileText, 'knowledge:document.knowledgeBase.kindDocument'], + ['code', Code2, 'knowledge:document.knowledgeBase.kindCode'], + ] as const + ).map(([option, Icon, label]) => ( + + ))} +
    - {/* KB Type selector - subtle style */} - -
    -
    - -
    -
    + {kind === 'code' ? ( + + ) : ( + /* KB Type selector - subtle style */ + +
    +
    + +
    - {isNotebook ? ( - - ) : ( - - )} -
    -
    -
    - {isNotebook - ? t('knowledge:document.knowledgeBase.typeNotebook') - : t('knowledge:document.knowledgeBase.typeClassic')} +
    + {isNotebook ? ( + + ) : ( + + )}
    -
    - {isNotebook - ? t('knowledge:document.knowledgeBase.notebookDesc') - : t('knowledge:document.knowledgeBase.classicDesc')} +
    +
    + {isNotebook + ? t('knowledge:document.knowledgeBase.typeNotebook') + : t('knowledge:document.knowledgeBase.typeClassic')} +
    +
    + {isNotebook + ? t('knowledge:document.knowledgeBase.notebookDesc') + : t('knowledge:document.knowledgeBase.classicDesc')} +
    -
    - + + )} {/* Group selector - only show when showGroupSelector is true */} {showGroupSelector && availableGroups && availableGroups.length > 0 && ( !open && setViewingDoc(null)} document={viewingDoc} knowledgeBaseId={knowledgeBase.id} - kbType={knowledgeBase.kb_type} + kbType={documentViewOf(knowledgeBase.kb_type) ?? undefined} canEdit={viewingDoc ? canManageDocument(viewingDoc) : false} knowledgeBaseName={knowledgeBase.name} knowledgeBaseNamespace={knowledgeBase.namespace || 'default'} @@ -1569,7 +1570,7 @@ export function DocumentList({ onUploadComplete={handleUploadComplete} onTableAdd={handleTableAdd} onWebAdd={handleWebAdd} - kbType={knowledgeBase.kb_type} + kbType={documentViewOf(knowledgeBase.kb_type) ?? undefined} folderId={selectedUploadFolderId} folderOptions={folderOptions} onFolderChange={setSelectedUploadFolderId} diff --git a/frontend/src/features/knowledge/document/components/KnowledgeBaseForm.tsx b/frontend/src/features/knowledge/document/components/KnowledgeBaseForm.tsx index 04c421e3ec..ff3f0f80c7 100644 --- a/frontend/src/features/knowledge/document/components/KnowledgeBaseForm.tsx +++ b/frontend/src/features/knowledge/document/components/KnowledgeBaseForm.tsx @@ -32,6 +32,10 @@ import { useMultimodalFeatureEnabled } from '@/features/knowledge/multimodal/hoo interface KnowledgeBaseFormProps { typeSection?: ReactNode + /** False for a code wiki: left blank, the repository's own name is used. */ + nameRequired?: boolean + /** Overrides the placeholder, to say what a blank name will be filled in with. */ + namePlaceholder?: string name: string description: string onNameChange: (value: string) => void @@ -136,6 +140,8 @@ function FormSection({ export function KnowledgeBaseForm({ typeSection, + nameRequired = true, + namePlaceholder, name, description, onNameChange, @@ -305,7 +311,8 @@ export function KnowledgeBaseForm({ - {t('knowledge:document.knowledgeBase.name')} * + {t('knowledge:document.knowledgeBase.name')} + {nameRequired && *} } > @@ -313,7 +320,7 @@ export function KnowledgeBaseForm({ id="knowledge-name" value={name} onChange={e => onNameChange(e.target.value)} - placeholder={t('knowledge:document.knowledgeBase.namePlaceholder')} + placeholder={namePlaceholder ?? t('knowledge:document.knowledgeBase.namePlaceholder')} maxLength={100} data-testid="kb-name-input" className="bg-base" diff --git a/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts b/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts index 3c18c9e804..3f288e9a86 100644 --- a/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts +++ b/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts @@ -146,6 +146,27 @@ export function useKnowledgeBaseDialogs({ const kbType = data.kb_type || createKbType + // A code wiki is created through its own endpoint: it needs a repository, + // passes a repository-access gate, and starts generating straight away. + // Everything after that point is the same knowledge base as any other. + if (kbType === 'code_wiki') { + const { codeWikiApi } = await import('@/apis/code-wiki') + const wiki = await codeWikiApi.create({ + name: data.name, + description: data.description, + namespace, + source_type: data.source_type!, + source_url: data.source_url!, + }) + setShowCreateDialog(false) + resetCreateDialogState() + await sidebar.refreshAll() + // Straight into the reader: generation has already started, and the empty + // state there shows its progress. + router.push(`/knowledge/code-wiki/${wiki.id}`) + return + } + const { createKnowledgeBase } = await import('@/apis/knowledge') await createKnowledgeBase({ name: data.name, diff --git a/frontend/src/features/knowledge/document/hooks/useKnowledgeSidebar.ts b/frontend/src/features/knowledge/document/hooks/useKnowledgeSidebar.ts index 219123f207..c208468623 100644 --- a/frontend/src/features/knowledge/document/hooks/useKnowledgeSidebar.ts +++ b/frontend/src/features/knowledge/document/hooks/useKnowledgeSidebar.ts @@ -14,6 +14,7 @@ import { knowledgeBaseApi } from '@/apis/knowledge-base' import { dingtalkDocApi } from '@/apis/dingtalk-doc' import { getKnowledgeBase } from '@/apis/knowledge' import { useUser } from '@/features/common/UserContext' +import { documentViewOf } from '@/types/knowledge' import type { KnowledgeBase, AllGroupedKnowledgeResponse, @@ -460,7 +461,7 @@ export function useKnowledgeSidebar(): UseKnowledgeSidebarReturn { const newItem: RecentAccessItem = { kbId: kb.id, kbName: kb.name, - kbType: (kb.kb_type as 'notebook' | 'classic') || 'notebook', + kbType: documentViewOf(kb.kb_type) ?? 'notebook', namespace: kb.namespace, accessedAt: Date.now(), } @@ -546,7 +547,7 @@ export function useKnowledgeSidebar(): UseKnowledgeSidebarReturn { ? { ...item, kbName: updatedKb.name, - kbType: updatedKb.kb_type || 'notebook', + kbType: documentViewOf(updatedKb.kb_type) ?? 'notebook', namespace: updatedKb.namespace, } : item diff --git a/frontend/src/features/tasks/components/message/MessagesArea.tsx b/frontend/src/features/tasks/components/message/MessagesArea.tsx index 770cb4db09..89d08b6b8e 100644 --- a/frontend/src/features/tasks/components/message/MessagesArea.tsx +++ b/frontend/src/features/tasks/components/message/MessagesArea.tsx @@ -72,6 +72,7 @@ import { MessageLoadingStage } from './MessageLoadingStage' import { SaveToKnowledgeDialog } from './SaveToKnowledgeDialog' import { DocumentDetailDialog } from '@/features/knowledge/document/components/DocumentDetailDialog' import { ToastAction } from '@/components/ui/toast' +import { documentViewOf } from '@/types/knowledge' import type { KnowledgeBaseWithGroupInfo, KnowledgeDocument } from '@/types/knowledge' type SendMessageOptions = { @@ -1483,7 +1484,7 @@ function MessagesArea({ }} document={knowledgeDocumentToView.document} knowledgeBaseId={knowledgeDocumentToView.knowledgeBase.id} - kbType={knowledgeDocumentToView.knowledgeBase.kb_type} + kbType={documentViewOf(knowledgeDocumentToView.knowledgeBase.kb_type) ?? undefined} canEdit={true} knowledgeBaseName={knowledgeDocumentToView.knowledgeBase.name} knowledgeBaseNamespace={knowledgeDocumentToView.knowledgeBase.namespace} diff --git a/frontend/src/i18n/locales/en/knowledge.json b/frontend/src/i18n/locales/en/knowledge.json index 4f254f8ab9..1232579985 100644 --- a/frontend/src/i18n/locales/en/knowledge.json +++ b/frontend/src/i18n/locales/en/knowledge.json @@ -252,7 +252,11 @@ "defaultViewUpdateSuccess": "Default opening view updated", "defaultViewUpdateFailed": "Failed to update default opening view", "targetGroup": "Target Group", - "selectGroup": "Select Group" + "selectGroup": "Select Group", + "kindDocument": "Documents", + "kindDocumentDesc": "Upload and organise material", + "kindCode": "Code", + "kindCodeDesc": "Generated from a repository" }, "breadcrumb": { "root": "All Documents" @@ -1069,9 +1073,20 @@ "title": "New code wiki", "repository": "Repository", "name": "Name", - "namePlaceholder": "Defaults to the repository name", + "namePlaceholder": "Leave blank to use the repository name", "submit": "Create", - "noBoundModel": "The code wiki agent has no model bound, so generation cannot run." + "noBoundModel": "The code wiki agent has no model bound, so generation cannot run.", + "repositoryRequired": "Select or enter a repository address", + "fromMyRepositories": "From my repositories", + "byUrl": "By repository address", + "checking": "Checking the repository…", + "notReadable": "Cannot read this repository. Check the address, or add a token for this domain.", + "publicRepository": "Public repository", + "privateRepository": "Private repository", + "language": "Generation language", + "languageZh": "Chinese", + "languageEn": "English", + "alreadyBuilt": "{{count}} other wiki(s) already document this repository — you could ask for a share" }, "reader": { "regenerate": "Regenerate", diff --git a/frontend/src/i18n/locales/zh-CN/knowledge.json b/frontend/src/i18n/locales/zh-CN/knowledge.json index b9ce32c742..59b4618c99 100644 --- a/frontend/src/i18n/locales/zh-CN/knowledge.json +++ b/frontend/src/i18n/locales/zh-CN/knowledge.json @@ -252,7 +252,11 @@ "defaultViewUpdateSuccess": "默认打开方式已更新", "defaultViewUpdateFailed": "默认打开方式更新失败", "targetGroup": "归属", - "selectGroup": "选择分组" + "selectGroup": "选择分组", + "kindDocument": "文档", + "kindDocumentDesc": "上传与整理资料", + "kindCode": "代码", + "kindCodeDesc": "从仓库生成 wiki" }, "breadcrumb": { "root": "全部文档" @@ -1069,9 +1073,20 @@ "title": "新建代码 Wiki", "repository": "仓库", "name": "名称", - "namePlaceholder": "默认取仓库名", + "namePlaceholder": "留空则使用仓库名", "submit": "创建", - "noBoundModel": "代码 Wiki 智能体未绑定模型，无法生成。" + "noBoundModel": "代码 Wiki 智能体未绑定模型，无法生成。", + "repositoryRequired": "请选择或填写仓库地址", + "fromMyRepositories": "从我的仓库选择", + "byUrl": "输入仓库地址", + "checking": "正在检查仓库…", + "notReadable": "无法读取该仓库，请检查地址或为该域名配置 token", + "publicRepository": "公开仓库", + "privateRepository": "私有仓库", + "language": "生成语言", + "languageZh": "中文", + "languageEn": "English", + "alreadyBuilt": "已有 {{count}} 人为此仓库建过 wiki，可以向其索要分享" }, "reader": { "regenerate": "重新生成", diff --git a/frontend/src/types/code-wiki.ts b/frontend/src/types/code-wiki.ts index 61b6960d05..c499e91bc9 100644 --- a/frontend/src/types/code-wiki.ts +++ b/frontend/src/types/code-wiki.ts @@ -5,10 +5,10 @@ /** * A code wiki is a knowledge base an agent writes from a source repository. * - * It is listed and read through its own endpoints rather than the general knowledge - * base ones: it belongs to the wiki account rather than to anyone who asks for it, so - * it matches none of that list's scopes, and who may read it is decided by who may - * read its repository. + * It belongs to whoever created it and is governed by the ordinary knowledge-base + * ACL, so it appears in the general list alongside every other kind. These endpoints + * exist only because its list items carry repository fields; they grant nothing the + * general list would not. */ export interface CodeWikiSummary { id: number @@ -32,12 +32,39 @@ export interface CodeWikiListResponse { } export interface CodeWikiCreateRequest { + /** Optional: left blank, the repository's own name is used. */ name: string description?: string - source_type: 'github' | 'gitlab' | 'gitea' + /** Where to file it, as for any knowledge base. Defaults to personal. */ + namespace?: string + source_type: CodeWikiSourceType source_url: string } +export type CodeWikiSourceType = 'github' | 'gitlab' | 'gitea' + +/** + * What is known about a repository before a wiki is bound to it. + * + * `exists: false` means "not readable with what you have". Private and absent are + * deliberately indistinguishable — telling them apart would disclose which private + * repositories exist. + */ +export interface CodeWikiResolution { + exists: boolean + /** `public` or `private`, when readable. */ + visibility: string + /** Saves listing branches, which has no path for a repository read anonymously. */ + default_branch: string + /** Repository path, offered as the wiki's name when none is given. */ + name: string + description: string + /** `public`, `member`, or `none`. */ + access: string + /** How many wikis already document this repository, whoever owns them. */ + existing_wiki_count: number +} + /** * What happened when a wiki was asked to regenerate. * diff --git a/frontend/src/types/knowledge.ts b/frontend/src/types/knowledge.ts index 99bd3097ae..796a22e394 100644 --- a/frontend/src/types/knowledge.ts +++ b/frontend/src/types/knowledge.ts @@ -303,7 +303,28 @@ export interface SummaryModelRef { // opening view, not a resource capability boundary. // - notebook: default to Notebook workspace view // - classic: default to document management view -export type KnowledgeBaseType = 'notebook' | 'classic' +/** + * How a knowledge base is filled and read. + * + * `code_wiki` is generated from a repository by an agent rather than filled by hand, + * and is read through its own three-pane reader. It is otherwise an ordinary + * knowledge base: same ownership, same ACL, same listing. + */ +export type KnowledgeBaseType = 'notebook' | 'classic' | 'code_wiki' + +/** + * The two views a document knowledge base can open in. + * + * Deliberately narrower than `KnowledgeBaseType`: a code wiki has neither view, it + * has a reader of its own. Components that render one or the other take this type so + * that a code wiki cannot be handed to them without the conversion being visible. + */ +export type DocumentViewType = 'notebook' | 'classic' + +/** The view to open a knowledge base in, or `null` when it is not a document one. */ +export function documentViewOf(kbType?: KnowledgeBaseType | null): DocumentViewType | null { + return kbType === 'notebook' || kbType === 'classic' ? kbType : null +} export type KnowledgeView = 'documents' | 'notebook' export type RagConfigMode = 'auto' | 'disabled' export type DirectAccessRequirement = 'read' | 'edit' @@ -361,6 +382,12 @@ export interface KnowledgeBaseCreate { summary_model_ref?: SummaryModelRef | null /** Default opening view: 'notebook' or 'classic' (documents) */ kb_type?: KnowledgeBaseType + /** Only for `kb_type: 'code_wiki'` — which platform hosts the repository. */ + source_type?: 'github' | 'gitlab' | 'gitea' + /** Only for `kb_type: 'code_wiki'` — the repository to document. */ + source_url?: string + /** Only for `kb_type: 'code_wiki'` — what language to generate the pages in. */ + language?: string /** Guided questions list (max 3) for notebook mode quick user interaction */ guided_questions?: string[] /** Maximum number of knowledge base tool calls allowed per conversation */ From f15e712c0e6df496a16fe971df95c9f3913f38d0 Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Wed, 5 Aug 2026 11:38:09 +0800 Subject: [PATCH 13/14] fix(knowledge): name a code wiki's existing wikis, and let a new one go unnamed Two things reported from smoke testing. Creating a wiki without a name was rejected with a 422: the schema still required one, though the whole point of leaving it blank is that the server fills in the repository's own name. It does that now, preferring what the provider calls the repository over what the URL parses to -- the provider is the authority, renames included -- and falling back to the URL when the repository could not be described. "1 other wiki already documents this repository" was not actionable: asking for a share needs somebody to ask. The wikis are named instead of counted, with their owner, and one the caller can already open is rendered as a link since there is nothing to ask for. Wikis the caller cannot open are still listed: they asked about a repository they can read, and what is disclosed is that a colleague documented it. The first version of the unnamed-wiki test could not tell the two name sources apart -- both were "wecode-ai/Wegent" -- so removing the resolved name left it passing. It now uses a name the URL does not parse to. Co-Authored-By: Claude Opus 5 --- backend/app/api/endpoints/knowledge.py | 24 +++- backend/app/schemas/knowledge.py | 30 ++++- .../services/knowledge/code_wiki/registry.py | 59 +++++++-- backend/tests/api/test_knowledge_code_wiki.py | 118 +++++++++++++++++- .../code-wiki/CodeWikiSourceFields.tsx | 46 +++++-- frontend/src/i18n/locales/en/knowledge.json | 3 +- .../src/i18n/locales/zh-CN/knowledge.json | 3 +- frontend/src/types/code-wiki.ts | 14 ++- 8 files changed, 257 insertions(+), 40 deletions(-) diff --git a/backend/app/api/endpoints/knowledge.py b/backend/app/api/endpoints/knowledge.py index 352e1117ac..af41329ac8 100644 --- a/backend/app/api/endpoints/knowledge.py +++ b/backend/app/api/endpoints/knowledge.py @@ -45,6 +45,7 @@ BatchDocumentIds, BatchOperationResult, CodeWikiCreate, + CodeWikiExisting, CodeWikiListItem, CodeWikiListResponse, CodeWikiPageNode, @@ -93,7 +94,7 @@ CODE_WIKI_NAMESPACE, claim_repository, existing_wiki_id, - wiki_count_for, + existing_wikis_for, ) from app.services.knowledge.code_wiki.resolution import resolve_repository from app.services.knowledge.code_wiki.run_mode import ChangedPath @@ -551,9 +552,14 @@ def resolve_code_wiki_source( name=resolved.name, description=resolved.description, access=resolved.access, - # Counted even when the caller cannot read the repository: the number says - # the work already exists, not that they may see it. - existing_wiki_count=wiki_count_for(db, source.source_url), + # Listed even when the caller cannot read the repository: what is disclosed + # is that a colleague documented it, to somebody who asked about it by name. + existing_wikis=[ + CodeWikiExisting(**wiki.__dict__) + for wiki in existing_wikis_for( + db, source.source_url, viewer_id=current_user.id + ) + ], ) @@ -644,12 +650,18 @@ def create_code_wiki( KnowledgeService.get_document_count(db, existing_id), ) + # Filled in here rather than in the browser: a pre-filled box would read as the + # caller's own input, and the repository is the authority on what it is called. + described = resolve_repository(db, current_user.id, source) + name = data.name.strip() or described.name or source.project_name + description = data.description or described.description or None + try: result = knowledge_orchestrator.create_knowledge_base( db=db, user=current_user, - name=data.name, - description=data.description, + name=name[:100], + description=description, namespace=data.namespace or CODE_WIKI_NAMESPACE, kb_type=KnowledgeBaseType.CODE_WIKI.value, source=source, diff --git a/backend/app/schemas/knowledge.py b/backend/app/schemas/knowledge.py index c456d453a5..1230e83498 100644 --- a/backend/app/schemas/knowledge.py +++ b/backend/app/schemas/knowledge.py @@ -305,7 +305,15 @@ class CodeWikiCreate(BaseModel): bases. Its kind is set by the endpoint, not by the caller. """ - name: str = Field(..., min_length=1, max_length=100) + name: str = Field( + "", + max_length=100, + description=( + "Left blank, the repository's own name is used. Filled in here rather " + "than in the browser, where a pre-filled box would read as the caller's " + "own input." + ), + ) description: Optional[str] = Field(None, max_length=500) namespace: Optional[str] = Field( None, @@ -431,6 +439,17 @@ class CodeWikiRunResponse(BaseModel): task_id: int = Field(0, description="Task running the agent, when started") +class CodeWikiExisting(BaseModel): + """A wiki of this repository somebody has already built.""" + + id: int + name: str + owner_name: str = Field("", description="Who to ask, when it is not accessible") + accessible: bool = Field( + False, description="Whether the caller can already open it" + ) + + class CodeWikiResolveRequest(BaseModel): """Ask what is known about a repository before binding a wiki to it.""" @@ -452,12 +471,11 @@ class CodeWikiResolveResponse(BaseModel): name: str = Field("", description="Repository path, offered as the wiki's name") description: str = Field("", description="Offered as the wiki's description") access: str = Field("none", description="public, member, or none") - existing_wiki_count: int = Field( - 0, + existing_wikis: List["CodeWikiExisting"] = Field( + default_factory=list, description=( - "How many code wikis already document this repository, whoever owns " - "them. Shown so a caller can ask for a share instead of paying for a " - "second generation." + "Code wikis that already document this repository, whoever owns them. " + "Named rather than counted: asking for a share needs somebody to ask." ), ) diff --git a/backend/app/services/knowledge/code_wiki/registry.py b/backend/app/services/knowledge/code_wiki/registry.py index 419a725b92..440ef00c4e 100644 --- a/backend/app/services/knowledge/code_wiki/registry.py +++ b/backend/app/services/knowledge/code_wiki/registry.py @@ -20,6 +20,7 @@ """ import logging +from dataclasses import dataclass from typing import Optional from sqlalchemy.orm import Session @@ -87,15 +88,55 @@ def existing_wiki_id( return row[0] if row else None -def wiki_count_for(db: Session, source_url: str) -> int: - """How many code wikis exist for this repository, whoever owns them. +@dataclass(frozen=True) +class ExistingWiki: + """A wiki of this repository that somebody has already built.""" - Shown before creating one so that somebody can ask for a share instead of paying - for a second generation. Counted across all owners on purpose — the point is that - the work has already been done, not that the caller can see it. + id: int + name: str + owner_name: str + # Whether the caller can already open it. False means the useful action is to + # ask its owner for a share, which is why the owner is named at all. + accessible: bool + + +def existing_wikis_for( + db: Session, source_url: str, *, viewer_id: int +) -> list[ExistingWiki]: + """Wikis already built from this repository, whoever owns them. + + Shown before creating another so that somebody can ask for a share instead of + paying for a second generation — which needs the owner's name, not just a count. + + Listing wikis the caller cannot open is the point rather than a leak to weigh: + the caller has demonstrated they can read the repository, and the disclosure is + that a colleague documented it. Names of accessible wikis come with an id so the + client can link straight to them. """ - return ( - db.query(WikiProject) - .filter(WikiProject.source_url == source_url, WikiProject.kind_id > 0) - .count() + from app.models.kind import Kind + from app.models.user import User + from app.services.knowledge.knowledge_service import KnowledgeService + + rows = ( + db.query(Kind, User.user_name) + .join(WikiProject, WikiProject.kind_id == Kind.id) + .outerjoin(User, User.id == Kind.user_id) + .filter( + WikiProject.source_url == source_url, + WikiProject.kind_id > 0, + Kind.is_active.is_(True), + ) + .order_by(Kind.created_at.asc()) + .all() ) + + visible = {kind.id for kind in KnowledgeService.list_knowledge_bases(db, viewer_id)} + return [ + ExistingWiki( + id=kind.id, + name=(kind.json or {}).get("spec", {}).get("name") or kind.name, + owner_name=owner_name or "", + accessible=kind.id in visible, + ) + for kind, owner_name in rows + ] diff --git a/backend/tests/api/test_knowledge_code_wiki.py b/backend/tests/api/test_knowledge_code_wiki.py index ce28d5f6cd..92bddc14c8 100644 --- a/backend/tests/api/test_knowledge_code_wiki.py +++ b/backend/tests/api/test_knowledge_code_wiki.py @@ -567,16 +567,17 @@ def test_an_unreadable_repository_resolves_to_200_not_404( assert response.json()["exists"] is False -def test_resolving_reports_how_many_wikis_already_exist( +def test_resolving_names_the_wikis_that_already_exist( test_client: TestClient, auth_headers: dict[str, str], kind_services_use_test_db, ): - """Shown so somebody can ask for a share instead of paying for a second - generation. Counted across all owners: the point is that the work exists.""" + """A count is not actionable: asking for a share needs somebody to ask. One the + caller can already open is reported as accessible, so the client links to it + instead.""" from app.services.knowledge.code_wiki.resolution import UNREADABLE - _create_wiki(test_client, auth_headers) + kb_id = _create_wiki(test_client, auth_headers) with patch( "app.api.endpoints.knowledge.resolve_repository", return_value=UNREADABLE @@ -585,7 +586,114 @@ def test_resolving_reports_how_many_wikis_already_exist( RESOLVE_URL, json=RESOLVE_PAYLOAD, headers=auth_headers ) - assert response.json()["existing_wiki_count"] == 1 + (existing,) = response.json()["existing_wikis"] + assert existing["id"] == kb_id + assert existing["accessible"] is True + assert existing["owner_name"] == "testuser" + + +def test_another_user_s_wiki_is_named_with_its_owner( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """The owner is the whole point of naming it: the caller cannot open this one, + so the useful action is to ask.""" + from app.core.security import create_access_token, get_password_hash + from app.services.knowledge.code_wiki.resolution import UNREADABLE + + _create_wiki(test_client, auth_headers) + stranger = User( + user_name="stranger", + password_hash=get_password_hash("irrelevant"), + email="stranger@example.com", + is_active=True, + ) + test_db.add(stranger) + test_db.commit() + token = create_access_token(data={"sub": stranger.user_name}) + + with patch( + "app.api.endpoints.knowledge.resolve_repository", return_value=UNREADABLE + ): + response = test_client.post( + RESOLVE_URL, + json=RESOLVE_PAYLOAD, + headers={"Authorization": f"Bearer {token}"}, + ) + + (existing,) = response.json()["existing_wikis"] + assert existing["accessible"] is False + assert existing["owner_name"] == "testuser" + + +def test_a_code_wiki_can_be_created_without_a_name( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """Left blank the repository's own name is used. Filled in on the server, not + pre-filled in the browser where it would read as the caller's own input.""" + from app.models.kind import Kind + from app.services.knowledge.code_wiki.resolution import ResolvedRepository + + with ( + patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ), + patch( + "app.api.endpoints.knowledge.resolve_repository", + return_value=ResolvedRepository( + exists=True, + visibility="public", + default_branch="main", + # Deliberately not what the URL parses to: the provider is the + # authority on what a repository is called, renames included. + name="wecode-ai/Wegent-Renamed", + description="An agent operating system", + access="public", + ), + ), + ): + response = test_client.post( + CREATE_URL, json={**PAYLOAD, "name": ""}, headers=auth_headers + ) + + assert response.status_code == 201, response.text + kind = test_db.get(Kind, response.json()["id"]) + assert kind.json["spec"]["name"] == "wecode-ai/Wegent-Renamed" + + +def test_an_unnamed_wiki_falls_back_to_the_url_when_the_repository_is_unreadable( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """The access gate can pass on a path the probe does not. A knowledge base with + a blank name would then be created, which no listing can render.""" + from app.models.kind import Kind + from app.services.knowledge.code_wiki.resolution import UNREADABLE + + with ( + patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ), + patch( + "app.api.endpoints.knowledge.resolve_repository", return_value=UNREADABLE + ), + ): + response = test_client.post( + CREATE_URL, json={**PAYLOAD, "name": ""}, headers=auth_headers + ) + + assert response.status_code == 201, response.text + kind = test_db.get(Kind, response.json()["id"]) + assert kind.json["spec"]["name"] == "wecode-ai/Wegent" def test_resolving_a_malformed_url_is_a_bad_request( diff --git a/frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx b/frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx index 08aa300161..d8cb594f80 100644 --- a/frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx +++ b/frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx @@ -249,16 +249,42 @@ function SourceStatus({ : t('knowledge:codeWiki.create.privateRepository')} {resolution.default_branch ? ` · ${resolution.default_branch}` : ''}

    - {resolution.existing_wiki_count > 0 && ( -

    - - {t('knowledge:codeWiki.create.alreadyBuilt', { - count: resolution.existing_wiki_count, - })} -

    + {resolution.existing_wikis.length > 0 && ( +
    +

    + + {t('knowledge:codeWiki.create.alreadyBuilt', { + count: resolution.existing_wikis.length, + })} +

    + {/* Named, not counted: asking for a share needs somebody to ask. One the + caller can already open is a link instead — there is nothing to ask + for. */} +
      + {resolution.existing_wikis.map(wiki => ( +
    • + {wiki.accessible ? ( + + {wiki.name} + + ) : ( + + {wiki.name} + {wiki.owner_name + ? ` — ${t('knowledge:codeWiki.create.ownedBy', { + owner: wiki.owner_name, + })}` + : ''} + + )} +
      • + ))} +
    +
    )}
    ) diff --git a/frontend/src/i18n/locales/en/knowledge.json b/frontend/src/i18n/locales/en/knowledge.json index 1232579985..35244cbda2 100644 --- a/frontend/src/i18n/locales/en/knowledge.json +++ b/frontend/src/i18n/locales/en/knowledge.json @@ -1086,7 +1086,8 @@ "language": "Generation language", "languageZh": "Chinese", "languageEn": "English", - "alreadyBuilt": "{{count}} other wiki(s) already document this repository — you could ask for a share" + "alreadyBuilt": "{{count}} other wiki(s) already document this repository — you could ask for a share", + "ownedBy": "owned by {{owner}} — ask them for a share" }, "reader": { "regenerate": "Regenerate", diff --git a/frontend/src/i18n/locales/zh-CN/knowledge.json b/frontend/src/i18n/locales/zh-CN/knowledge.json index 59b4618c99..803a2ab8f0 100644 --- a/frontend/src/i18n/locales/zh-CN/knowledge.json +++ b/frontend/src/i18n/locales/zh-CN/knowledge.json @@ -1086,7 +1086,8 @@ "language": "生成语言", "languageZh": "中文", "languageEn": "English", - "alreadyBuilt": "已有 {{count}} 人为此仓库建过 wiki，可以向其索要分享" + "alreadyBuilt": "已有 {{count}} 人为此仓库建过 wiki，可以向其索要分享", + "ownedBy": "归属 {{owner}}，可向其索要分享" }, "reader": { "regenerate": "重新生成", diff --git a/frontend/src/types/code-wiki.ts b/frontend/src/types/code-wiki.ts index c499e91bc9..34c99bced1 100644 --- a/frontend/src/types/code-wiki.ts +++ b/frontend/src/types/code-wiki.ts @@ -43,6 +43,16 @@ export interface CodeWikiCreateRequest { export type CodeWikiSourceType = 'github' | 'gitlab' | 'gitea' +/** A wiki of this repository somebody has already built. */ +export interface CodeWikiExisting { + id: number + name: string + /** Who to ask, when it is not accessible. */ + owner_name: string + /** Whether the caller can already open it. */ + accessible: boolean +} + /** * What is known about a repository before a wiki is bound to it. * @@ -61,8 +71,8 @@ export interface CodeWikiResolution { description: string /** `public`, `member`, or `none`. */ access: string - /** How many wikis already document this repository, whoever owns them. */ - existing_wiki_count: number + /** Wikis that already document this repository, whoever owns them. */ + existing_wikis: CodeWikiExisting[] } /** From cd51e342c6b88748de3775f41ff93929c594e1df Mon Sep 17 00:00:00 2001 From: yanhe1 Date: Wed, 5 Aug 2026 12:23:27 +0800 Subject: [PATCH 14/14] refactor(knowledge): stop describing the repository twice when creating a wiki Filling in a blank name on the server meant resolving the repository there, after the access gate had already asked the provider about it. That was only cheap because the create form had just resolved the same repository for the same user and warmed the cache -- an implicit coupling that costs a network round-trip whenever it does not hold, such as a cold cache or a direct API call. The client sends what it already resolved instead. The input box is still not pre-filled, which is the property worth keeping: a filled box reads as the caller's own input. The server keeps one fallback, the name parsed from the URL, so that a caller who skipped the probe does not create a knowledge base with no name at all. Co-Authored-By: Claude Opus 5 --- backend/app/api/endpoints/knowledge.py | 13 ++-- backend/tests/api/test_knowledge_code_wiki.py | 71 ++++++++++--------- .../components/CreateKnowledgeBaseDialog.tsx | 6 ++ .../document/hooks/useKnowledgeBaseDialogs.ts | 4 +- frontend/src/types/knowledge.ts | 4 ++ 5 files changed, 56 insertions(+), 42 deletions(-) diff --git a/backend/app/api/endpoints/knowledge.py b/backend/app/api/endpoints/knowledge.py index af41329ac8..58e177dce9 100644 --- a/backend/app/api/endpoints/knowledge.py +++ b/backend/app/api/endpoints/knowledge.py @@ -650,18 +650,19 @@ def create_code_wiki( KnowledgeService.get_document_count(db, existing_id), ) - # Filled in here rather than in the browser: a pre-filled box would read as the - # caller's own input, and the repository is the authority on what it is called. - described = resolve_repository(db, current_user.id, source) - name = data.name.strip() or described.name or source.project_name - description = data.description or described.description or None + # A blank name means "use the repository's". The client sends what it already + # resolved for the form, so this does not probe the provider again -- it has + # already been asked once by the gate above. Falling back to the URL keeps a + # caller that skipped the probe, or called the API directly, from creating a + # knowledge base with no name at all. + name = data.name.strip() or source.project_name try: result = knowledge_orchestrator.create_knowledge_base( db=db, user=current_user, name=name[:100], - description=description, + description=data.description, namespace=data.namespace or CODE_WIKI_NAMESPACE, kb_type=KnowledgeBaseType.CODE_WIKI.value, source=source, diff --git a/backend/tests/api/test_knowledge_code_wiki.py b/backend/tests/api/test_knowledge_code_wiki.py index 92bddc14c8..2212f9d043 100644 --- a/backend/tests/api/test_knowledge_code_wiki.py +++ b/backend/tests/api/test_knowledge_code_wiki.py @@ -634,32 +634,21 @@ def test_a_code_wiki_can_be_created_without_a_name( test_db: Session, kind_services_use_test_db, ): - """Left blank the repository's own name is used. Filled in on the server, not - pre-filled in the browser where it would read as the caller's own input.""" + """Left blank, the repository's name is used. The client sends what the form + already resolved, so this does not ask the provider a second time — the access + gate has asked it once already.""" from app.models.kind import Kind - from app.services.knowledge.code_wiki.resolution import ResolvedRepository - with ( - patch( - "app.api.endpoints.knowledge.assert_user_can_read_source", - return_value={"has_access": True}, - ), - patch( - "app.api.endpoints.knowledge.resolve_repository", - return_value=ResolvedRepository( - exists=True, - visibility="public", - default_branch="main", - # Deliberately not what the URL parses to: the provider is the - # authority on what a repository is called, renames included. - name="wecode-ai/Wegent-Renamed", - description="An agent operating system", - access="public", - ), - ), + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, ): response = test_client.post( - CREATE_URL, json={**PAYLOAD, "name": ""}, headers=auth_headers + CREATE_URL, + # Deliberately not what the URL parses to: the provider is the authority + # on what a repository is called, renames included. + json={**PAYLOAD, "name": "wecode-ai/Wegent-Renamed"}, + headers=auth_headers, ) assert response.status_code == 201, response.text @@ -667,25 +656,19 @@ def test_a_code_wiki_can_be_created_without_a_name( assert kind.json["spec"]["name"] == "wecode-ai/Wegent-Renamed" -def test_an_unnamed_wiki_falls_back_to_the_url_when_the_repository_is_unreadable( +def test_an_unnamed_wiki_falls_back_to_the_url( test_client: TestClient, auth_headers: dict[str, str], test_db: Session, kind_services_use_test_db, ): - """The access gate can pass on a path the probe does not. A knowledge base with - a blank name would then be created, which no listing can render.""" + """A caller that skipped the probe, or called the API directly, would otherwise + create a knowledge base with no name at all, which no listing can render.""" from app.models.kind import Kind - from app.services.knowledge.code_wiki.resolution import UNREADABLE - with ( - patch( - "app.api.endpoints.knowledge.assert_user_can_read_source", - return_value={"has_access": True}, - ), - patch( - "app.api.endpoints.knowledge.resolve_repository", return_value=UNREADABLE - ), + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, ): response = test_client.post( CREATE_URL, json={**PAYLOAD, "name": ""}, headers=auth_headers @@ -696,6 +679,26 @@ def test_an_unnamed_wiki_falls_back_to_the_url_when_the_repository_is_unreadable assert kind.json["spec"]["name"] == "wecode-ai/Wegent" +def test_creating_does_not_describe_the_repository_again( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """The access gate already asked the provider. Resolving again here worked only + because the form had just warmed the cache — an implicit coupling that fails + whenever it has not.""" + with ( + patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ), + patch("app.api.endpoints.knowledge.resolve_repository") as resolve, + ): + test_client.post(CREATE_URL, json={**PAYLOAD, "name": ""}, headers=auth_headers) + + resolve.assert_not_called() + + def test_resolving_a_malformed_url_is_a_bad_request( test_client: TestClient, auth_headers: dict[str, str] ): diff --git a/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx b/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx index 484a9fd963..f9ab610950 100644 --- a/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx +++ b/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx @@ -249,6 +249,12 @@ export function CreateKnowledgeBaseDialog({ source_type: source.source_type, source_url: source.source_url, language: source.language, + // Left blank, the repository's own name is used. Sent from what the + // form already resolved rather than pre-filled into the box, which + // would read as the caller's own input — and rather than resolved + // again on the server, which has asked the provider once already. + resolved_name: source.resolution?.name, + resolved_description: source.resolution?.description, } : {}), }) diff --git a/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts b/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts index 3f288e9a86..9b10d97b3f 100644 --- a/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts +++ b/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts @@ -152,8 +152,8 @@ export function useKnowledgeBaseDialogs({ if (kbType === 'code_wiki') { const { codeWikiApi } = await import('@/apis/code-wiki') const wiki = await codeWikiApi.create({ - name: data.name, - description: data.description, + name: data.name || data.resolved_name || '', + description: data.description || data.resolved_description, namespace, source_type: data.source_type!, source_url: data.source_url!, diff --git a/frontend/src/types/knowledge.ts b/frontend/src/types/knowledge.ts index 796a22e394..e535f8be85 100644 --- a/frontend/src/types/knowledge.ts +++ b/frontend/src/types/knowledge.ts @@ -388,6 +388,10 @@ export interface KnowledgeBaseCreate { source_url?: string /** Only for `kb_type: 'code_wiki'` — what language to generate the pages in. */ language?: string + /** What the provider calls the repository, used when the name is left blank. */ + resolved_name?: string + /** What the provider says the repository is, used when no description is given. */ + resolved_description?: string /** Guided questions list (max 3) for notebook mode quick user interaction */ guided_questions?: string[] /** Maximum number of knowledge base tool calls allowed per conversation */