diff --git a/backend/.env.example b/backend/.env.example index fb12ed109f..f956864b47 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -213,6 +213,11 @@ WIKI_ENABLED=True # The team's bot MUST have a model configured (bind_model or custom config) # If no model is configured, wiki generation will be disabled in the frontend WIKI_DEFAULT_TEAM_NAME=wiki-team +# Team that runs knowledge-base code wikis (matches init_data/02-public-resources.yaml) +# Separate from WIKI_DEFAULT_TEAM_NAME: the two paths give their agents different +# instructions and different submission rules, so they cannot share one team. +# Its bot MUST have a model configured (bind_model or custom config) +WIKI_CODE_WIKI_TEAM_NAME=code-wiki-team # Default agent type for wiki generation WIKI_DEFAULT_AGENT_TYPE=ClaudeCode # Default language for wiki documentation generation (en = English, zh = Chinese) diff --git a/backend/alembic/versions/20260731_bd9c871a93d2_add_knowledge_content_origin_and_wiki_.py b/backend/alembic/versions/20260731_bd9c871a93d2_add_knowledge_content_origin_and_wiki_.py new file mode 100644 index 0000000000..fb36d68e6c --- /dev/null +++ b/backend/alembic/versions/20260731_bd9c871a93d2_add_knowledge_content_origin_and_wiki_.py @@ -0,0 +1,77 @@ +"""add knowledge content origin and wiki generation kb link + +Revision ID: bd9c871a93d2 +Revises: b9c0d1e2f3a4 + +Adds the two things a code wiki needs to coexist with ordinary knowledge content: + +- ``origin`` on documents and folders, marking whether a row is agent-generated or + user-owned. It defaults to ``user`` so that every existing row, and every row this + service does not create itself, is excluded from the generated-content projection. + Getting this backwards would let a regeneration delete content nobody can restore. +- ``kind_id`` on wiki generations, binding a version line to a knowledge base. The + versions previously hung off ``wiki_projects``, whose ``source_url`` is globally + unique; leaving them there would share one version line between knowledge bases + tracking the same repository. ``0`` marks rows that predate code wikis. + +All three columns are NOT NULL with a server default, as required for existing rows to +backfill without a nullable intermediate state. +""" + +from collections.abc import Sequence +from typing import Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "bd9c871a93d2" +down_revision: Union[str, Sequence[str], None] = "b9c0d1e2f3a4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.add_column( + "knowledge_documents", + sa.Column( + "origin", + sa.String(length=20), + nullable=False, + server_default="user", + comment="Content ownership: 'generated' (agent-owned) or 'user'", + ), + ) + op.add_column( + "knowledge_folders", + sa.Column( + "origin", + sa.String(length=20), + nullable=False, + server_default="user", + comment="Content ownership: 'generated' (agent-owned) or 'user'", + ), + ) + op.add_column( + "wiki_generations", + sa.Column( + "kind_id", + sa.Integer(), + nullable=False, + server_default="0", + comment="Knowledge base this version line belongs to; 0 = legacy row", + ), + ) + op.create_index( + "ix_wiki_generations_kind_id", "wiki_generations", ["kind_id"], unique=False + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index("ix_wiki_generations_kind_id", table_name="wiki_generations") + op.drop_column("wiki_generations", "kind_id") + op.drop_column("knowledge_folders", "origin") + op.drop_column("knowledge_documents", "origin") diff --git a/backend/alembic/versions/20260804_2b5791acc5fa_link_wiki_projects_to_their_code_wiki.py b/backend/alembic/versions/20260804_2b5791acc5fa_link_wiki_projects_to_their_code_wiki.py new file mode 100644 index 0000000000..c027999416 --- /dev/null +++ b/backend/alembic/versions/20260804_2b5791acc5fa_link_wiki_projects_to_their_code_wiki.py @@ -0,0 +1,48 @@ +"""link wiki projects to their code wiki + +Revision ID: 2b5791acc5fa +Revises: bd9c871a93d2 + +``wiki_projects.source_url`` is already UNIQUE, which makes this table the only place +that can enforce "one repository, one code wiki" against two people creating at the +same moment. Recording the knowledge base here, rather than checking a JSON field on +the knowledge base itself, turns a check-then-insert into a database constraint. + +``0`` marks a project row with no code wiki, which is every row that exists today. +""" + +from collections.abc import Sequence +from typing import Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "2b5791acc5fa" +down_revision: Union[str, Sequence[str], None] = "bd9c871a93d2" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.add_column( + "wiki_projects", + sa.Column( + "kind_id", + sa.Integer(), + nullable=False, + server_default="0", + comment="Code wiki knowledge base built from this repository; 0 = none", + ), + ) + op.create_index( + "ix_wiki_projects_kind_id", "wiki_projects", ["kind_id"], unique=False + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index("ix_wiki_projects_kind_id", table_name="wiki_projects") + op.drop_column("wiki_projects", "kind_id") diff --git a/backend/alembic/versions/20260804_c3d4e5f6a7b8_allow_several_wikis_per_repository.py b/backend/alembic/versions/20260804_c3d4e5f6a7b8_allow_several_wikis_per_repository.py new file mode 100644 index 0000000000..a8aadcb5e4 --- /dev/null +++ b/backend/alembic/versions/20260804_c3d4e5f6a7b8_allow_several_wikis_per_repository.py @@ -0,0 +1,54 @@ +"""allow several wikis per repository + +Revision ID: c3d4e5f6a7b8 +Revises: 2b5791acc5fa + +A code wiki belongs to whoever created it, so a wiki built by one person is invisible +to everyone else under the ordinary knowledge-base ACL. "One repository, one wiki" +therefore stopped being a saving and became a way to take a wiki away from the second +person to ask for one. + +``wiki_projects`` accordingly holds one row per ``(repository, wiki)`` rather than one +per repository. The UNIQUE moves from ``source_url`` alone to the pair, which is still +a database constraint rather than a check-then-insert: it settles two requests racing +for the same pair, and it lets ``COUNT(*) WHERE source_url = ?`` answer "how many +wikis already exist for this repository" without reading a JSON field. + +Legacy wiki rows carry ``kind_id = 0`` and stay at most one per repository, which the +same constraint gives for free. +""" + +from collections.abc import Sequence +from typing import Union + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "c3d4e5f6a7b8" +down_revision: Union[str, Sequence[str], None] = "2b5791acc5fa" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# MySQL names the constraint after the column it was declared on. +OLD_UNIQUE = "source_url" +NEW_UNIQUE = "uq_wiki_projects_source_url_kind_id" + + +def upgrade() -> None: + """Upgrade schema.""" + # Created before the old one is dropped: between the two statements the table is + # covered by both rather than by neither, so a concurrent insert cannot slip a + # duplicate pair in through the gap. + op.create_unique_constraint(NEW_UNIQUE, "wiki_projects", ["source_url", "kind_id"]) + op.drop_constraint(OLD_UNIQUE, "wiki_projects", type_="unique") + + +def downgrade() -> None: + """Downgrade schema. + + Fails if any repository has more than one wiki, which is correct: silently + discarding one of them would destroy a generated knowledge base. Delete the + surplus wikis first if this has to be reversed. + """ + op.create_unique_constraint(OLD_UNIQUE, "wiki_projects", ["source_url"]) + op.drop_constraint(NEW_UNIQUE, "wiki_projects", type_="unique") diff --git a/backend/app/api/endpoints/knowledge.py b/backend/app/api/endpoints/knowledge.py index 69d25c6848..58e177dce9 100644 --- a/backend/app/api/endpoints/knowledge.py +++ b/backend/app/api/endpoints/knowledge.py @@ -13,7 +13,15 @@ from datetime import datetime from typing import Dict, List, Optional -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status +from fastapi import ( + APIRouter, + BackgroundTasks, + Depends, + HTTPException, + Query, + Response, + status, +) from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -29,12 +37,23 @@ from app.core.config import settings from app.core.exceptions import CustomHTTPException from app.db.session import SessionLocal +from app.models.kind import Kind from app.models.user import User from app.schemas.knowledge import ( AccessibleKnowledgeResponse, AllGroupedKnowledgeResponse, BatchDocumentIds, BatchOperationResult, + CodeWikiCreate, + CodeWikiExisting, + CodeWikiListItem, + CodeWikiListResponse, + CodeWikiPageNode, + CodeWikiPageTree, + CodeWikiResolveRequest, + CodeWikiResolveResponse, + CodeWikiRunCreate, + CodeWikiRunResponse, DocumentContentUpdate, DocumentDetailResponse, DocumentMoveRequest, @@ -42,6 +61,7 @@ KnowledgeBaseCreate, KnowledgeBaseListResponse, KnowledgeBaseResponse, + KnowledgeBaseType, KnowledgeBaseTypeUpdate, KnowledgeBaseUpdate, KnowledgeDocumentCreate, @@ -64,6 +84,27 @@ KnowledgeService, knowledge_base_qa_service, ) +from app.services.knowledge.code_wiki.generation import GenerationInFlight +from app.services.knowledge.code_wiki.navigation import page_tree +from app.services.knowledge.code_wiki.publisher import ( + PUBLISHED_AT_KEY, + PUBLISHED_COMMIT_KEY, +) +from app.services.knowledge.code_wiki.registry import ( + CODE_WIKI_NAMESPACE, + claim_repository, + existing_wiki_id, + existing_wikis_for, +) +from app.services.knowledge.code_wiki.resolution import resolve_repository +from app.services.knowledge.code_wiki.run_mode import ChangedPath +from app.services.knowledge.code_wiki.runner import CodeWikiRunError, start_run +from app.services.knowledge.code_wiki.source import ( + SourceAccessDenied, + SourceRepository, + assert_user_can_read_source, + assert_user_can_write_source, +) from app.services.knowledge.orchestrator import ( DEFAULT_KNOWLEDGE_LIST_LIMIT, MAX_DOCUMENT_READ_LIMIT, @@ -424,6 +465,18 @@ def create_knowledge_base( - **namespace=**: Team knowledge base (requires Maintainer+ permission) - **members**: Optional initial members to add after creation """ + # A code wiki must be created through its own endpoint, which binds a repository + # and checks the caller can read it. Refusing here rather than quietly ignoring + # the field keeps this endpoint from becoming a way around that check. + if data.kb_type == KnowledgeBaseType.CODE_WIKI: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Use POST /knowledge-bases/code-wikis to create a " + "'code_wiki' knowledge base" + ), + ) + try: # Use Orchestrator for unified business logic (REST API and MCP tools share the same logic) result = knowledge_orchestrator.create_knowledge_base( @@ -468,6 +521,345 @@ def create_knowledge_base( ) +@router.post("/code-wikis/resolve", response_model=CodeWikiResolveResponse) +@trace_sync("resolve_code_wiki_source", "knowledge.api") +def resolve_code_wiki_source( + data: CodeWikiResolveRequest, + current_user: User = Depends(security.get_current_user), + db: Session = Depends(get_db), +): + """Describe a repository before a wiki is bound to it. + + Registered above ``/{knowledge_base_id}`` routes by being declared here: a + literal path segment must not be reachable only when it fails to parse as an id. + + Answers 200 with ``exists: false`` rather than 404 for a repository the caller + cannot read. This is a form assisting input, not an assertion that something is + missing, and the two cases are indistinguishable on purpose. + """ + try: + source = SourceRepository.from_url(data.source_type, data.source_url) + except SourceAccessDenied as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) + ) from e + + resolved = resolve_repository(db, current_user.id, source) + return CodeWikiResolveResponse( + exists=resolved.exists, + visibility=resolved.visibility, + default_branch=resolved.default_branch, + name=resolved.name, + description=resolved.description, + access=resolved.access, + # Listed even when the caller cannot read the repository: what is disclosed + # is that a colleague documented it, to somebody who asked about it by name. + existing_wikis=[ + CodeWikiExisting(**wiki.__dict__) + for wiki in existing_wikis_for( + db, source.source_url, viewer_id=current_user.id + ) + ], + ) + + +@router.get("/code-wikis", response_model=CodeWikiListResponse) +@trace_sync("list_code_wikis", "knowledge.api") +def list_code_wikis( + page: int = Query(1, ge=1, description="Page number"), + limit: int = Query(20, ge=1, le=100, description="Items per page"), + current_user: User = Depends(security.get_current_user), + db: Session = Depends(get_db), +): + """Code wikis the caller may read, judged by the ordinary knowledge-base ACL. + + A code wiki belongs to whoever created it, so this is the same visibility every + other knowledge base has. The endpoint stays separate only because its list items + carry repository fields; it grants nothing the general list would not. + """ + visible = [ + kind + for kind in KnowledgeService.list_knowledge_bases( + db, current_user.id, scope=ResourceScope.ALL + ) + if (kind.json or {}).get("spec", {}).get("kbType") + == KnowledgeBaseType.CODE_WIKI.value + ] + + window = visible[(page - 1) * limit : page * limit] + # One grouped query for the whole page rather than a COUNT per wiki. + counts = KnowledgeService.get_document_counts(db, [kind.id for kind in window]) + + return CodeWikiListResponse( + items=[_code_wiki_list_item(kind, counts.get(kind.id, 0)) for kind in window], + total=len(visible), + ) + + +def _code_wiki_list_item(kind: Kind, document_count: int) -> CodeWikiListItem: + spec = (kind.json or {}).get("spec", {}) + source = spec.get("source") or {} + return CodeWikiListItem( + id=kind.id, + name=spec.get("name", kind.name), + description=spec.get("description"), + project_name=str(source.get("projectName", "") or ""), + source_url=str(source.get("sourceUrl", "") or ""), + last_published_at=spec.get(PUBLISHED_AT_KEY), + last_published_commit=str(spec.get(PUBLISHED_COMMIT_KEY, "") or ""), + document_count=document_count, + created_at=kind.created_at, + updated_at=kind.updated_at, + ) + + +@router.post( + "/code-wikis", + response_model=KnowledgeBaseResponse, + status_code=status.HTTP_201_CREATED, +) +@trace_sync("create_code_wiki", "knowledge.api") +def create_code_wiki( + data: CodeWikiCreate, + response: Response, + current_user: User = Depends(security.get_current_user), + db: Session = Depends(get_db), +): + """Create a code wiki bound to a source repository, or return the existing one. + + The requester must be able to read the repository, so that a wiki cannot be built + for one they have no access to. Who may read the result is decided the same way: + the wiki belongs to the configured wiki account rather than to the requester, so + knowledge-base ACLs grant nobody else access and the repository is what does. + + One repository has one wiki. Asking for a repository that already has one returns + it — the caller wanted that repository's wiki, not the act of creating it — and + the response is 200 rather than 201 to say which happened. + """ + try: + source = SourceRepository.from_url(data.source_type, data.source_url) + assert_user_can_read_source(db, current_user.id, source) + except SourceAccessDenied as e: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) from e + + existing_id = existing_wiki_id(db, source, owner_id=current_user.id) + if existing_id: + response.status_code = status.HTTP_200_OK + return KnowledgeBaseResponse.from_kind( + KnowledgeService._get_knowledge_base_record(db, existing_id), + KnowledgeService.get_document_count(db, existing_id), + ) + + # A blank name means "use the repository's". The client sends what it already + # resolved for the form, so this does not probe the provider again -- it has + # already been asked once by the gate above. Falling back to the URL keeps a + # caller that skipped the probe, or called the API directly, from creating a + # knowledge base with no name at all. + name = data.name.strip() or source.project_name + + try: + result = knowledge_orchestrator.create_knowledge_base( + db=db, + user=current_user, + name=name[:100], + description=data.description, + namespace=data.namespace or CODE_WIKI_NAMESPACE, + kb_type=KnowledgeBaseType.CODE_WIKI.value, + source=source, + ) + claim_repository(db, source, result.id) + db.commit() + add_span_event( + "knowledge.code_wiki.created", + { + "kb_id": str(result.id), + "project_name": source.project_name, + "owner_id": str(current_user.id), + }, + ) + _start_the_first_run(db, current_user, result.id) + return result + except IntegrityError as e: + # The same caller asking twice at once. UNIQUE (source_url, kind_id) settles + # it at the database rather than in a check-then-insert window; the loser + # returns the winner's wiki, since that is what it asked for. + db.rollback() + settled = existing_wiki_id(db, source, owner_id=current_user.id) + if settled: + response.status_code = status.HTTP_200_OK + return KnowledgeBaseResponse.from_kind( + KnowledgeService._get_knowledge_base_record(db, settled), + KnowledgeService.get_document_count(db, settled), + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Knowledge base with name '{data.name}' already exists", + ) from e + except ValueError as e: + db.rollback() + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) + ) from e + + +@router.get("/{knowledge_base_id}/code-wiki/pages", response_model=CodeWikiPageTree) +@trace_sync("get_code_wiki_pages", "knowledge.api") +def get_code_wiki_pages( + knowledge_base_id: int, + current_user: User = Depends(security.get_current_user), + db: Session = Depends(get_db), +): + """The navigation for a code wiki: every published page, nested and ordered. + + Access is the repository's, not the knowledge base's — a code wiki belongs to the + wiki account, so its ACL grants nobody else anything. + """ + knowledge_base = _readable_code_wiki(db, current_user, knowledge_base_id) + return CodeWikiPageTree( + pages=[_as_page_node(node) for node in page_tree(db, knowledge_base)] + ) + + +def _as_page_node(node) -> CodeWikiPageNode: + return CodeWikiPageNode( + path=node.path, + title=node.title, + document_id=node.document_id, + has_content=node.has_content, + children=[_as_page_node(child) for child in node.children], + ) + + +def _readable_code_wiki(db: Session, user: User, knowledge_base_id: int) -> Kind: + """Load a code wiki the caller may read, or refuse. + + The ordinary knowledge-base ACL decides, the same as for any other knowledge + base. Refusal is 404 rather than 403 so that the reply does not confirm the + existence of a wiki the caller cannot see. + """ + knowledge_base, has_access = KnowledgeService.get_knowledge_base( + db=db, knowledge_base_id=knowledge_base_id, user_id=user.id + ) + if knowledge_base is None or not has_access: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Code wiki not found" + ) + return knowledge_base + + +def _start_the_first_run(db: Session, user: User, knowledge_base_id: int) -> None: + """Begin generating the wiki that was just created. + + Without this a new wiki is empty until somebody finds the regenerate button, + which is not a flow anyone would guess. Failures are logged rather than raised: + the knowledge base is already committed and returning an error for it would say + the creation failed when it did not. The reader shows the empty state, and the + same button still starts a run. + """ + knowledge_base = KnowledgeService._get_knowledge_base_record(db, knowledge_base_id) + if knowledge_base is None: # pragma: no cover - just committed + return + try: + start_run(db, knowledge_base=knowledge_base, user=user) + except (CodeWikiRunError, GenerationInFlight) as e: + logger.warning( + "[code_wiki] first run not started for kb %s: %s", knowledge_base_id, e + ) + except Exception: # pragma: no cover - defensive + db.rollback() + logger.exception( + "[code_wiki] first run not started for kb %s", knowledge_base_id + ) + + +def _assert_caller_may_regenerate( + db: Session, user: User, knowledge_base: Kind +) -> None: + """Refuse a caller who may read the wiki but not change its repository. + + A knowledge base that is not a code wiki, or one with no repository recorded, + is left to ``start_run`` to reject: it already says which of the two it is, and + answering "you lack write access" for something with nothing to write to would + be a worse account of the refusal. + """ + spec = (knowledge_base.json or {}).get("spec", {}) + if spec.get("kbType") != KnowledgeBaseType.CODE_WIKI.value: + return + source = SourceRepository.from_spec(spec.get("source")) + if source is None or not source.project_name: + return + try: + assert_user_can_write_source(db, user.id, source) + except SourceAccessDenied as e: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) from e + + +@router.post( + "/{knowledge_base_id}/code-wiki/generations", + response_model=CodeWikiRunResponse, + status_code=status.HTTP_202_ACCEPTED, +) +@trace_sync("start_code_wiki_run", "knowledge.api") +def start_code_wiki_run( + knowledge_base_id: int, + data: CodeWikiRunCreate, + current_user: User = Depends(security.get_current_user), + db: Session = Depends(get_db), +): + """Regenerate a code wiki now, without waiting for its schedule. + + **Write access to the repository is required**, not merely the ability to read + the wiki. A run rewrites every page, so it is closer to changing the repository + than to reading its documentation — and a wiki shared with a reader would + otherwise let them spend a generation on somebody else's knowledge base. + + Answers 202 even when no run was needed. "The repository has not changed since the + published version" is a successful outcome, not a failure, and the response says + which it was. + """ + knowledge_base = _readable_code_wiki(db, current_user, knowledge_base_id) + _assert_caller_may_regenerate(db, current_user, knowledge_base) + + try: + started = start_run( + db, + knowledge_base=knowledge_base, + user=current_user, + head_commit=data.head_commit, + changed_paths=( + None + if data.changed_paths is None + else [ + ChangedPath(path=item.path, status=item.change_type) + for item in data.changed_paths + ] + ), + ) + except GenerationInFlight as e: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) from e + except CodeWikiRunError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) + ) from e + + add_span_event( + "knowledge.code_wiki.run_requested", + { + "kb_id": str(knowledge_base_id), + "started": str(started.started), + "mode": started.mode, + "user_id": str(current_user.id), + }, + ) + return CodeWikiRunResponse( + started=started.started, + mode=started.mode, + reason=started.reason, + generation_id=started.generation.id if started.generation else 0, + task_id=started.task_id, + ) + + @router.get("/{knowledge_base_id}", response_model=KnowledgeBaseResponse) @trace_sync("get_knowledge_base", "knowledge.api") def get_knowledge_base( diff --git a/backend/app/api/endpoints/wiki.py b/backend/app/api/endpoints/wiki.py index 7111f1e6bb..b676dc44d0 100644 --- a/backend/app/api/endpoints/wiki.py +++ b/backend/app/api/endpoints/wiki.py @@ -13,6 +13,7 @@ from app.core.wiki_config import wiki_settings from app.db.session import get_wiki_db from app.models.user import User +from app.models.wiki import WikiGeneration from app.schemas.wiki import ( WikiContentInDB, WikiContentWriteRequest, @@ -20,6 +21,7 @@ WikiGenerationDetail, WikiGenerationInDB, WikiGenerationListResponse, + WikiPageRead, WikiProjectDetail, WikiProjectInDB, WikiProjectListResponse, @@ -79,6 +81,50 @@ def _verify_internal_token( ) +def _internal_caller( + authorization: str = Header(default=""), + db: Session = Depends(get_db), +) -> Optional[User]: + """Authenticate an internal caller and say which user it is, if any. + + ``None`` means the fixed internal token was used, which is a trusted operator + rather than a person and is not scoped to one generation. + """ + _verify_internal_token(authorization=authorization, db=db) + token = authorization[7:].strip() + if token == wiki_settings.INTERNAL_API_TOKEN: + return None + try: + return security.get_current_user_from_token(token, db) + except Exception: # pragma: no cover - _verify_internal_token already accepted it + return None + + +def _assert_caller_owns_generation( + wiki_db: Session, caller: Optional[User], generation_id: int +) -> None: + """Refuse a caller asking about a generation that is not theirs. + + Authenticating a JWT says who is asking, not what they may ask about. Without + this any signed-in user could read any generation's pages, which is a different + thing from the documented "your own version" — and that version is a copy of a + wiki whose repository they may have no access to at all. + """ + if caller is None: + return + + generation = ( + wiki_db.query(WikiGeneration).filter(WikiGeneration.id == generation_id).first() + ) + if generation is None: + raise HTTPException(status_code=404, detail="Generation not found") + if generation.user_id != caller.id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="This generation belongs to another account", + ) + + def _resolve_user_id( account_id: Optional[int], current_user: User, main_db: Session ) -> int: @@ -217,6 +263,36 @@ def save_wiki_generation_contents( return None +@internal_router.get("/generations/{generation_id}/pages", response_model=WikiPageRead) +def read_wiki_generation_page( + generation_id: int, + path: str = Query(..., min_length=1, description="Stable page path to read"), + caller: Optional[User] = Depends(_internal_caller), + wiki_db: Session = Depends(get_wiki_db), +): + """Read one page of the version the agent is writing into (internal use). + + An incremental version begins as a complete copy of the published wiki, so the + agent's own generation is also the current wiki — which makes "read your own + version" both the capability it needs and the narrowest scope that provides it. + Without this the instruction to revise a page cannot be followed: the agent knows + the page's path and cannot see a word of what it says. + + Answers 404 when the path holds no page. That is a useful answer rather than a + failure — in an incremental run it means the page is new. + """ + _assert_caller_owns_generation(wiki_db, caller, generation_id) + page = wiki_service.get_generation_page( + wiki_db=wiki_db, generation_id=generation_id, path=path + ) + if page is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Generation {generation_id} has no page at '{path}'", + ) + return page + + @router.get( "/generations/{generation_id}/contents", response_model=list[WikiContentInDB] ) @@ -402,22 +478,36 @@ def get_wiki_stats_summary( # ========== Config Endpoints ========== @router.get("/config") def get_wiki_config( + code_wiki: bool = Query( + default=False, + description="Report on the code wiki team rather than the legacy wiki team", + ), current_user: User = Depends(security.get_current_user), main_db: Session = Depends(get_db), ): """Get wiki configuration including default team info and bound model""" from app.services.adapters.team_kinds import team_kinds_service - default_team_name = wiki_settings.DEFAULT_TEAM_NAME + # Which team to report on. The legacy wiki and a code wiki run different teams, + # so answering for the wrong one tells the caller a model is bound when the run + # it is about to start has none. + default_team_name = ( + wiki_settings.CODE_WIKI_TEAM_NAME + if code_wiki + else wiki_settings.DEFAULT_TEAM_NAME + ) default_user_id = wiki_settings.DEFAULT_USER_ID default_team = None has_bound_model = False bound_model_name = None if default_team_name: - # Determine which user_id to use for team lookup - # If DEFAULT_USER_ID is set (> 0), use it; otherwise use current user - lookup_user_id = default_user_id if default_user_id > 0 else current_user.id + # A code wiki runs as its own owner, so the team has to be resolved for the + # caller: answering for the legacy wiki account would report a bound model + # that the run about to start does not have. + lookup_user_id = ( + current_user.id if code_wiki or default_user_id <= 0 else default_user_id + ) # Find team by name and namespace team = team_kinds_service.get_team_by_name_and_namespace( diff --git a/backend/app/core/wiki_config.py b/backend/app/core/wiki_config.py index d8858ddc82..6c2a0229a5 100644 --- a/backend/app/core/wiki_config.py +++ b/backend/app/core/wiki_config.py @@ -19,6 +19,13 @@ class WikiSettings(BaseSettings): "wiki-team" # Default execution team name (matches init_data/01-default-resources.yaml) ) DEFAULT_AGENT_TYPE: str = "ClaudeCode" # Default agent type + # Team that runs knowledge-base code wikis (env var: WIKI_CODE_WIKI_TEAM_NAME). + # Separate from DEFAULT_TEAM_NAME because the two paths hand their agents + # different instructions and different submission rules; pointing both at one team + # would give the legacy wiki the page-path write contract it cannot satisfy. + CODE_WIKI_TEAM_NAME: str = ( + "code-wiki-team" # Matches init_data/02-public-resources.yaml + ) DEFAULT_USER_ID: int = 0 # Default user ID for task creation (0 = use current user) DEFAULT_LANGUAGE: str = ( "en" # Default language for wiki documentation generation (en/zh) diff --git a/backend/app/models/knowledge.py b/backend/app/models/knowledge.py index 1f23ca7b60..36cac6f850 100644 --- a/backend/app/models/knowledge.py +++ b/backend/app/models/knowledge.py @@ -47,6 +47,22 @@ class DocumentSourceType(str, PyEnum): TEXT = "text" # Pasted text TABLE = "table" # External table (DingTalk, Feishu, etc.) WEB = "web" # Web page (scraped URL) + CODE = "code" # Source file indexed for retrieval, not a browsable document + + +class ContentOrigin(str, PyEnum): + """Ownership of knowledge content, deciding whether regeneration may touch it. + + ``GENERATED`` content is owned by the generating agent: it may be overwritten by + a later run and removed by generation-set reconciliation. ``USER`` content is + owned by a person and is never touched by regeneration. + + ``USER`` is the safe default: mislabelling generated content as user-owned only + stops automatic cleanup, while the reverse would expose user content to deletion. + """ + + GENERATED = "generated" + USER = "user" class DocumentIndexStatus(str, PyEnum): @@ -131,6 +147,14 @@ class KnowledgeDocument(Base): source_config = Column( JSON, nullable=False, default={} ) # Source configuration (e.g., {"url": "..."} for table) + # Content ownership; see ContentOrigin. Defaults to ``user`` so that any row this + # service did not create is protected from the generated-content projection. + origin = Column( + String(20), + nullable=False, + default=ContentOrigin.USER.value, + server_default=ContentOrigin.USER.value, + ) # --- Helper properties for converted attachment reference --- @@ -210,6 +234,14 @@ class KnowledgeFolder(Base): # Self-referencing parent folder, 0 = root level parent_id = Column(Integer, nullable=False, default=0) name = Column(String(255), nullable=False) + # Content ownership; a top-level folder decides ownership for its subtree. + # Defaults to ``user`` for the same fail-safe reason as on documents. + origin = Column( + String(20), + nullable=False, + default=ContentOrigin.USER.value, + server_default=ContentOrigin.USER.value, + ) created_at = Column(DateTime, nullable=False, default=func.now()) updated_at = Column( DateTime, nullable=False, default=func.now(), onupdate=func.now() diff --git a/backend/app/models/wiki.py b/backend/app/models/wiki.py index d653b50e41..cb825d33da 100644 --- a/backend/app/models/wiki.py +++ b/backend/app/models/wiki.py @@ -17,6 +17,7 @@ Integer, String, Text, + UniqueConstraint, ) from sqlalchemy.sql import func @@ -33,16 +34,35 @@ class WikiProject(WikiBase): project_name = Column(String(200), nullable=False, index=True) project_type = Column(String(50), nullable=False, default="git", index=True) source_type = Column(String(50), nullable=False, default="github", index=True) - source_url = Column(String(500), nullable=False, unique=True) + source_url = Column(String(500), nullable=False) source_id = Column(String(100), nullable=True) source_domain = Column(String(100), nullable=True) description = Column(Text) ext = Column(JSON, comment="Project extension data") + # The code wiki this row registers, or 0 for a legacy wiki project. One row per + # (repository, wiki): a code wiki belongs to its creator, so a repository may + # have several, one per person who built one. + kind_id = Column( + Integer, + nullable=False, + default=0, + server_default="0", + index=True, + comment="Code wiki knowledge base built from this repository; 0 = legacy", + ) is_active = Column(Boolean, nullable=False, default=True) created_at = Column(DateTime, default=func.now()) updated_at = Column(DateTime, default=func.now(), onupdate=func.now()) __table_args__ = ( + # The pair rather than the URL alone. It is what settles two requests racing + # for the same (repository, wiki), which a check against a JSON field on the + # knowledge base could not — that leaves a window between read and insert + # exactly where it matters. Legacy rows carry kind_id = 0 and are therefore + # still limited to one per repository. + UniqueConstraint( + "source_url", "kind_id", name="uq_wiki_projects_source_url_kind_id" + ), { "sqlite_autoincrement": True, "mysql_engine": "InnoDB", @@ -82,6 +102,11 @@ class WikiGeneration(WikiBase): nullable=False, index=True, ) + # Knowledge base this version line belongs to (references kinds.id, no FK). + # Versions are owned by the KB rather than by the project: wiki_projects.source_url + # is globally unique, so project-scoped versions would be shared between knowledge + # bases tracking the same repository. 0 marks a row predating code_wiki. + kind_id = Column(Integer, nullable=False, default=0, server_default="0", index=True) user_id = Column(Integer, nullable=False, index=True) task_id = Column(big_integer_id_type(), nullable=False, default=0, index=True) team_id = Column(Integer, nullable=False) diff --git a/backend/app/repository/file_status.py b/backend/app/repository/file_status.py new file mode 100644 index 0000000000..e478ecca16 --- /dev/null +++ b/backend/app/repository/file_status.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Translating a host's words for a changed file into git's name-status letters. + +GitHub and Gitea both describe a compare entry with a word ("added", "modified"), +and they agree on all but one of them. The letters are what callers reason about, so +the translation lives here once rather than being restated per provider — two copies +of a near-identical mapping is exactly the shape that drifts. + +GitLab is absent on purpose: it reports booleans (``new_file``, ``deleted_file``, +``renamed_file``) rather than a word, so it has nothing to look up. +""" + +# Superset of what GitHub and Gitea emit. Sharing it is safe because the two never +# disagree on a word — Gitea says "deleted" where GitHub says "removed", and neither +# emits the other's spelling. +FILE_STATUS_LETTERS: dict[str, str] = { + "added": "A", + "changed": "M", + "copied": "A", + "deleted": "D", + "modified": "M", + "removed": "D", + "renamed": "R", +} + +# What a word we do not recognise becomes. An edit is the conservative reading: it +# neither invents a structural move nor hides a file from the diff. +DEFAULT_FILE_STATUS = "M" + + +def file_status_letter(status: str) -> str: + """Render one compare entry's status as a name-status letter.""" + return FILE_STATUS_LETTERS.get((status or "").lower(), DEFAULT_FILE_STATUS) diff --git a/backend/app/repository/gerrit_provider.py b/backend/app/repository/gerrit_provider.py index 665938a798..668e334bc4 100644 --- a/backend/app/repository/gerrit_provider.py +++ b/backend/app/repository/gerrit_provider.py @@ -64,7 +64,7 @@ def _get_git_infos( entries.append( { "git_domain": info.get("git_domain", ""), - "git_token": info.get("git_token", ""), + "git_token": self.decrypt_token(info.get("git_token", "")), "user_name": info.get("user_name", ""), "type": info.get("type", ""), "auth_type": info.get("auth_type", "digest"), diff --git a/backend/app/repository/gitea_provider.py b/backend/app/repository/gitea_provider.py index 298297fe72..a9b685f69b 100644 --- a/backend/app/repository/gitea_provider.py +++ b/backend/app/repository/gitea_provider.py @@ -17,11 +17,16 @@ from app.core.cache import cache_manager from app.core.config import settings from app.models.user import User +from app.repository.file_status import file_status_letter from app.repository.interfaces.repository_provider import RepositoryProvider from app.schemas.github import Branch, Repository from shared.utils.sensitive_data_masker import mask_string from shared.utils.url_util import build_url +# Reading repository state happens on a scheduled path, but an unresponsive instance +# must still fail rather than hold the worker. +REPO_STATE_TIMEOUT_SECONDS = 15 + class GiteaProvider(RepositoryProvider): """ @@ -61,7 +66,7 @@ def _get_git_infos( entries.append( { "git_domain": info.get("git_domain", ""), - "git_token": info.get("git_token", ""), + "git_token": self.decrypt_token(info.get("git_token", "")), "type": info.get("type", ""), "user_name": info.get("user_name", ""), } @@ -222,7 +227,8 @@ async def get_repositories( for repo in mapped_repos ] ) - except requests.exceptions.RequestException: + except requests.exceptions.RequestException as e: + self._log_domain_failure("list repositories", git_domain, e) continue return all_repos @@ -568,7 +574,8 @@ async def search_repositories( for r in filtered_repos ] ) - except requests.exceptions.RequestException: + except requests.exceptions.RequestException as e: + self._log_domain_failure("search repositories", git_domain, e) continue return all_results @@ -850,3 +857,96 @@ def check_user_project_access( except requests.exceptions.RequestException as e: self.logger.error(f"Failed to check Gitea repository access: {str(e)}") raise HTTPException(status_code=502, detail=f"Gitea API error: {str(e)}") + + # ---- repository state, for deciding whether a code wiki needs regenerating ---- + + def get_default_branch_head( + self, token: str, git_domain: str, repo_name: str + ) -> Dict[str, str]: + """Return the default branch and the commit it points at.""" + api_base_url = self._get_api_base_url(git_domain) + headers = self._build_headers(token) + + repo = requests.get( + f"{api_base_url}/repos/{repo_name}", + headers=headers, + timeout=REPO_STATE_TIMEOUT_SECONDS, + ) + repo.raise_for_status() + branch_name = (repo.json() or {}).get("default_branch") or "main" + + branch = requests.get( + f"{api_base_url}/repos/{repo_name}/branches/{branch_name}", + headers=headers, + timeout=REPO_STATE_TIMEOUT_SECONDS, + ) + branch.raise_for_status() + commit = (branch.json() or {}).get("commit") or {} + return {"branch": branch_name, "commit": commit.get("id", "")} + + def get_changed_files( + self, token: str, git_domain: str, repo_name: str, base: str, head: str + ) -> Optional[List[Dict[str, str]]]: + """List the files that changed between two commits. + + Returns: + One entry per file, or ``None`` when this Gitea cannot answer. The + compare endpoint arrived in Gitea 1.17 and self-hosted instances lag, so + an older one reports the diff as unknown and gets a full rebuild rather + than an error. + """ + api_base_url = self._get_api_base_url(git_domain) + response = requests.get( + f"{api_base_url}/repos/{repo_name}/compare/{base}...{head}", + headers=self._build_headers(token), + timeout=REPO_STATE_TIMEOUT_SECONDS, + ) + if response.status_code == 404: + self.logger.info( + "Gitea at %s has no compare endpoint; reporting the diff as unknown", + git_domain, + ) + return None + response.raise_for_status() + + payload = response.json() or {} + return [ + { + "path": entry.get("filename", ""), + "status": file_status_letter(entry.get("status", "")), + } + for entry in (payload.get("files") or []) + if entry.get("filename") + ] + + def describe_repository( + self, token: str, git_domain: str, repo_name: str + ) -> Optional[Dict[str, Any]]: + """Metadata a caller needs before binding a repository, or ``None``. + + ``None`` means "not readable with what was supplied"; private and absent are + deliberately not told apart. + """ + api_base_url = self._get_api_base_url(git_domain) + headers = {"Accept": "application/json"} + if token: + headers["Authorization"] = f"token {self.decrypt_token(token)}" + try: + response = requests.get( + f"{api_base_url}/repos/{repo_name}", + headers=headers, + timeout=REPO_STATE_TIMEOUT_SECONDS, + ) + except requests.exceptions.RequestException as e: + self._log_domain_failure("describe repository", git_domain, e) + return None + if response.status_code != 200: + return None + + data = response.json() or {} + return { + "visibility": "private" if data.get("private") else "public", + "default_branch": data.get("default_branch") or "", + "name": data.get("full_name") or repo_name, + "description": data.get("description") or "", + } diff --git a/backend/app/repository/gitee_provider.py b/backend/app/repository/gitee_provider.py index 70b4e2d078..18f27022e9 100644 --- a/backend/app/repository/gitee_provider.py +++ b/backend/app/repository/gitee_provider.py @@ -61,7 +61,7 @@ def _get_git_infos( entries.append( { "git_domain": info.get("git_domain", ""), - "git_token": info.get("git_token", ""), + "git_token": self.decrypt_token(info.get("git_token", "")), "type": info.get("type", ""), } ) @@ -195,8 +195,8 @@ async def get_repositories( for repo in repos ] ) - except requests.exceptions.RequestException: - # skip failed domain, continue others + except requests.exceptions.RequestException as e: + self._log_domain_failure("list repositories", git_domain, e) continue return all_repos @@ -317,9 +317,14 @@ def validate_token(self, token: str, git_domain: str = None) -> Dict[str, Any]: # Use custom domain if provided, otherwise use default api_base_url = self._get_api_base_url(git_domain) + # Re-validating an already stored credential hands this the ciphertext, the + # same way the other providers are given it. Decrypting here keeps that case + # from reading as an invalid token. + decrypt_token = self.decrypt_token(token) + try: response = requests.get( - f"{api_base_url}/user", params={"access_token": token} + f"{api_base_url}/user", params={"access_token": decrypt_token} ) if response.status_code == 401: @@ -559,8 +564,8 @@ async def search_repositories( for r in filtered_repos ] ) - except requests.exceptions.RequestException: - # skip this domain on error + except requests.exceptions.RequestException as e: + self._log_domain_failure("search repositories", git_domain, e) continue return all_results diff --git a/backend/app/repository/github_provider.py b/backend/app/repository/github_provider.py index 4b6d684ba0..ac1baa42d2 100644 --- a/backend/app/repository/github_provider.py +++ b/backend/app/repository/github_provider.py @@ -17,11 +17,21 @@ from app.core.cache import cache_manager from app.core.config import settings from app.models.user import User +from app.repository.file_status import file_status_letter from app.repository.interfaces.repository_provider import RepositoryProvider from app.schemas.github import Branch, Repository from shared.utils.sensitive_data_masker import mask_string from shared.utils.url_util import build_url +# Repository access checks run inside a user-facing request (creating a code wiki), +# so an unresponsive provider must fail rather than hold the worker. Only this check +# is bounded here; the other calls in this module remain as they were. +ACCESS_CHECK_TIMEOUT_SECONDS = 15 + +# GitHub's compare endpoint returns at most this many files and gives no reliable +# signal that it truncated, so hitting it is treated as "the diff is unknown". +GITHUB_COMPARE_FILE_LIMIT = 300 + class GitHubProvider(RepositoryProvider): """ @@ -61,7 +71,7 @@ def _get_git_infos( entries.append( { "git_domain": info.get("git_domain", ""), - "git_token": info.get("git_token", ""), + "git_token": self.decrypt_token(info.get("git_token", "")), "type": info.get("type", ""), } ) @@ -194,8 +204,8 @@ async def get_repositories( for repo in repos ] ) - except requests.exceptions.RequestException: - # skip failed domain, continue others + except requests.exceptions.RequestException as e: + self._log_domain_failure("list repositories", git_domain, e) continue return all_repos @@ -570,8 +580,8 @@ async def search_repositories( for r in filtered_repos ] ) - except requests.exceptions.RequestException: - # skip this domain on error + except requests.exceptions.RequestException as e: + self._log_domain_failure("search repositories", git_domain, e) continue return all_results @@ -859,7 +869,11 @@ def check_user_project_access( # First get the current user info from the token try: - user_response = requests.get(f"{api_base_url}/user", headers=headers) + user_response = requests.get( + f"{api_base_url}/user", + headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) if user_response.status_code == 401: return { "has_access": False, @@ -880,6 +894,7 @@ def check_user_project_access( permission_response = requests.get( f"{api_base_url}/repos/{repo_name}/collaborators/{username}/permission", headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, ) if permission_response.status_code == 404: @@ -924,6 +939,7 @@ def check_user_project_access( repo_response = requests.get( f"{api_base_url}/repos/{repo_name}", headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, ) if repo_response.status_code == 200: return { @@ -942,3 +958,115 @@ def check_user_project_access( } self.logger.error(f"Failed to check repository access: {str(e)}") raise HTTPException(status_code=502, detail=f"GitHub API error: {str(e)}") + + # ---- repository state, for deciding whether a code wiki needs regenerating ---- + + def get_default_branch_head( + self, token: str, git_domain: str, repo_name: str + ) -> Dict[str, str]: + """Return the default branch and the commit it points at. + + Two targeted calls rather than ``get_branches``, which pages through the + whole branch list to answer a question about one branch. + """ + api_base_url = self._get_api_base_url(git_domain) + headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + } + + repo = requests.get( + f"{api_base_url}/repos/{repo_name}", + headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + repo.raise_for_status() + branch_name = repo.json().get("default_branch") or "main" + + branch = requests.get( + f"{api_base_url}/repos/{repo_name}/branches/{branch_name}", + headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + branch.raise_for_status() + commit = (branch.json() or {}).get("commit") or {} + return {"branch": branch_name, "commit": commit.get("sha", "")} + + def get_changed_files( + self, token: str, git_domain: str, repo_name: str, base: str, head: str + ) -> Optional[List[Dict[str, str]]]: + """List the files that changed between two commits. + + Returns: + One entry per file, or ``None`` when the answer would be incomplete. + GitHub truncates ``compare`` at 300 files, and a partial diff read as a + complete one would pick an incremental run for a change big enough to + need a rebuild. + """ + api_base_url = self._get_api_base_url(git_domain) + response = requests.get( + f"{api_base_url}/repos/{repo_name}/compare/{base}...{head}", + headers={ + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + }, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + response.raise_for_status() + payload = response.json() or {} + + files = payload.get("files") or [] + if len(files) >= GITHUB_COMPARE_FILE_LIMIT: + self.logger.info( + "Compare of %s is at GitHub's %s file limit; reporting the diff as " + "unknown rather than truncated", + repo_name, + GITHUB_COMPARE_FILE_LIMIT, + ) + return None + + return [ + { + "path": entry.get("filename", ""), + "status": file_status_letter(entry.get("status", "")), + } + for entry in files + if entry.get("filename") + ] + + def describe_repository( + self, token: str, git_domain: str, repo_name: str + ) -> Optional[Dict[str, Any]]: + """Metadata a caller needs before binding a repository, or ``None``. + + GitHub answers 404 rather than 403 for a private repository an anonymous + caller cannot see, which is the behaviour to preserve rather than unpick: + "not readable" is the whole answer a caller is entitled to. + + Anonymous requests are rate limited to 60 an hour per address, so callers + are expected to cache this rather than probe per keystroke. + """ + api_base_url = self._get_api_base_url(git_domain) + headers = {"Accept": "application/vnd.github.v3+json"} + if token: + headers["Authorization"] = f"token {self.decrypt_token(token)}" + try: + response = requests.get( + f"{api_base_url}/repos/{repo_name}", + headers=headers, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + except requests.exceptions.RequestException as e: + self._log_domain_failure("describe repository", git_domain, e) + return None + if response.status_code != 200: + return None + + data = response.json() or {} + return { + "visibility": data.get("visibility") + or ("private" if data.get("private") else "public"), + "default_branch": data.get("default_branch") or "", + "name": data.get("full_name") or repo_name, + "description": data.get("description") or "", + } diff --git a/backend/app/repository/gitlab_provider.py b/backend/app/repository/gitlab_provider.py index 85c8126c94..1e0e6ffb34 100644 --- a/backend/app/repository/gitlab_provider.py +++ b/backend/app/repository/gitlab_provider.py @@ -9,6 +9,7 @@ import asyncio import logging from typing import Any, Dict, List, Optional +from urllib.parse import quote import requests from fastapi import HTTPException @@ -20,6 +21,11 @@ from app.schemas.github import Branch, Repository from shared.utils.url_util import build_url +# Repository access checks run inside a user-facing request (creating a code wiki), +# so an unresponsive provider must fail rather than hold the worker. Only this check +# is bounded here; the other calls in this module remain as they were. +ACCESS_CHECK_TIMEOUT_SECONDS = 15 + class GitLabProvider(RepositoryProvider): """ @@ -43,7 +49,11 @@ def _get_git_infos( git_domain: Optional domain to filter a specific GitLab entry Returns: - List of dictionaries containing git_domain, git_token, type + List of dictionaries containing git_domain, git_token, type. The token is + decrypted here, at the one place entries are built, so every caller below + holds something usable. Tokens are stored encrypted, and passing the + ciphertext to GitLab reads as an authentication failure rather than as a + configuration error. Raises: HTTPException: Raised when GitLab information is not configured @@ -59,7 +69,7 @@ def _get_git_infos( entries.append( { "git_domain": info.get("git_domain", ""), - "git_token": info.get("git_token", ""), + "git_token": self.decrypt_token(info.get("git_token", "")), "type": info.get("type", ""), } ) @@ -286,8 +296,11 @@ async def get_repositories( self._fetch_all_repositories_async(user, git_token, git_domain) ) - except requests.exceptions.RequestException: - # skip failed domain, continue others + except requests.exceptions.RequestException as e: + # Skip the failed domain and keep the others, but say so: silently + # returning fewer repositories is indistinguishable from the user + # owning none, which hides an expired or rejected token. + self._log_domain_failure("list repositories", git_domain, e) continue return all_repos @@ -626,8 +639,10 @@ async def search_repositories( for r in filtered_repos ] ) - except requests.exceptions.RequestException: - # skip this domain on error + except requests.exceptions.RequestException as e: + # Same reasoning as get_repositories: an empty search result and a + # rejected token must not look alike in the logs. + self._log_domain_failure("search repositories", git_domain, e) continue return all_results @@ -894,6 +909,7 @@ def check_user_project_access( method="GET", url=f"{api_base_url}/user", token=decrypt_token, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, ) user_data = user_response.json() user_id = user_data.get("id") @@ -920,6 +936,7 @@ def check_user_project_access( method="GET", url=f"{api_base_url}/projects/{encoded_project_id}/members/all/{user_id}", token=decrypt_token, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, ) member_data = member_response.json() access_level = member_data.get("access_level", 0) @@ -949,3 +966,118 @@ def check_user_project_access( } self.logger.error(f"Failed to check project access: {str(e)}") raise HTTPException(status_code=502, detail=f"GitLab API error: {str(e)}") + + # ---- repository state, for deciding whether a code wiki needs regenerating ---- + + def get_default_branch_head( + self, token: str, git_domain: str, repo_name: str + ) -> Dict[str, str]: + """Return the default branch and the commit it points at.""" + api_base_url = self._get_api_base_url(git_domain) + encoded = quote(repo_name, safe="") + + project = self._make_request_with_auth_retry( + method="GET", + url=f"{api_base_url}/projects/{encoded}", + token=token, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + branch_name = (project.json() or {}).get("default_branch") or "main" + + branch = self._make_request_with_auth_retry( + method="GET", + url=f"{api_base_url}/projects/{encoded}/repository/branches/" + f"{quote(branch_name, safe='')}", + token=token, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + commit = (branch.json() or {}).get("commit") or {} + return {"branch": branch_name, "commit": commit.get("id", "")} + + def get_changed_files( + self, token: str, git_domain: str, repo_name: str, base: str, head: str + ) -> Optional[List[Dict[str, str]]]: + """List the files that changed between two commits. + + Returns: + One entry per file, or ``None`` when GitLab reports that it gave up + computing the diff. A partial diff read as a complete one would pick an + incremental run for a change big enough to need a rebuild. + """ + api_base_url = self._get_api_base_url(git_domain) + encoded = quote(repo_name, safe="") + + response = self._make_request_with_auth_retry( + method="GET", + url=f"{api_base_url}/projects/{encoded}/repository/compare", + token=token, + params={"from": base, "to": head}, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + payload = response.json() or {} + + if payload.get("compare_timeout"): + self.logger.info( + "GitLab timed out comparing %s; reporting the diff as unknown", + repo_name, + ) + return None + + changed: List[Dict[str, str]] = [] + for entry in payload.get("diffs") or []: + path = entry.get("new_path") or entry.get("old_path") or "" + if not path: + continue + if entry.get("new_file"): + status = "A" + elif entry.get("deleted_file"): + status = "D" + elif entry.get("renamed_file"): + status = "R" + else: + status = "M" + changed.append({"path": path, "status": status}) + return changed + + def describe_repository( + self, token: str, git_domain: str, repo_name: str + ) -> Optional[Dict[str, Any]]: + """Metadata a caller needs before binding a repository, or ``None``. + + ``None`` means "not readable with what was supplied" and deliberately does + not distinguish private from absent: answering that would tell an anonymous + caller which private repositories exist. + + Works without a token, which is the point — a public repository is readable + by anyone, and requiring a credential to see that would make a wiki more + closed than the repository it documents. + """ + api_base_url = self._get_api_base_url(git_domain) + url = f"{api_base_url}/projects/{quote(repo_name, safe='')}" + try: + if token: + response = self._make_request_with_auth_retry( + method="GET", + url=url, + token=token, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + else: + response = requests.get( + url, + headers={"Accept": "application/json"}, + timeout=ACCESS_CHECK_TIMEOUT_SECONDS, + ) + if response.status_code != 200: + return None + except requests.exceptions.RequestException as e: + self._log_domain_failure("describe repository", git_domain, e) + return None + + data = response.json() or {} + return { + "visibility": data.get("visibility") or "private", + "default_branch": data.get("default_branch") or "", + "name": data.get("path_with_namespace") or repo_name, + "description": data.get("description") or "", + } diff --git a/backend/app/repository/interfaces/repository_provider.py b/backend/app/repository/interfaces/repository_provider.py index 957eb34e65..825df21e0d 100644 --- a/backend/app/repository/interfaces/repository_provider.py +++ b/backend/app/repository/interfaces/repository_provider.py @@ -6,6 +6,7 @@ Repository provider interface, defining methods related to code repositories """ +import logging from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional @@ -69,6 +70,28 @@ def decrypt_token(self, token: str) -> str: def _is_token_encrypted(self, token: str) -> bool: return is_token_encrypted(token) + def _log_domain_failure( + self, action: str, git_domain: str, error: Exception + ) -> None: + """Report a domain that dropped out of an aggregated result. + + Providers gather repositories across every domain a user configured and skip + the ones that fail, so a rejected credential otherwise reaches the caller as + a short list rather than as an error. The status is worth separating out: + 401 means the token was refused, which is a different problem for the user + than the host being unreachable. + """ + response = getattr(error, "response", None) + status = getattr(response, "status_code", None) + logging.getLogger(type(self).__module__).warning( + "Could not %s from %s domain %s (status=%s): %s", + action, + getattr(self, "type", "git"), + git_domain or "unknown", + status if status is not None else "none", + error, + ) + @abstractmethod def validate_token(self, token: str) -> Dict[str, Any]: """ diff --git a/backend/app/schemas/kind.py b/backend/app/schemas/kind.py index e9dc763df2..f2352883d2 100644 --- a/backend/app/schemas/kind.py +++ b/backend/app/schemas/kind.py @@ -502,7 +502,7 @@ class TeamSpec(QuickPhraseMixin): """Team specification""" members: List[TeamMember] - collaborationModel: str # pipeline、route、coordinate、collaborate + collaborationModel: str # solo、pipeline、route、coordinate、collaborate bind_mode: Optional[List[str]] = None # ['chat', 'code'] or empty list for none description: Optional[str] = None # Team description icon: Optional[str] = None # Icon ID from preset icon library @@ -1008,7 +1008,24 @@ class KnowledgeBaseSpec(BaseModel): ) kbType: Optional[str] = Field( "notebook", - description="Default opening view: 'notebook' opens Notebook view by default, 'classic' opens document view by default", + description=( + "What this knowledge base is: 'notebook' or 'classic' select the default " + "opening view and may be switched freely; 'code_wiki' binds the knowledge " + "base to a source repository and is fixed at creation." + ), + ) + source: Optional[Dict[str, Any]] = Field( + None, + description="Source repository a code wiki is generated from", + ) + publishedGenerationId: int = Field( + 0, + description=( + "Generation whose content is currently projected into this knowledge base; " + "0 means nothing has been published yet. Sole authority for which version " + "is live — never infer it from the latest completed generation, because a " + "generation that finished but failed its publish gate is not published." + ), ) document_count: Optional[int] = Field( default=0, description="Cached document count" diff --git a/backend/app/schemas/knowledge.py b/backend/app/schemas/knowledge.py index 35eb41d663..1230e83498 100644 --- a/backend/app/schemas/knowledge.py +++ b/backend/app/schemas/knowledge.py @@ -54,6 +54,9 @@ class DocumentSourceType(str, Enum): TABLE = "table" WEB = "web" ATTACHMENT = "attachment" + # Source file indexed for retrieval rather than a browsable document. Declared so + # ensure_source_type_enum does not silently coerce it to FILE. + CODE = "code" class DocumentIndexStatus(str, Enum): @@ -77,6 +80,25 @@ class ResourceScope(str, Enum): ALL = "all" +class KnowledgeBaseType(str, Enum): + """What a knowledge base is, stored at ``spec.kbType``. + + ``notebook`` and ``classic`` differ only in the default opening view and can be + switched freely. ``code_wiki`` is a different thing altogether: it is bound to a + source repository, generated and maintained by an agent, and published by the + server rather than edited by hand. + + The three are mutually exclusive, so one field carries all of them. A knowledge + base may move between ``notebook`` and ``classic``, but **never** into or out of + ``code_wiki``: doing so would either orphan a repository binding and its version + history, or produce a code wiki with no repository at all. + """ + + NOTEBOOK = "notebook" + CLASSIC = "classic" + CODE_WIKI = "code_wiki" + + # ============== Knowledge Base Schemas ============== # Note: RetrieverRef, EmbeddingModelRef, HybridWeights, RetrievalConfig # are imported from app.schemas.kind to maintain single source of truth @@ -134,9 +156,17 @@ class KnowledgeBaseCreate(MultimodalAnalysisFieldsMixin): default="read", description="Minimum capability required for direct knowledge base access", ) - kb_type: Optional[str] = Field( - "notebook", - description="Default opening view: 'notebook' opens Notebook view by default, 'classic' opens document view by default", + kb_type: KnowledgeBaseType = Field( + KnowledgeBaseType.NOTEBOOK, + description=( + "'notebook' opens Notebook view by default, 'classic' opens document view. " + "'code_wiki' is a repository-backed, agent-generated wiki and can only be " + "created through the dedicated code wiki endpoint." + ), + ) + source: Optional[Dict[str, Any]] = Field( + None, + description="Source repository a code wiki is generated from (code wikis only)", ) retrieval_config: Optional[RetrievalConfigCreate] = Field( None, description="Retrieval configuration" @@ -267,8 +297,197 @@ def validate_guided_questions(cls, v): return v +class CodeWikiCreate(BaseModel): + """Request to create a code wiki bound to a source repository. + + Kept separate from ``KnowledgeBaseCreate`` because a code wiki needs a repository + and passes a repository-access gate, neither of which applies to other knowledge + bases. Its kind is set by the endpoint, not by the caller. + """ + + name: str = Field( + "", + max_length=100, + description=( + "Left blank, the repository's own name is used. Filled in here rather " + "than in the browser, where a pre-filled box would read as the caller's " + "own input." + ), + ) + description: Optional[str] = Field(None, max_length=500) + namespace: Optional[str] = Field( + None, + max_length=100, + description=( + "Where to file the wiki, as for any other knowledge base. Defaults to " + "the creator's personal namespace." + ), + ) + source_type: Literal["github", "gitlab", "gitea"] = Field( + ..., + description=( + "Which platform hosts the repository. Required because a self-hosted " + "GitLab or Gitea cannot be told apart by its domain." + ), + ) + source_url: str = Field( + ..., + min_length=1, + max_length=500, + description=( + "Repository URL. The host and project are derived from it, so the " + "repository checked for access is necessarily the one that gets cloned." + ), + ) + + +class CodeWikiChangedPath(BaseModel): + """One file the repository changed since the published commit.""" + + path: str = Field(..., min_length=1, max_length=1000) + change_type: Literal["A", "M", "D", "R"] = Field( + "M", + description="Git status letter: A added, M modified, D deleted, R renamed", + ) + + +class CodeWikiListItem(BaseModel): + """One code wiki, as a list shows it. + + Everything here is read from the knowledge base's spec, written when the version + was published. A list that had to join every wiki against its generations for + these three fields would pay for them on every page load. + """ + + id: int + name: str + description: Optional[str] = None + project_name: str = Field("", description="Repository the wiki documents") + source_url: str = Field("", description="Repository URL") + last_published_at: Optional[str] = Field( + None, description="When the live version was published; null if never" + ) + last_published_commit: str = Field( + "", description="Commit the live version documents" + ) + document_count: int = 0 + created_at: datetime + updated_at: datetime + + +class CodeWikiListResponse(BaseModel): + """Code wikis the caller may read.""" + + items: List[CodeWikiListItem] + total: int + + +class CodeWikiPageNode(BaseModel): + """One node of the reader's navigation. + + Every node is a page. The hierarchy comes from the page paths and the order from + the knowledge base's recorded order, so the client renders what it is given + rather than reassembling a tree from a paginated document list and a separate + array — two things it would have to keep consistent itself. + """ + + path: str = Field(..., description="Stable page path; identity, not a label") + title: str = Field(..., description="What the page is called") + document_id: int = Field(0, description="0 for a section with no page of its own") + has_content: bool = True + children: List["CodeWikiPageNode"] = Field(default_factory=list) + + +class CodeWikiPageTree(BaseModel): + """The navigation for one code wiki.""" + + pages: List[CodeWikiPageNode] + + +class CodeWikiRunCreate(BaseModel): + """Request to regenerate a code wiki now. + + Both fields are optional and describe the repository's current state. Supplying + neither is safe but expensive: with no commit to compare against, the run cannot + tell what changed and rebuilds the whole wiki rather than guessing. + """ + + head_commit: str = Field( + "", + max_length=64, + description=( + "Commit the repository is at now. Compared against the published commit " + "to decide whether anything needs regenerating at all." + ), + ) + changed_paths: Optional[List[CodeWikiChangedPath]] = Field( + None, + description=( + "Files changed since the published commit. Absent means unknown, which " + "forces a full rebuild; an empty list means nothing changed." + ), + ) + + +class CodeWikiRunResponse(BaseModel): + """What happened when a code wiki was asked to regenerate.""" + + started: bool = Field(..., description="Whether a run was actually created") + mode: str = Field("", description="Run mode chosen: full, incremental or skip") + reason: str = Field("", description="Why that mode was chosen") + generation_id: int = Field(0, description="The version being written, when started") + task_id: int = Field(0, description="Task running the agent, when started") + + +class CodeWikiExisting(BaseModel): + """A wiki of this repository somebody has already built.""" + + id: int + name: str + owner_name: str = Field("", description="Who to ask, when it is not accessible") + accessible: bool = Field( + False, description="Whether the caller can already open it" + ) + + +class CodeWikiResolveRequest(BaseModel): + """Ask what is known about a repository before binding a wiki to it.""" + + source_type: Literal["github", "gitlab", "gitea"] + source_url: str = Field(..., min_length=1, max_length=500) + + +class CodeWikiResolveResponse(BaseModel): + """What the create form needs, in one answer rather than three probes. + + ``exists`` false means "not readable with what you have"; private and absent are + deliberately not told apart, since distinguishing them would disclose which + private repositories exist. + """ + + exists: bool + visibility: str = Field("", description="public or private, when readable") + default_branch: str = Field("", description="Saves listing branches to pick one") + name: str = Field("", description="Repository path, offered as the wiki's name") + description: str = Field("", description="Offered as the wiki's description") + access: str = Field("none", description="public, member, or none") + existing_wikis: List["CodeWikiExisting"] = Field( + default_factory=list, + description=( + "Code wikis that already document this repository, whoever owns them. " + "Named rather than counted: asking for a share needs somebody to ask." + ), + ) + + class KnowledgeBaseTypeUpdate(BaseModel): - """Schema for updating the default opening view.""" + """Schema for updating the default opening view. + + The pattern deliberately excludes ``code_wiki``: it is not a view preference but a + binding to a source repository, so it can be neither adopted nor abandoned by + toggling a view. Turning a code wiki into a notebook would orphan its repository + and version history; the reverse would produce a code wiki with no repository. + """ kb_type: str = Field( ..., @@ -286,9 +505,9 @@ class KnowledgeBaseResponse(MultimodalAnalysisResponseFieldsMixin): user_id: int namespace: str direct_access_requirement: Literal["read", "edit"] = "read" - kb_type: Optional[str] = Field( - "notebook", - description="Default opening view: 'notebook' opens Notebook view by default, 'classic' opens document view by default", + kb_type: KnowledgeBaseType = Field( + KnowledgeBaseType.NOTEBOOK, + description="What this knowledge base is; see KnowledgeBaseType", ) document_count: int is_active: bool @@ -354,7 +573,7 @@ def from_kind(cls, kind, document_count: int = 0): # Extract summary_model_ref from spec summary_model_ref = spec.get("summaryModelRef") # Extract kb_type from spec, default to 'notebook' for backward compatibility - kb_type = spec.get("kbType", "notebook") + kb_type = spec.get("kbType", KnowledgeBaseType.NOTEBOOK.value) # Extract guided questions from spec guided_questions = spec.get("guidedQuestions") diff --git a/backend/app/schemas/wiki.py b/backend/app/schemas/wiki.py index 7c144bbe3d..deb05500d3 100644 --- a/backend/app/schemas/wiki.py +++ b/backend/app/schemas/wiki.py @@ -136,6 +136,11 @@ class WikiContentSection(BaseModel): content: str parent_id: Optional[int] = None ext: Optional[Dict[str, Any]] = None + # Stable identity of the page, e.g. "architecture/backend". Pages are matched on + # it rather than on the title, so that rewording a heading revises the existing + # page instead of replacing it — which would change the document id the RAG index + # and stored citations depend on. Optional only for the legacy write path. + path: Optional[str] = None class WikiContentSummary(BaseModel): @@ -146,6 +151,19 @@ class WikiContentSummary(BaseModel): model: Optional[str] = None tokens_used: Optional[int] = None structure_order: Optional[List[str]] = None + # Commit the agent actually documented. It read the working tree, whereas the + # trigger only knew what it was told, and this is the value the next run's mode + # decision compares against — so a wrong one here costs a needless full rebuild + # or, worse, a skipped set of changes. + head_commit: Optional[str] = None + + +class WikiPageRead(BaseModel): + """One page of a version, as the agent reads it back.""" + + path: str + title: str + content: str class WikiContentWriteRequest(BaseModel): @@ -154,6 +172,10 @@ class WikiContentWriteRequest(BaseModel): generation_id: int sections: List[WikiContentSection] summary: Optional[WikiContentSummary] = None + # Pages the agent is declaring gone. Only it knows which page covered a module + # that no longer exists, and an incremental version starts as a copy of the + # published one, so not writing a page cannot mean removing it. + removed_paths: List[str] = Field(default_factory=list) class WikiContentCreate(BaseModel): diff --git a/backend/app/services/knowledge/code_wiki/__init__.py b/backend/app/services/knowledge/code_wiki/__init__.py new file mode 100644 index 0000000000..6b5336fb2b --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/__init__.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""A knowledge base generated and maintained by an agent from a source repository. + +The modules divide along what each one decides: + +- ``source`` / ``repo_state`` — which repository, and what it is at right now. +- ``run_mode`` — whether a run is needed at all, and how much of the wiki it rebuilds. +- ``version_store`` / ``page_path`` — the versions the agent writes into, and the page + identity that lets an unchanged page keep its document id across runs. +- ``prompts`` — what the agent is told, which differs by mode. +- ``projection_plan`` / ``projection`` — what publishing a version would change, and + the fixed ordering that applies it. +- ``publish_gate`` / ``publisher`` — whether a finished version may go live, and the + single place that moves ``spec.publishedGenerationId``. +- ``generation`` / ``runner`` — the spine: starting a run and concluding it. +- ``side_effects`` — the adapters for the work that cannot join a transaction. + +Nothing is re-exported here on purpose. ``app.services.knowledge`` resolves its own +exports lazily to avoid import cycles, and eagerly importing this package's modules +from here would pull that whole chain in at package-import time. Importing by module +path also keeps neighbours like ``publisher`` and ``publish_gate`` distinguishable at +the call site. +""" diff --git a/backend/app/services/knowledge/code_wiki/generation.py b/backend/app/services/knowledge/code_wiki/generation.py new file mode 100644 index 0000000000..d3adf0fa9b --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/generation.py @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Starting and finishing a code wiki generation run. + +This is the spine the rest of the pieces hang from. Everything else decides something +— which mode, which pages changed, whether a version may be published — but nothing +happens until a run is started and later concluded, and those are the two moments +where the invariants have to be enforced together: + +**Starting** picks a mode, refuses to begin while another run is genuinely in flight, +reclaims one whose worker is gone, creates the version and seeds it. Seeding belongs +here rather than in the agent because a version that is not a complete snapshot would +be projected as one, and every page the run did not touch would read as an orphan. + +**Finishing** records how the run ended and, only for a run that succeeded, offers the +version to the publish gate. A failed run leaves the published pointer where it was, +which is what makes the next scheduled run pick the work up again rather than skip it. +""" + +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Optional, Sequence + +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.user import User +from app.models.wiki import WikiGeneration, WikiGenerationStatus, WikiGenerationType +from app.services.knowledge.code_wiki.projection import ProjectionSideEffects +from app.services.knowledge.code_wiki.publish_gate import PublishPolicy +from app.services.knowledge.code_wiki.publisher import ( + PublishResult, + publish_generation, + published_generation_id, + read_version_pages, +) +from app.services.knowledge.code_wiki.run_mode import ( + ChangedPath, + RunMode, + RunModeDecision, + RunModePolicy, + decide_run_mode, +) +from app.services.knowledge.code_wiki.version_store import ( + reclaim_stale_generations, + seed_from_published, +) + +logger = logging.getLogger(__name__) + +SOURCE_COMMIT_KEY = "commit" + + +class GenerationInFlight(RuntimeError): + """Raised when a run is already working on this wiki.""" + + +@dataclass(frozen=True) +class StartedGeneration: + """A run that has been created and is ready for the agent.""" + + generation: Optional[WikiGeneration] + decision: RunModeDecision + seeded_pages: int = 0 + + @property + def started(self) -> bool: + return self.generation is not None + + +def published_commit(db: Session, knowledge_base: Kind) -> str: + """Commit the currently published wiki was generated from.""" + current = published_generation_id(knowledge_base) + if not current: + return "" + generation = db.get(WikiGeneration, current) + if generation is None: + return "" + return str((generation.source_snapshot or {}).get(SOURCE_COMMIT_KEY, "") or "") + + +def start_generation( + db: Session, + *, + knowledge_base: Kind, + user: User, + head_commit: str, + changed_paths: Optional[Sequence[ChangedPath]] = None, + total_source_files: Optional[int] = None, + team_id: int = 0, + task_id: int = 0, + policy: Optional[RunModePolicy] = None, + now: Optional[datetime] = None, +) -> StartedGeneration: + """Begin a run, unless there is nothing to do or one is already going. + + Args: + db: Session. + knowledge_base: The code wiki. + user: Identity the run executes under. + head_commit: Repository HEAD the run would document. + changed_paths: Diff since the published commit, or ``None`` when unknown — + in which case a full rebuild is chosen rather than a guess. + total_source_files: Repository size, used by the change-ratio threshold. + team_id: Team the generation task belongs to. + task_id: Task driving the run, when one exists yet. + policy: Thresholds promoting an incremental run to a full one. + now: Reference time, for tests. + + Returns: + The started run, or a decision explaining why none was needed. + + Raises: + GenerationInFlight: If a live run already owns this wiki. + """ + reclaimed = reclaim_stale_generations(db, kind_id=knowledge_base.id, now=now) + if reclaimed: + logger.warning( + "[code_wiki] kb %s had %s abandoned run(s) before starting", + knowledge_base.id, + len(reclaimed), + ) + + # Locked before the in-flight query, because that query locks only the rows it + # returns. With no live run there are no rows, so two concurrent starts would + # both find nothing and both insert — two agents documenting one repository, two + # seeded versions, and whichever finishes last silently overwrites the other. + # The knowledge base row always exists, so locking it serialises starts per wiki + # without depending on gap-lock behaviour that the isolation level can turn off. + db.query(Kind).filter(Kind.id == knowledge_base.id).with_for_update().first() + + in_flight = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == knowledge_base.id, + WikiGeneration.status.in_( + [WikiGenerationStatus.PENDING, WikiGenerationStatus.RUNNING] + ), + ) + .with_for_update() + .first() + ) + if in_flight is not None: + raise GenerationInFlight( + f"generation {in_flight.id} is already running for this wiki" + ) + + last_commit = published_commit(db, knowledge_base) + since_full = _runs_since_the_last_full(db, knowledge_base.id, now=now) + decision = decide_run_mode( + head_commit=head_commit, + last_commit=last_commit or None, + changed_paths=changed_paths, + total_source_files=total_source_files, + incrementals_since_full=since_full[0], + days_since_full=since_full[1], + # Passed only when given: the callee defaults it, and forwarding ``None`` + # would replace that default with nothing. + **({"policy": policy} if policy is not None else {}), + ) + + if RunMode(decision.mode) == RunMode.SKIP: + logger.info( + "[code_wiki] kb %s needs no run: %s", knowledge_base.id, decision.reason + ) + return StartedGeneration(generation=None, decision=decision) + + generation = WikiGeneration( + project_id=0, + kind_id=knowledge_base.id, + user_id=user.id, + task_id=task_id, + team_id=team_id, + generation_type=( + WikiGenerationType.FULL + if RunMode(decision.mode) == RunMode.FULL + else WikiGenerationType.INCREMENTAL + ), + source_snapshot={SOURCE_COMMIT_KEY: head_commit}, + status=WikiGenerationStatus.RUNNING, + ext={"runModeReason": decision.reason}, + completed_at=datetime(1970, 1, 1), + ) + db.add(generation) + db.flush() + + seeded = 0 + if decision.seeds_from_published: + # Before the agent starts, so that it revises a complete snapshot rather than + # producing a partial one the projection would read as a mass deletion. + outcome = seed_from_published( + db, + target_generation_id=generation.id, + published_generation_id=published_generation_id(knowledge_base), + ) + seeded = outcome.copied_pages + + logger.info( + "[code_wiki] started %s generation %s for kb %s (%s), seeded %s pages", + decision.mode, + generation.id, + knowledge_base.id, + decision.reason, + seeded, + ) + return StartedGeneration( + generation=generation, decision=decision, seeded_pages=seeded + ) + + +def finish_generation( + db: Session, + *, + knowledge_base: Kind, + generation: WikiGeneration, + user: User, + effects: ProjectionSideEffects, + succeeded: bool, + error_message: str = "", + policy: Optional[PublishPolicy] = None, + now: Optional[datetime] = None, +) -> Optional[PublishResult]: + """Conclude a run and, if it succeeded, offer its version for publishing. + + A failed run deliberately leaves the published pointer alone. The next scheduled + run then still sees the repository as undocumented at its current commit and does + the work again, rather than skipping changes nobody has written up. + + Returns: + The publish outcome, or ``None`` when the run failed. + """ + finished = (now or datetime.now(timezone.utc)).replace(tzinfo=None) + + if not succeeded: + generation.status = WikiGenerationStatus.FAILED + generation.completed_at = finished + ext = dict(generation.ext or {}) + if error_message: + ext["errorMessage"] = error_message + generation.ext = ext + db.commit() + logger.warning( + "[code_wiki] generation %s failed for kb %s: %s", + generation.id, + knowledge_base.id, + error_message or "no reason given", + ) + return None + + generation.status = WikiGenerationStatus.COMPLETED + generation.completed_at = finished + db.flush() + + return publish_generation( + db, + knowledge_base=knowledge_base, + generation=generation, + user_id=user.id, + effects=effects, + policy=policy, + ) + + +def _runs_since_the_last_full( + db: Session, kind_id: int, now: Optional[datetime] = None +) -> tuple[int, Optional[float]]: + """How much has accumulated since this wiki was last rebuilt from scratch. + + Feeds the periodic-rebuild thresholds, which had the branches for this from the + start and were never given the inputs — so they defaulted to "no drift yet" and + the wiki was never rebuilt no matter how long it ran on increments. + + Returns: + The number of completed incremental runs, and the age in days of the last + full one — ``None`` when there has never been a full run to age. + """ + latest_full = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == kind_id, + WikiGeneration.generation_type == WikiGenerationType.FULL, + WikiGeneration.status == WikiGenerationStatus.COMPLETED, + ) + .order_by(WikiGeneration.completed_at.desc()) + .first() + ) + if latest_full is None: + return 0, None + + incrementals = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == kind_id, + WikiGeneration.generation_type == WikiGenerationType.INCREMENTAL, + WikiGeneration.status == WikiGenerationStatus.COMPLETED, + WikiGeneration.completed_at > latest_full.completed_at, + ) + .count() + ) + reference = (now or datetime.now(timezone.utc)).replace(tzinfo=None) + age_days = (reference - latest_full.completed_at).total_seconds() / 86400 + return incrementals, max(0.0, age_days) + + +def version_page_count(db: Session, generation_id: int) -> int: + """How many pages a version holds, for status displays.""" + return len(read_version_pages(db, generation_id)) diff --git a/backend/app/services/knowledge/code_wiki/mermaid_check.py b/backend/app/services/knowledge/code_wiki/mermaid_check.py new file mode 100644 index 0000000000..1350ad8d13 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/mermaid_check.py @@ -0,0 +1,243 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Structural checks on Mermaid diagrams in generated Markdown. + +Diagrams are rendered in the browser, so a malformed one spoils a figure rather than a +page. Withholding an otherwise good page over it would cost more than it saves, so +these findings are returned to the writing agent as warnings and never block +publication. + +The checks are structural on purpose: they catch the mistakes a model actually makes — +a misspelled diagram type, an unclosed fence, unbalanced brackets — without running +Mermaid itself, which is a JavaScript parser this service cannot host. What they +cannot do is certify that a diagram renders; they only report what is definitely +wrong. The warning contract is stable, so swapping in a real parser later would not +change the callers. +""" + +import re +from dataclasses import dataclass +from typing import List, Sequence + +FENCE_PATTERN = re.compile(r"^(\s*)(`{3,}|~{3,})\s*([A-Za-z0-9_+-]*)\s*$") + +# Diagram types Mermaid recognises. A block starting with anything else will not +# render, and a misspelling here is the most common failure in generated diagrams. +# Tracks the Mermaid the frontend actually resolves to (11.15.x via ^11.4.0). A type +# missing here is reported as a broken diagram and sent back to the agent to "fix", +# which wastes a round on a diagram that renders perfectly well — so this list has to +# be widened whenever the pinned version gains a declaration. +# +# Newer types carry a "-beta" suffix that Mermaid drops as they stabilise, and both +# spellings are listed rather than guessed between: the cost of an extra entry is +# nothing, and the cost of a missing one is a wasted round trip through the model. +KNOWN_DIAGRAM_TYPES: frozenset[str] = frozenset( + { + "architecture-beta", + "block-beta", + "c4component", + "c4container", + "c4context", + "c4deployment", + "c4dynamic", + "classdiagram", + "classdiagram-v2", + "erdiagram", + "flowchart", + "flowchart-elk", + "gantt", + "gitgraph", + "graph", + "journey", + "kanban", + "mindmap", + "packet", + "packet-beta", + "pie", + "quadrantchart", + "radar", + "radar-beta", + "requirementdiagram", + "sankey-beta", + "sequencediagram", + "statediagram", + "statediagram-v2", + "timeline", + "treemap", + "treemap-beta", + "xychart-beta", + "zenuml", + } +) + +BRACKET_PAIRS = {"(": ")", "[": "]", "{": "}"} + +# Diagram types where brackets delimit nodes, so an unbalanced one is a real mistake. +# The check is limited to these because elsewhere brackets are not delimiters at all: +# an ER diagram writes cardinality as ``||--o{``, which is deliberately "unbalanced", +# and the text-heavy types put arbitrary prose in labels. Reporting those would send +# the agent off to fix diagrams that are already correct. +BRACKET_CHECKED_TYPES: frozenset[str] = frozenset( + { + "block-beta", + "classdiagram", + "flowchart", + "graph", + "mindmap", + "statediagram", + "statediagram-v2", + } +) + + +@dataclass(frozen=True) +class MermaidWarning: + """One problem found in a diagram, addressed to the agent that wrote it.""" + + # 1-based line of the fence that opens the diagram, so the agent can find it. + line: int + message: str + + def __str__(self) -> str: + return f"line {self.line}: {self.message}" + + +def _diagram_type_of(body: Sequence[str]) -> str: + """First meaningful token of a diagram, lowercased; empty when there is none.""" + for line in _without_frontmatter(body): + stripped = line.strip() + if not stripped or stripped.startswith("%%"): + continue + # "flowchart TD", "stateDiagram-v2", "graph LR" — the type is the first token. + return re.split(r"[\s:]", stripped, maxsplit=1)[0].lower() + return "" + + +def _without_frontmatter(body: Sequence[str]) -> Sequence[str]: + """Drop a leading ``--- ... ---`` block. + + Mermaid allows YAML frontmatter before the diagram type, which several of the + types added for the pinned version use. Read as the diagram itself, the opening + ``---`` becomes the type, and a diagram that renders is reported as unknown. + """ + first = next((index for index, line in enumerate(body) if line.strip()), None) + if first is None or body[first].strip() != "---": + return body + + for index in range(first + 1, len(body)): + if body[index].strip() == "---": + return body[index + 1 :] + # Unterminated: not frontmatter, whatever else it is. + return body + + +def _unbalanced_bracket(body: Sequence[str]) -> str: + """Report the first unbalanced bracket, ignoring anything inside quotes.""" + stack: List[str] = [] + for line in body: + in_quote = False + for char in line: + if char == '"': + in_quote = not in_quote + continue + if in_quote: + continue + if char in BRACKET_PAIRS: + stack.append(char) + elif char in BRACKET_PAIRS.values(): + if not stack: + return f"unexpected closing '{char}'" + if BRACKET_PAIRS[stack.pop()] != char: + return f"mismatched closing '{char}'" + if in_quote: + return "unclosed quote" + if stack: + return f"unclosed '{stack[-1]}'" + return "" + + +def check_mermaid_blocks(markdown: str) -> List[MermaidWarning]: + """Find structural problems in the Mermaid diagrams of a Markdown document. + + Returns an empty list when nothing is definitely wrong. Nested fences inside other + fenced blocks are skipped, so a Mermaid example quoted inside a code sample is not + mistaken for a diagram. + """ + warnings: List[MermaidWarning] = [] + lines = markdown.splitlines() + + index = 0 + while index < len(lines): + match = FENCE_PATTERN.match(lines[index]) + if not match: + index += 1 + continue + + _, marker, info = match.groups() + opened_at = index + 1 + is_mermaid = info.lower() == "mermaid" + + body: List[str] = [] + index += 1 + closed = False + while index < len(lines): + closing = FENCE_PATTERN.match(lines[index]) + # A closing fence must use the same character and be at least as long as + # the opening one. Length matters because wrapping an example in a longer + # fence is how a diagram gets quoted rather than declared: without this, + # the inner ``` ends the outer ````, and the rest of the example is read + # as live markdown. + if ( + closing + and closing.group(2)[0] == marker[0] + and len(closing.group(2)) >= len(marker) + and not closing.group(3) + ): + closed = True + index += 1 + break + body.append(lines[index]) + index += 1 + + if not is_mermaid: + # A non-Mermaid fence is only consumed so its contents cannot be mistaken + # for a diagram; problems inside it are not ours to report. + continue + + if not closed: + warnings.append(MermaidWarning(opened_at, "diagram fence is never closed")) + continue + + diagram_type = _diagram_type_of(body) + if not diagram_type: + warnings.append(MermaidWarning(opened_at, "diagram is empty")) + continue + if diagram_type not in KNOWN_DIAGRAM_TYPES: + warnings.append( + MermaidWarning( + opened_at, + f"'{diagram_type}' is not a Mermaid diagram type, so this " + "diagram will not render", + ) + ) + continue + + if diagram_type in BRACKET_CHECKED_TYPES: + problem = _unbalanced_bracket(body) + if problem: + warnings.append(MermaidWarning(opened_at, problem)) + + return warnings + + +def describe_warnings(warnings: Sequence[MermaidWarning]) -> str: + """Render warnings as an instruction the writing agent can act on.""" + if not warnings: + return "" + listed = "\n".join(f"- {warning}" for warning in warnings) + return ( + "The Mermaid diagrams below will not render. Fix them and write the page " + f"again:\n{listed}" + ) diff --git a/backend/app/services/knowledge/code_wiki/navigation.py b/backend/app/services/knowledge/code_wiki/navigation.py new file mode 100644 index 0000000000..ef2c566065 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/navigation.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Turning published pages into the tree a reader navigates. + +The hierarchy is in the paths and the order is on the knowledge base, so assembling +them belongs here rather than in the client: the client would have to fetch a +paginated document list and an order array and keep the two consistent while it +merged them, which is a second place for the tree to be wrong. + +A section that holds pages but has no page of its own becomes a node with no +document. The publish gate reports that as a warning rather than refusing the +version, so the reader has to render it — as a heading that cannot be opened. +""" + +import logging +from dataclasses import dataclass, field + +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import KnowledgeDocument +from app.services.knowledge.code_wiki.page_path import collation_key +from app.services.knowledge.code_wiki.projection_plan import PAGE_PATH_KEY +from app.services.knowledge.code_wiki.publisher import PAGE_ORDER_KEY +from app.services.knowledge.content_scope import generated_wiki_pages + +logger = logging.getLogger(__name__) + + +@dataclass +class PageNode: + """One entry in the navigation.""" + + path: str + title: str + document_id: int = 0 + children: list["PageNode"] = field(default_factory=list) + + @property + def has_content(self) -> bool: + return self.document_id != 0 + + +def page_tree(db: Session, knowledge_base: Kind) -> list[PageNode]: + """The published pages of ``knowledge_base``, nested and ordered.""" + documents = generated_wiki_pages( + db.query(KnowledgeDocument).filter( + KnowledgeDocument.kind_id == knowledge_base.id + ) + ).all() + + by_key: dict[str, PageNode] = {} + for document in documents: + path = (document.source_config or {}).get(PAGE_PATH_KEY) + if not path: + logger.warning( + "[code_wiki] document %s has no page path; not navigable", document.id + ) + continue + by_key[collation_key(path)] = PageNode( + path=path, title=document.name, document_id=document.id + ) + + _add_missing_sections(by_key) + return _nest(by_key, _declared_order(knowledge_base)) + + +def _add_missing_sections(by_key: dict[str, PageNode]) -> None: + """Invent a node for a section that holds pages but is not one itself. + + Without it the pages under that section would have nowhere to hang and would + surface at the top level, which reads as though they were unrelated. + """ + for key in list(by_key): + path = by_key[key].path + while "/" in path: + path = path.rsplit("/", 1)[0] + section_key = collation_key(path) + if section_key in by_key: + break + by_key[section_key] = PageNode( + path=path, title=path.rsplit("/", 1)[-1], document_id=0 + ) + + +def _declared_order(knowledge_base: Kind) -> dict[str, int]: + spec = (knowledge_base.json or {}).get("spec", {}) + declared = spec.get(PAGE_ORDER_KEY) or [] + return {collation_key(str(path)): index for index, path in enumerate(declared)} + + +def _nest(by_key: dict[str, PageNode], order: dict[str, int]) -> list[PageNode]: + """Attach each node to its parent, and sort every level the same way.""" + roots: list[PageNode] = [] + for key, node in by_key.items(): + parent_key = ( + collation_key(node.path.rsplit("/", 1)[0]) if "/" in node.path else None + ) + parent = by_key.get(parent_key) if parent_key else None + if parent is None: + roots.append(node) + else: + parent.children.append(node) + + def rank(node: PageNode) -> tuple[int, str]: + # Anything the agent did not rank sorts after what it did, then by path so + # the result is at least stable rather than dependent on row order. + key = collation_key(node.path) + return (order.get(key, len(order)), key) + + def sort(nodes: list[PageNode]) -> list[PageNode]: + nodes.sort(key=rank) + for node in nodes: + sort(node.children) + return nodes + + return sort(roots) diff --git a/backend/app/services/knowledge/code_wiki/page_path.py b/backend/app/services/knowledge/code_wiki/page_path.py new file mode 100644 index 0000000000..8433095809 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/page_path.py @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""The stable identity of a wiki page. + +A page is identified by its path, not by its title. The distinction matters because +the projection matches a published version against the knowledge base by path: a page +whose path is unchanged keeps its ``KnowledgeDocument`` row, and with it the document +id that the RAG index is keyed on and that stored citations point at. Matching on +title instead — as the original write API did — would turn every reworded heading into +a delete plus an insert, re-embedding the page and breaking references to it. + +Paths are also what the projection turns into folders, so they are validated here +against the same limits the folder tree enforces. Rejecting a bad path when the agent +writes it keeps the failure next to its cause; discovering it at publish time would +fail a whole version for one malformed entry. +""" + +import re +import unicodedata +from typing import Iterable + +from app.services.knowledge.folder_policy import MAX_FOLDER_DEPTH + +# Each segment becomes a folder or document name, both String(255). +MAX_SEGMENT_LENGTH = 255 + +# Bound on the whole path. Generous next to the depth and segment limits; it exists to +# stop pathological input rather than to constrain real page layouts. +MAX_PATH_LENGTH = 500 + +# Directory segments a path may have. The leaf is the document, so it does not count: +# ``a/b/c/d/page`` places a document in the deepest folder the tree allows. +MAX_DIRECTORY_DEPTH = MAX_FOLDER_DEPTH + +# Extension the projection appends when it materialises the page. Accepted on input and +# stripped, because a model writing markdown will naturally include it. +MARKDOWN_SUFFIX = ".md" + +_CONTROL_CHARACTERS = re.compile(r"[\x00-\x1f\x7f]") + +# Reserved by common filesystems and by our own path grammar. +_FORBIDDEN_IN_SEGMENT = set('\\:*?"<>|') + + +class InvalidPagePath(ValueError): + """Raised when a page path cannot be used as an identity.""" + + +def normalize_page_path(raw: str) -> str: + """Return the canonical form of ``raw``, or raise :class:`InvalidPagePath`. + + Normalization is deliberately limited to differences that carry no meaning — + surrounding whitespace, repeated separators, a trailing ``.md``. Anything + ambiguous is rejected rather than guessed at, so that a malformed path surfaces as + an error the agent can correct instead of silently becoming a different page. + """ + if not isinstance(raw, str): + raise InvalidPagePath("Page path must be a string") + + path = unicodedata.normalize("NFC", raw).strip() + if not path: + raise InvalidPagePath("Page path must not be empty") + + if _CONTROL_CHARACTERS.search(path): + raise InvalidPagePath("Page path must not contain control characters") + + if path.startswith("/"): + raise InvalidPagePath(f"Page path must be relative, got '{raw}'") + + if "\\" in path: + raise InvalidPagePath( + f"Page path must use '/' as its separator, got '{raw}'", + ) + + segments = [segment.strip() for segment in path.split("/")] + segments = [segment for segment in segments if segment] + if not segments: + raise InvalidPagePath(f"Page path has no usable segments: '{raw}'") + + # Strip the suffix before validating, so that every segment checked below is the + # one that ends up in the result. Validating first and stripping afterwards lets + # a leaf like "...md" pass the relative-segment check and then become ".." — the + # very thing the check exists to reject. + leaf = segments[-1] + if leaf.lower().endswith(MARKDOWN_SUFFIX): + leaf = leaf[: -len(MARKDOWN_SUFFIX)].strip() + if not leaf: + raise InvalidPagePath(f"Page path has an empty file name: '{raw}'") + segments[-1] = leaf + + for segment in segments: + if segment in (".", ".."): + raise InvalidPagePath( + f"Page path must not contain relative segments, got '{raw}'", + ) + if _FORBIDDEN_IN_SEGMENT & set(segment): + raise InvalidPagePath( + f"Page path segment '{segment}' contains a reserved character", + ) + if len(segment) > MAX_SEGMENT_LENGTH: + raise InvalidPagePath( + f"Page path segment exceeds {MAX_SEGMENT_LENGTH} characters", + ) + + if len(segments) - 1 > MAX_DIRECTORY_DEPTH: + raise InvalidPagePath( + f"Page path nests {len(segments) - 1} folders deep, over the " + f"{MAX_DIRECTORY_DEPTH} the folder tree allows: '{raw}'", + ) + + normalized = "/".join(segments) + if len(normalized) > MAX_PATH_LENGTH: + raise InvalidPagePath(f"Page path exceeds {MAX_PATH_LENGTH} characters") + + return normalized + + +def split_page_path(path: str) -> tuple[tuple[str, ...], str]: + """Split a normalized path into its folder segments and document name.""" + segments = path.split("/") + return tuple(segments[:-1]), segments[-1] + + +def collation_key(path: str) -> str: + """Return the key under which two paths count as the same page. + + Comparison is case-insensitive because the knowledge tables collate that way + (``utf8mb4_unicode_ci``): ``Architecture`` and ``architecture`` would resolve to + one folder in the database, so treating them as distinct pages in a version would + produce a collision the projection could not honour. + """ + return path.casefold() + + +def assert_unique_within_version(paths: Iterable[str]) -> None: + """Raise if any two paths in one version identify the same page. + + Args: + paths: Normalized paths making up a single version. + + Raises: + InvalidPagePath: If two entries collide, naming both originals. + """ + seen: dict[str, str] = {} + for path in paths: + key = collation_key(path) + if key in seen and seen[key] != path: + raise InvalidPagePath( + f"Page paths '{seen[key]}' and '{path}' differ only by case and " + "would resolve to the same page", + ) + if key in seen: + raise InvalidPagePath(f"Page path '{path}' appears more than once") + seen[key] = path diff --git a/backend/app/services/knowledge/code_wiki/projection.py b/backend/app/services/knowledge/code_wiki/projection.py new file mode 100644 index 0000000000..7783631e4c --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/projection.py @@ -0,0 +1,380 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Applying a projection plan to the knowledge base. + +The ordering here is the whole point, and it is not negotiable: + + before the transaction write the new attachments (row and object) + inside the transaction repoint, insert, delete rows, drop empty folders + after the transaction delete superseded attachments, clear RAG, reindex + +Attachment bytes live in object storage in production, so they are outside the +transaction. Writing them first costs an orphaned object when a publish fails, which a +sweep collects. Deleting them first would destroy live content that a rollback then +cannot bring back — the same act, reordered, is the difference between litter and data +loss. + +The same reasoning forbids overwriting an attachment in place for an updated page: the +overwrite happens before the commit, so a failed transaction would leave the document +pointing at content that was never published and the previous content gone. Updates +therefore write a new attachment and repoint. + +``KnowledgeService.delete_document`` is deliberately not used, because it commits +internally — calling it per page would turn one transaction into many. Its cascade +still has to happen, though: deleting a document row without clearing the RAG index +leaves the retrieval layer answering from a page that no longer exists and citing an +id that resolves to nothing. Since the vector store cannot join the transaction, those +deletions are recorded and retried instead. +""" + +import logging +from dataclasses import dataclass, field +from typing import Callable, Optional, Protocol, Sequence + +from sqlalchemy.orm import Session + +from app.models.knowledge import ContentOrigin, KnowledgeDocument +from app.services.knowledge.code_wiki.page_path import split_page_path +from app.services.knowledge.code_wiki.projection_plan import ( + CONTENT_HASH_KEY, + PAGE_PATH_KEY, + PageSource, + ProjectionPlan, +) + +logger = logging.getLogger(__name__) + +# Key under which doc_refs awaiting RAG deletion are parked on the knowledge base. +# The vector store is external, so its cleanup cannot be part of the transaction; a +# failure there must survive a restart rather than vanish into a log line. +PENDING_INDEX_CLEANUP_KEY = "pendingIndexCleanup" + +DOCUMENT_EXTENSION = "md" + + +class AttachmentWriter(Protocol): + """Creates attachment content. Called only before the transaction commits.""" + + def __call__(self, *, filename: str, content: str) -> int: + """Return the id of a newly created attachment.""" + + +@dataclass +class ProjectionSideEffects: + """Everything the projection does outside its own transaction. + + Injected rather than imported so the ordering rules above can be asserted on + directly: a test can record the sequence of calls and prove that no attachment is + deleted before the commit. + """ + + write_attachment: AttachmentWriter + delete_attachment: Callable[[int], None] + delete_rag_document: Callable[[int], None] + enqueue_reindex: Callable[[int], None] + + +@dataclass(frozen=True) +class ProjectionOutcome: + """What the projection did.""" + + plan: ProjectionPlan + created_document_ids: tuple[int, ...] = () + updated_document_ids: tuple[int, ...] = () + deleted_document_ids: tuple[int, ...] = () + # doc_refs whose RAG deletion did not succeed and must be retried. + unfinished_index_cleanup: tuple[str, ...] = field(default=()) + + +def _folder_resolver(db: Session, kind_id: int): + """Return a function creating (or finding) the folder chain for a page path.""" + from app.models.knowledge import KnowledgeFolder + + # Loaded once. A projection touches a handful of folders and asks about them + # repeatedly, so one query up front replaces a query per distinct folder. + cache: dict[tuple[int, str], int] = { + (folder.parent_id, folder.name.casefold()): folder.id + for folder in db.query(KnowledgeFolder) + .filter(KnowledgeFolder.kind_id == kind_id) + .all() + } + + def resolve(segments: Sequence[str]) -> int: + parent_id = 0 + for segment in segments: + key = (parent_id, segment.casefold()) + if key not in cache: + created = KnowledgeFolder( + kind_id=kind_id, + parent_id=parent_id, + name=segment, + origin=ContentOrigin.GENERATED.value, + ) + db.add(created) + db.flush() + cache[key] = created.id + parent_id = cache[key] + return parent_id + + return resolve + + +# What ``knowledge_documents.name`` will hold. The column is 255 wide while a page +# path may be 500, which is one reason the path cannot simply be the name. +MAX_DOCUMENT_NAME = 255 + + +def _display_name(source: PageSource) -> str: + """What a reader sees this page called. + + The agent's title, which was previously discarded — documents were named after + the path's last segment, so "Backend Architecture" at ``architecture/backend`` + became a document called ``backend``, and two pages at ``architecture/backend`` + and ``services/backend`` were indistinguishable in any flat listing. + + Falls back to the leaf when a title is missing, because a page has to be called + something, and truncates rather than letting the insert fail on a long one. + """ + _, leaf = split_page_path(source.path) + title = (source.title or "").strip() or leaf + return title[:MAX_DOCUMENT_NAME] + + +def _stamp(document: KnowledgeDocument, source: PageSource) -> None: + """Record the page identity and fingerprint the next plan compares against.""" + config = dict(document.source_config or {}) + config[PAGE_PATH_KEY] = source.path + config[CONTENT_HASH_KEY] = source.fingerprint + document.source_config = config + + +def apply_projection_plan( + db: Session, + *, + kind_id: int, + user_id: int, + plan: ProjectionPlan, + effects: ProjectionSideEffects, +) -> ProjectionOutcome: + """Write a plan into the knowledge base. + + The caller owns the transaction and must commit; this function flushes but never + commits, so that a failure anywhere leaves the knowledge base exactly as it was. + + Args: + db: Session holding the projection transaction. + kind_id: Knowledge base being projected into. + user_id: Owner recorded on documents the projection creates. + plan: What to do, from ``compute_projection_plan``. + effects: Side effects outside the transaction. + + Returns: + What was done, including any RAG cleanup left to retry. + """ + resolve_folder = _folder_resolver(db, kind_id) + + # --- before the transaction commits: content into storage ------------------- + # Written first on purpose. An orphaned object is litter; content deleted before + # a rollback is gone. + new_attachments: dict[str, int] = {} + for source in (*plan.adds, *(update.source for update in plan.updates)): + _, leaf = split_page_path(source.path) + new_attachments[source.path] = effects.write_attachment( + filename=f"{leaf}.{DOCUMENT_EXTENSION}", + content=source.content, + ) + + def add_document(source: PageSource) -> int: + """Create the row for a page that is not in the knowledge base.""" + folders, _ = split_page_path(source.path) + document = KnowledgeDocument( + kind_id=kind_id, + attachment_id=new_attachments[source.path], + name=_display_name(source), + file_extension=DOCUMENT_EXTENSION, + file_size=len(source.content.encode("utf-8")), + user_id=user_id, + folder_id=resolve_folder(folders), + origin=ContentOrigin.GENERATED.value, + # Left inactive; the existing indexing state machine turns a document on + # once its index succeeds, which is how every other document behaves. + is_active=False, + ) + _stamp(document, source) + db.add(document) + db.flush() + return document.id + + # --- inside the transaction: rows only -------------------------------------- + created: list[int] = [] + for source in plan.adds: + created.append(add_document(source)) + + superseded_attachments: list[int] = [] + updated: list[int] = [] + for update in plan.updates: + document = db.get(KnowledgeDocument, update.existing.document_id) + if document is None: + # The row went away between planning and here. Skipping would leave the + # published version missing a page the plan accounted for, and strand the + # attachment already written for it, so it is added instead. + logger.warning( + "[code_wiki] document %s vanished before projection; adding it back", + update.existing.document_id, + ) + created.append(add_document(update.source)) + continue + if document.attachment_id: + superseded_attachments.append(document.attachment_id) + document.attachment_id = new_attachments[update.source.path] + document.file_size = len(update.source.content.encode("utf-8")) + # Follows the title, which is why the fingerprint covers it: a reworded + # heading has to reach the document's name, not just its content. + document.name = _display_name(update.source) + _stamp(document, update.source) + updated.append(document.id) + + removed_refs: list[str] = [] + deleted: list[int] = [] + for page in plan.deletes: + document = db.get(KnowledgeDocument, page.document_id) + if document is None: + continue + if document.attachment_id: + superseded_attachments.append(document.attachment_id) + converted = document.converted_attachment_id + if converted: + superseded_attachments.append(converted) + # The RAG index keys documents by their id as a string; capture it before the + # row is gone, because after the commit there is nothing left to derive it from. + removed_refs.append(str(document.id)) + deleted.append(document.id) + db.delete(document) + + db.flush() + _remove_emptied_generated_folders(db, kind_id) + db.flush() + + return ProjectionOutcome( + plan=plan, + created_document_ids=tuple(created), + updated_document_ids=tuple(updated), + deleted_document_ids=tuple(deleted), + unfinished_index_cleanup=tuple(removed_refs), + ) + + +def finish_projection( + outcome: ProjectionOutcome, + *, + superseded_attachment_ids: Sequence[int], + effects: ProjectionSideEffects, +) -> tuple[str, ...]: + """Run the work that must wait until the transaction has committed. + + Every step here is retriable and none of it can undo the publish, so a failure is + reported rather than raised: the pages are already correct, and refusing to + acknowledge that would only cause the whole version to be produced again. + + Returns: + doc_refs whose RAG deletion failed and must be retried. + """ + for attachment_id in superseded_attachment_ids: + try: + effects.delete_attachment(attachment_id) + except Exception as exc: # pragma: no cover - defensive + logger.warning( + "[code_wiki] superseded attachment %s not deleted: %s", + attachment_id, + exc, + ) + + unfinished: list[str] = [] + for doc_ref in outcome.unfinished_index_cleanup: + try: + effects.delete_rag_document(int(doc_ref)) + except Exception as exc: + logger.warning( + "[code_wiki] RAG cleanup for doc_ref %s failed, will retry: %s", + doc_ref, + exc, + ) + unfinished.append(doc_ref) + + for document_id in ( + *outcome.created_document_ids, + *outcome.updated_document_ids, + ): + try: + effects.enqueue_reindex(document_id) + except Exception as exc: # pragma: no cover - defensive + logger.warning( + "[code_wiki] reindex not enqueued for document %s: %s", + document_id, + exc, + ) + + return tuple(unfinished) + + +def _remove_emptied_generated_folders(db: Session, kind_id: int) -> None: + """Drop generated folders left with no documents and no children. + + Scoped to generated folders: a user folder that happens to be empty is theirs to + keep, and the projection has no business tidying it away. + """ + from app.models.knowledge import KnowledgeFolder + + folders = ( + db.query(KnowledgeFolder) + .filter( + KnowledgeFolder.kind_id == kind_id, + KnowledgeFolder.origin == ContentOrigin.GENERATED.value, + ) + .all() + ) + if not folders: + return + + used_folder_ids = { + row[0] + for row in db.query(KnowledgeDocument.folder_id) + .filter(KnowledgeDocument.kind_id == kind_id) + .distinct() + .all() + } + + # Walked deepest-first. Emptying a leaf can empty its parent, so a parent is only + # judged once its children have been — which is what the old repeat-until-stable + # loop was doing, one re-query per level. + children: dict[int, list[KnowledgeFolder]] = {} + for folder in folders: + children.setdefault(folder.parent_id, []).append(folder) + + doomed: set[int] = set() + + def survives(folder: KnowledgeFolder) -> bool: + # Every child is visited before the decision: a short circuit would leave the + # ones after the first survivor unexamined, and they may be empty themselves. + kept_child = any([survives(child) for child in children.get(folder.id, ())]) + if folder.id in used_folder_ids or kept_child: + return True + doomed.add(folder.id) + return False + + # Roots are the generated folders whose parent is not itself generated: the tree + # top (parent 0) and anything a user folder contains. Depth is capped at four, so + # the recursion cannot run away. + generated_ids = {folder.id for folder in folders} + for folder in folders: + if folder.parent_id not in generated_ids: + survives(folder) + + if not doomed: + return + for folder in folders: + if folder.id in doomed: + db.delete(folder) + db.flush() diff --git a/backend/app/services/knowledge/code_wiki/projection_plan.py b/backend/app/services/knowledge/code_wiki/projection_plan.py new file mode 100644 index 0000000000..5ed87bd0fa --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/projection_plan.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Working out what projecting a version would change. + +The plan is computed and returned before anything is written, so that the decision +and its execution are separable: it can be logged, asserted on in tests, and run as a +dry run. "This publish removes 37 pages" is the kind of fact that should be visible +and refusable, which it cannot be if the deletions only exist as they happen. + +Matching is by page path, never by title, because the path is what keeps a page's +``KnowledgeDocument`` id — and with it its RAG index entry and any stored citation — +stable across regenerations. + +Both sides are complete snapshots: a version is seeded so that it holds every page, +not only the ones a run revised. That is what makes a deletion a plain set difference +rather than a guess about whether the run's output was authoritative. +""" + +import hashlib +from dataclasses import dataclass +from typing import Iterable, Mapping + +from app.services.knowledge.code_wiki.page_path import collation_key + +# Keys under which the projection records a page's identity and content fingerprint on +# the document it owns. They live in ``source_config`` (an existing JSON column) rather +# than in new columns: the path is derivable from the folder tree and the hash from the +# attachment, so neither is worth a migration. +PAGE_PATH_KEY = "wiki_page_path" +CONTENT_HASH_KEY = "wiki_content_hash" + + +def content_fingerprint(title: str, content: str) -> str: + """Return the fingerprint used to decide whether a page needs rewriting. + + Covers the title as well as the body, because the title is now what the document + is named. Fingerprinting the body alone would classify a page whose heading was + reworded as unchanged, and the rename would be silently dropped. + + The cost is that a title-only edit rewrites the attachment, which is wasted work + — but a title almost always appears in the body as its heading, so the two change + together in practice. + """ + return hashlib.sha256(f"{title}\n{content}".encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class PageSource: + """A page as the version holds it.""" + + path: str + title: str + content: str + + @property + def fingerprint(self) -> str: + return content_fingerprint(self.title, self.content) + + +@dataclass(frozen=True) +class ProjectedPage: + """A page as the knowledge base currently holds it.""" + + document_id: int + path: str + content_hash: str + + +@dataclass(frozen=True) +class PageUpdate: + """A page whose content changed.""" + + existing: ProjectedPage + source: PageSource + + +@dataclass(frozen=True) +class ProjectionPlan: + """What projecting a version would do, before any of it is done.""" + + adds: tuple[PageSource, ...] = () + updates: tuple[PageUpdate, ...] = () + skips: tuple[str, ...] = () + deletes: tuple[ProjectedPage, ...] = () + + @property + def is_empty(self) -> bool: + """Whether the knowledge base already matches the version.""" + return not (self.adds or self.updates or self.deletes) + + @property + def touched_pages(self) -> int: + return len(self.adds) + len(self.updates) + len(self.deletes) + + def describe(self) -> str: + """One line for logs and for showing a publish decision to a person.""" + return ( + f"{len(self.adds)} added, {len(self.updates)} updated, " + f"{len(self.deletes)} removed, {len(self.skips)} unchanged" + ) + + +def compute_projection_plan( + desired: Iterable[PageSource], + existing: Iterable[ProjectedPage], +) -> ProjectionPlan: + """Compare a version against the knowledge base. + + Args: + desired: Every page in the version being published. + existing: Every generated wiki page currently in the knowledge base. Callers + must scope this to content the projection owns; user content and code + targets appear in neither snapshot and would otherwise be read as orphans + and deleted. + + Returns: + The plan. Pages present on both sides with an equal fingerprint are skipped + entirely — no attachment written, no reindex, no row touched — which is where + nearly all of an incremental run's savings come from. + """ + existing_by_key: Mapping[str, ProjectedPage] = { + collation_key(page.path): page for page in existing + } + + adds: list[PageSource] = [] + updates: list[PageUpdate] = [] + skips: list[str] = [] + seen: set[str] = set() + + for source in desired: + key = collation_key(source.path) + seen.add(key) + current = existing_by_key.get(key) + if current is None: + adds.append(source) + elif current.content_hash == source.fingerprint: + skips.append(source.path) + else: + updates.append(PageUpdate(existing=current, source=source)) + + deletes = [page for key, page in existing_by_key.items() if key not in seen] + + return ProjectionPlan( + adds=tuple(adds), + updates=tuple(updates), + skips=tuple(skips), + deletes=tuple(deletes), + ) diff --git a/backend/app/services/knowledge/code_wiki/prompts.py b/backend/app/services/knowledge/code_wiki/prompts.py new file mode 100644 index 0000000000..51b40fce04 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/prompts.py @@ -0,0 +1,268 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Instructions for the agent that writes a code wiki. + +The previous prompt asked the model to "analyse the repository structure and generate +documentation", which is roughly the whole problem restated. The wiki it produced was +the generic kind that tells a reader nothing they could not get faster by opening the +code — which is what this refactor set out to fix, and no amount of storage work +fixes it. + +So the instructions here are constraints rather than encouragement. Each section +forbids a specific failure that generated wikis actually exhibit: + +- **Grounding** — inventing modules and APIs that sound plausible. +- **Discovery** — reading the whole repository, then running out of budget before + writing anything useful. +- **Planning** — writing pages one at a time so that nothing links to anything. +- **Git** — describing what the code is, when what a reader cannot recover by + reading it is *why* it became that. +- **Selection** — padding pages with restatements of the obvious. + +The write contract is stated as strictly as the content rules, because the projection +depends on it: a page's path is its identity, and a page that arrives without one, or +under a path that shifts between runs, is republished as a delete plus an insert. +""" + +from dataclasses import dataclass +from typing import Optional, Sequence + +# Coverage a wiki is expected to provide. Adapted from the openwiki repository guide, +# whose set has been through real use. +REQUIRED_COVERAGE: tuple[str, ...] = ( + "a concise quickstart: how to run, test and debug this repository", + "an architecture overview: the pieces and how they fit", + "a source map: which directory holds what, and where to start reading", + "key workflows: what happens end to end for the operations that matter", + "domain concepts: the vocabulary this codebase assumes you know", + "operations notes: how it is deployed, configured and recovered", + "testing guidance: what is covered, how to run it, what to add", + "integration points: what it talks to, and what talks to it", +) + +_GROUNDING = """\ +## Grounding + +- Do not invent files, modules, APIs, configuration keys or behaviour. Every claim of + substance must come from a file you have opened or from git history you have read. +- Where the evidence is thin, say so plainly in the page. A sentence admitting that a + workflow is undocumented is worth more than a confident paragraph that is wrong, + because a reader can act on the first and is misled by the second. +- Do not describe intent you cannot support. "This is designed for X" needs a comment, + a commit message or a document behind it.\ +""" + +_DISCOVERY = """\ +## Discovery + +- Do not read every file. Budget spent reading is budget not spent writing. +- Start from the repository tree, dependency manifests, entry points, routing and + schema files, then read representatively within each area you intend to document. +- Prefer search over reading: locate with grep-like tools, then open only the regions + that matter. Never glob the entire tree. +- Skip vendored code, generated output, lockfiles and build artefacts entirely.\ +""" + +_PLANNING = """\ +## Planning + +- Before writing any page, decide the full set of pages and their paths, and record + for each one the evidence it will rest on. +- Decide the links between pages at the same time, as `concept -> relationship -> + concept`. Cross-links designed after the fact do not get written, and a wiki whose + pages do not reference each other is a pile of documents. +- Prefer the smallest set of pages that explains the repository. A first pass that is + accurate and navigable beats a broad one that is thin everywhere.\ +""" + +_GIT_HISTORY = """\ +## Git history + +- Use history to explain *why* the code is the way it is. That is the one thing a + reader cannot recover by reading the code itself, and it is what makes a wiki worth + maintaining at all. +- Look at how the important workflows and entry points arrived at their current shape. + A constraint that was added deliberately is worth documenting; one that is + incidental is worth not documenting.\ +""" + +_SELECTION = """\ +## What to keep out + +- Put every sentence through one test: **would this change what someone does here?** + If not, delete it. +- Record only what cannot be recovered by reading the code. Do not restate the + obvious structure, and do not narrate the happy path of a function that reads + clearly. +- Draw only conclusions the evidence supports. Where there is none, say there is + none. Do not fill the gap.\ +""" + + +def _write_contract(deletion_allowed: bool) -> str: + """Rules for handing pages back, which the projection depends on.""" + deletion = ( + """\ +- To remove a page, declare its removal explicitly by path. Simply not writing a page + does **not** remove it in this mode, because your version begins as a copy of the + published one.\ +""" + if deletion_allowed + else """\ +- Your version begins empty. A page you do not write is not in the wiki, so write + every page the wiki should contain, including the ones you did not change.\ +""" + ) + + return f"""\ +## How to hand back pages + +- Every page has a **path**, which is its identity: `architecture/backend`, + `modules/indexing`. Lowercase, `/`-separated, no file extension. +- **Keep a page's path stable across runs.** The path is what lets an unchanged page + keep its place, its links and its search index. Changing it republishes the page as + a deletion plus an insertion, so reword titles freely and move paths rarely. +- Paths nest at most 4 folders deep, and two paths may not differ only by case. +- **A section that holds pages needs a page of its own.** If you write + `architecture/backend`, write `architecture` too — the navigation is built from + these paths, and a section with no page becomes a heading a reader cannot open. + Give it the overview of what its pages cover. +- The order you list in `--structure-order` when you finish is the order readers see + the pages in. Put the overview first and arrange sections so the wiki reads front + to back; anything you leave out of that list is shown after everything you ranked. +- Write each page's **complete content** every time. There is no patch format; what + you send replaces the page. +- Write `index` as the overview page: a short list of links, each with one line + saying what the page covers and when to read it, followed by the sections. +{deletion}\ +""" + + +@dataclass(frozen=True) +class WikiRunContext: + """What the agent is being asked to document.""" + + project_name: str + generation_id: int + head_commit: str = "" + language: str = "English" + # Incremental runs only. + previous_commit: str = "" + changed_paths: Sequence[str] = () + existing_pages: Sequence[str] = () + + +def build_full_prompt(context: WikiRunContext) -> str: + """Instructions for rebuilding a wiki from nothing.""" + coverage = "\n".join(f"- {item}" for item in REQUIRED_COVERAGE) + return f"""\ +You are documenting the repository **{context.project_name}** for engineers who have +to work in it, and for agents that will answer questions about it. + +Write for practical navigation. A page earns its place by helping someone find the +part of the system they need and understand what they must know before changing it. + +## Coverage + +Aim to cover, in as few pages as does the job: +{coverage} + +{_GROUNDING} + +{_DISCOVERY} + +{_PLANNING} + +{_GIT_HISTORY} + +{_SELECTION} + +{_write_contract(deletion_allowed=False)} + +## This run + +- Generation: `{context.generation_id}` +- Commit: `{context.head_commit or "current HEAD"}` +- Language: {context.language} +- This is a **full rebuild**: the wiki is being written from scratch.\ +""" + + +def build_incremental_prompt(context: WikiRunContext) -> str: + """Instructions for revising a wiki after a set of changes.""" + changed = ( + "\n".join(f"- {path}" for path in context.changed_paths) + or "- (no file list was available)" + ) + existing = ( + "\n".join(f"- {path}" for path in context.existing_pages) + or "- (the wiki is currently empty)" + ) + return f"""\ +You are updating the wiki for **{context.project_name}** to account for recent +changes. The wiki already exists and is mostly correct; your job is to bring the +parts the changes affect back in line, not to rewrite it. + +## What changed + +Between `{context.previous_commit or "the last documented commit"}` and +`{context.head_commit or "current HEAD"}`: +{changed} + +## Pages that exist now + +{existing} + +## What to do + +- Revise only the pages the changes actually affect. Leave the rest untouched — an + unchanged page costs nothing to keep and is expensive to rewrite. +- **Read a page before you revise it.** Your version already holds every published + page, and what you send replaces the whole page, so revising one without reading it + first silently discards whatever it said: + `node wiki_submit.js read --generation-id --path ` + It exits with no output when the page does not exist yet, which means it is new. +- Refresh the `index` overview if the set of pages changed. +- If a change makes a page's subject disappear, remove that page explicitly. +- Add pages only where the changes introduced something the wiki has no place for. + +{_GROUNDING} + +{_DISCOVERY} + +{_GIT_HISTORY} + +{_SELECTION} + +{_write_contract(deletion_allowed=True)} + +## This run + +- Generation: `{context.generation_id}` +- Language: {context.language} +- This is an **incremental update**: your version starts as a copy of the published + wiki, so pages you do not touch are carried over unchanged.\ +""" + + +def build_prompt(context: WikiRunContext, *, full: bool) -> str: + """Instructions for a run, chosen by mode.""" + return build_full_prompt(context) if full else build_incremental_prompt(context) + + +def build_diagram_correction(warnings: Sequence[str]) -> Optional[str]: + """Ask the agent to fix diagrams that will not render. + + Sent back as a follow-up rather than treated as a failure: a broken diagram is a + local display fault, and a version is not worth discarding over one. + """ + if not warnings: + return None + listed = "\n".join(f"- {warning}" for warning in warnings) + return ( + "Some Mermaid diagrams in the pages you wrote will not render. Fix them and " + "write those pages again, keeping their paths unchanged:\n" + f"{listed}" + ) diff --git a/backend/app/services/knowledge/code_wiki/publish_gate.py b/backend/app/services/knowledge/code_wiki/publish_gate.py new file mode 100644 index 0000000000..c0c3226609 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/publish_gate.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Deciding whether a finished version may be published. + +A version being a complete snapshot is guaranteed by construction — seeding puts every +page there — but not by the agent's honesty. An agent that writes four pages and +reports success produces a version holding four pages, and the projection would +faithfully delete the rest. + +So the version is checked before the published pointer moves. What makes this workable +is that the thing being checked is a complete, inspectable, retained snapshot rather +than a knowledge base already half-rewritten: a rejected version stays in the store +with its verdict attached, and the published one is still whatever it was. + +Removal is the substantive check. It is what makes agent-declared deletion safe to +allow at all: deletions are permitted, mass deletion is not. + +It is measured against the *set of published paths the version no longer contains*, +not against the two page counts. Counting would miss the case that matters most: a +version holding the same number of pages under mostly different paths is a mass +deletion plus a mass insertion, every affected page loses its document id, and every +stored citation and index entry pointing at it breaks. The count says nothing changed. +""" + +import logging +from dataclasses import dataclass, field +from typing import Optional, Sequence + +from app.services.knowledge.code_wiki.mermaid_check import ( + check_mermaid_blocks, + describe_warnings, +) +from app.services.knowledge.code_wiki.page_path import collation_key +from app.services.knowledge.code_wiki.projection_plan import PageSource + +logger = logging.getLogger(__name__) + +# Key under which the verdict is recorded on the generation. +PUBLISH_GATE_EXT_KEY = "publishGate" + + +@dataclass(frozen=True) +class PublishPolicy: + """Limits a version must respect to be published.""" + + # Share of the previously published pages that may disappear in one publish. + # Deleting pages is legitimate; deleting most of them is a malfunction, and the + # difference cannot be told apart by inspecting any single page. + max_removed_share: float = 0.5 + # A published wiki always has at least an overview page. A version with none is + # not an empty repository, it is a run that produced nothing. + min_pages: int = 1 + # Mermaid problems are reported, never blocking: a diagram that will not render is + # a local display fault, and holding a whole version for one is a poor trade. + block_on_mermaid: bool = False + + +DEFAULT_POLICY = PublishPolicy() + + +@dataclass(frozen=True) +class GateVerdict: + """Whether a version may be published, and what was noticed.""" + + passed: bool + reason: str = "" + warnings: tuple[str, ...] = field(default=()) + + def to_ext(self, checked_at: str) -> dict: + """Render for storage on the generation. + + Recorded to explain *why* a version is not live. It never decides that — + the published pointer alone does — because a second source of truth for + "which version is published" is a second thing that can be wrong. + """ + return { + "result": "passed" if self.passed else "rejected", + "reason": self.reason, + "warnings": list(self.warnings), + "checkedAt": checked_at, + } + + +def evaluate_publish_gate( + pages: Sequence[PageSource], + *, + published_paths: Sequence[str], + policy: Optional[PublishPolicy] = None, +) -> GateVerdict: + """Judge a finished version against the currently published one. + + Args: + pages: Every page in the version being considered. + published_paths: Paths currently published, empty when nothing has been. + Paths rather than a count: publishing a same-sized version under + different paths deletes every page it renamed, and a count cannot see it. + policy: Limits to apply. + + Returns: + The verdict. Warnings never cause a rejection on their own. + """ + policy = policy or DEFAULT_POLICY + diagram_warnings = _diagram_warnings(pages) + warnings = _structure_warnings(pages) + diagram_warnings + + if len(pages) < policy.min_pages: + return GateVerdict( + passed=False, + reason=( + f"version has {len(pages)} pages, fewer than the minimum " + f"{policy.min_pages}; the run produced nothing usable" + ), + warnings=warnings, + ) + + if published_paths: + kept = {collation_key(page.path) for page in pages} + removed = [path for path in published_paths if collation_key(path) not in kept] + removed_share = len(removed) / len(published_paths) + if removed_share > policy.max_removed_share: + return GateVerdict( + passed=False, + reason=( + f"version removes {removed_share:.0%} of the published pages " + f"({len(removed)} of {len(published_paths)}), over the " + f"{policy.max_removed_share:.0%} limit" + ), + warnings=warnings, + ) + + # Only the diagram warnings, despite `warnings` carrying more: a policy named + # for diagrams must not start rejecting versions over a missing section page. + if diagram_warnings and policy.block_on_mermaid: + return GateVerdict( + passed=False, + reason="diagram problems were found and the policy blocks on them", + warnings=warnings, + ) + + return GateVerdict(passed=True, warnings=warnings) + + +def _sections_without_a_page(pages: Sequence[PageSource]) -> list[str]: + """Sections that hold pages but have no page of their own. + + Reported, never blocking — the same trade as a broken diagram. The navigation is + built from paths, so such a section renders as a group heading that cannot be + opened: a little worse to read, and nowhere near worth discarding a version that + is otherwise complete. Asking for section pages belongs in the instructions, not + in a gate that throws away the run. + """ + present = {collation_key(page.path) for page in pages} + # Keyed the same way membership is tested. Collecting the raw string instead + # would report one section twice when two pages spell its prefix differently. + missing: dict[str, str] = {} + for page in pages: + if "/" not in page.path: + continue + section = page.path.rsplit("/", 1)[0] + key = collation_key(section) + if key not in present: + missing.setdefault(key, section) + return sorted(missing.values()) + + +def _structure_warnings(pages: Sequence[PageSource]) -> tuple[str, ...]: + """Navigation problems worth reporting but not worth refusing a version over.""" + return tuple( + f"{section}: holds pages but has no page of its own" + for section in _sections_without_a_page(pages) + ) + + +def _diagram_warnings(pages: Sequence[PageSource]) -> tuple[str, ...]: + """Diagrams that will not render, which the agent can be asked to fix.""" + collected: list[str] = [] + for page in pages: + mermaid_warnings = check_mermaid_blocks(page.content) + if mermaid_warnings: + collected.append(f"{page.path}: {describe_warnings(mermaid_warnings)}") + return tuple(collected) diff --git a/backend/app/services/knowledge/code_wiki/publisher.py b/backend/app/services/knowledge/code_wiki/publisher.py new file mode 100644 index 0000000000..1641215c03 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/publisher.py @@ -0,0 +1,413 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Publishing a code wiki version. + +This is the one place that moves ``spec.publishedGenerationId``. Everything else about +"which version is live" is derived from that pointer; nothing infers it from the newest +completed generation, because a generation can finish and still be rejected. + +The sequence is deliberately linear: + +1. read the version and the pages the knowledge base currently holds, +2. judge the version against the published one, +3. plan the difference, +4. apply the rows, advance the pointer, and commit both together, +5. clean up what could not be part of that transaction. + +The pointer moves *inside* the transaction that writes the rows, not after it. They +describe one fact — which version the knowledge base holds — and committing them +separately would leave a window where the pointer and the content disagree, in a +process that can die at any point. Step 5 is outside because the vector store and the +indexing queue cannot join a database transaction; what it cannot finish is parked and +swept later. + +A rollback is the same operation with an older version as its input, which is why no +separate machinery exists for it. +""" + +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Optional, Sequence + +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import KnowledgeDocument +from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.services.knowledge.code_wiki.page_path import collation_key +from app.services.knowledge.code_wiki.projection import ( + PENDING_INDEX_CLEANUP_KEY, + ProjectionSideEffects, + apply_projection_plan, + finish_projection, +) +from app.services.knowledge.code_wiki.projection_plan import ( + CONTENT_HASH_KEY, + PAGE_PATH_KEY, + PageSource, + ProjectedPage, + ProjectionPlan, + compute_projection_plan, +) +from app.services.knowledge.code_wiki.publish_gate import ( + PUBLISH_GATE_EXT_KEY, + GateVerdict, + PublishPolicy, + evaluate_publish_gate, +) +from app.services.knowledge.code_wiki.version_store import page_path_of +from app.services.knowledge.content_scope import generated_wiki_pages + +logger = logging.getLogger(__name__) + +PUBLISHED_GENERATION_KEY = "publishedGenerationId" + +# Written alongside the pointer, in the same transaction, because they describe what +# that transaction just did. A list needs them and would otherwise join every wiki +# against its generations to get two fields. +PUBLISHED_AT_KEY = "lastPublishedAt" +PUBLISHED_COMMIT_KEY = "lastPublishedCommit" + +# The order pages are shown in, as a list of paths. Kept here rather than on each +# document because a reorder must not touch a page whose content did not change: +# the fingerprint would still match, so the projection would skip it and the new +# position would never be written. One array, one write, no document churn. +PAGE_ORDER_KEY = "pageOrder" + + +@dataclass(frozen=True) +class PublishResult: + """What a publish attempt did.""" + + published: bool + verdict: GateVerdict + plan: Optional[ProjectionPlan] = None + reason: str = "" + + +def read_version_pages(db: Session, generation_id: int) -> tuple[PageSource, ...]: + """Read a version as the projection wants to see it. + + Entries without a page path predate page identity and are skipped rather than + guessed at: projecting one would create a document the next run could not match, + and so would delete and recreate on every publish. + """ + declared = _declared_order(db, generation_id) + + ranked: list[tuple[int, int, PageSource]] = [] + for entry in ( + db.query(WikiContent).filter(WikiContent.generation_id == generation_id).all() + ): + path = page_path_of(entry) + if not path: + logger.warning( + "[code_wiki] version entry %s has no page path; not projected", + entry.id, + ) + continue + # Pages the agent did not rank sort after the ones it did, in the order they + # were written. Alphabetical would be worse than arbitrary here: it puts "api" + # ahead of the overview, and a wiki read in that order reads wrong. + rank = declared.get(collation_key(path), len(declared)) + ranked.append( + ( + rank, + entry.id, + PageSource(path=path, title=entry.title, content=entry.content), + ) + ) + + ranked.sort(key=lambda item: (item[0], item[1])) + return tuple(page for _, _, page in ranked) + + +def _declared_order(db: Session, generation_id: int) -> dict[str, int]: + """The page order the agent declared when it finished, keyed for matching. + + Sent as ``summary.structure_order`` and until now recorded and never read. It is + the only statement of order there is: a path carries hierarchy but says nothing + about which section comes first. + """ + generation = db.get(WikiGeneration, generation_id) + if generation is None: + return {} + summary = (generation.ext or {}).get("content_write", {}).get("summary", {}) or {} + declared = summary.get("structure_order") or [] + return { + collation_key(str(path)): index + for index, path in enumerate(declared) + if str(path).strip() + } + + +def read_projected_pages(db: Session, kind_id: int) -> tuple[ProjectedPage, ...]: + """Read the generated wiki pages the knowledge base currently holds. + + Scoped through ``generated_wiki_pages`` rather than by an inline filter: user + content and code targets must never enter this comparison, because anything + missing from the version is treated as an orphan and deleted. + """ + query = generated_wiki_pages( + db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == kind_id) + ) + + pages: list[ProjectedPage] = [] + for document in query.all(): + config = document.source_config or {} + path = config.get(PAGE_PATH_KEY) + if not path: + # A generated page the projection did not create. Leaving it out means it + # is never deleted, which is the safe direction to be wrong in. + logger.warning( + "[code_wiki] generated document %s has no page path; left alone", + document.id, + ) + continue + pages.append( + ProjectedPage( + document_id=document.id, + path=path, + content_hash=config.get(CONTENT_HASH_KEY, ""), + ) + ) + return tuple(pages) + + +def published_generation_id(knowledge_base: Kind) -> int: + spec = (knowledge_base.json or {}).get("spec", {}) + try: + return int(spec.get(PUBLISHED_GENERATION_KEY, 0) or 0) + except (TypeError, ValueError): + return 0 + + +def _update_spec(knowledge_base: Kind, **values) -> None: + payload = dict(knowledge_base.json or {}) + spec = dict(payload.get("spec", {})) + spec.update(values) + payload["spec"] = spec + knowledge_base.json = payload + + +def _record_verdict(generation: WikiGeneration, verdict: GateVerdict) -> None: + ext = dict(generation.ext or {}) + ext[PUBLISH_GATE_EXT_KEY] = verdict.to_ext( + datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + ) + generation.ext = ext + + +def publish_generation( + db: Session, + *, + knowledge_base: Kind, + generation: WikiGeneration, + user_id: int, + effects: ProjectionSideEffects, + policy: Optional[PublishPolicy] = None, + require_completed: bool = True, +) -> PublishResult: + """Project a version into the knowledge base and make it the published one. + + Args: + db: Session. Committed here once the rows are in place. + knowledge_base: The code wiki's Kind. + generation: Version to publish. + user_id: Owner recorded on documents the projection creates. + effects: Side effects outside the transaction. + policy: Publish gate limits. + require_completed: Whether the generation must have finished successfully. + Set false to re-project an older version, which is what a rollback is. + + Returns: + Whether the version was published, with the gate's verdict either way. + """ + if require_completed and generation.status != WikiGenerationStatus.COMPLETED: + verdict = GateVerdict( + passed=False, + reason=f"generation is {generation.status}, not completed", + ) + _record_verdict(generation, verdict) + db.commit() + return PublishResult(published=False, verdict=verdict, reason=verdict.reason) + + # Settle what the last publish could not before computing this one. Doing it here + # rather than on a timer keeps this function the only writer of the parked list, + # which is what removes the race: two writers doing read-modify-write on one spec + # key will eventually drop a ref parked between a sweep's read and its write. + # + # It runs before the gate on purpose. The debt has nothing to do with whether this + # version is publishable, and a rejected version should still clear it. + retry_pending_index_cleanup(db, knowledge_base=knowledge_base, effects=effects) + + desired = read_version_pages(db, generation.id) + existing = read_projected_pages(db, knowledge_base.id) + + # The gate is asked what readers would lose, so it is given what readers can see + # now — the projected pages — rather than the published version's own page list. + # Those agree unless the knowledge base has drifted, and where they disagree the + # projection is the one that decides what actually gets deleted. + verdict = evaluate_publish_gate( + desired, + published_paths=[page.path for page in existing], + policy=policy, + ) + if not verdict.passed: + logger.warning( + "[code_wiki] generation %s rejected for kb %s: %s", + generation.id, + knowledge_base.id, + verdict.reason, + ) + _record_verdict(generation, verdict) + db.commit() + return PublishResult(published=False, verdict=verdict, reason=verdict.reason) + + plan = compute_projection_plan(desired, existing) + logger.info( + "[code_wiki] publishing generation %s into kb %s: %s", + generation.id, + knowledge_base.id, + plan.describe(), + ) + + superseded = _attachments_being_replaced(db, plan) + outcome = apply_projection_plan( + db, + kind_id=knowledge_base.id, + user_id=user_id, + plan=plan, + effects=effects, + ) + + _record_verdict(generation, verdict) + _update_spec( + knowledge_base, + **{ + PUBLISHED_GENERATION_KEY: generation.id, + PUBLISHED_AT_KEY: datetime.now(timezone.utc) + .replace(tzinfo=None) + .isoformat(), + PUBLISHED_COMMIT_KEY: str( + (generation.source_snapshot or {}).get("commit", "") or "" + ), + PAGE_ORDER_KEY: [page.path for page in desired], + }, + ) + db.commit() + + # Past this point the version is live. Nothing below may raise: the pages are + # already correct, and failing now would only cause the run to be repeated. + unfinished = finish_projection( + outcome, superseded_attachment_ids=superseded, effects=effects + ) + if unfinished: + _park_unfinished_cleanup(db, knowledge_base, unfinished) + db.commit() + + return PublishResult(published=True, verdict=verdict, plan=plan) + + +def _attachments_being_replaced(db: Session, plan: ProjectionPlan) -> tuple[int, ...]: + """Ids of attachments the plan supersedes, read before the rows change. + + Includes each page's converted attachment: overwriting the source without it + leaves a stale conversion that the document still points at. + """ + touched = [update.existing.document_id for update in plan.updates] + touched += [page.document_id for page in plan.deletes] + if not touched: + return () + + doomed: list[int] = [] + # One query rather than a lookup per document. Those lookups were served from the + # session's identity map today, because read_projected_pages had just loaded the + # same rows — a dependency on call order that nothing states and nothing enforces. + for document in ( + db.query(KnowledgeDocument).filter(KnowledgeDocument.id.in_(touched)).all() + ): + if document.attachment_id: + doomed.append(document.attachment_id) + converted = document.converted_attachment_id + if converted: + doomed.append(converted) + return tuple(doomed) + + +def _park_unfinished_cleanup( + db: Session, knowledge_base: Kind, doc_refs: Sequence[str] +) -> None: + """Record index deletions still owed, so a sweep can finish them. + + The vector store is external and cannot be rolled into the transaction, so a + failure has to outlive the process rather than disappear into a log line: a + document row that is gone while its chunks remain leaves retrieval answering + from a page that no longer exists. + """ + # Stored as strings throughout. The retry reads them back and writes them out + # again, so mixing in an int here would produce an entry that no longer matches + # the membership test below and the ref would be parked twice. + pending = [ + str(ref) + for ref in (knowledge_base.json or {}) + .get("spec", {}) + .get(PENDING_INDEX_CLEANUP_KEY) + or [] + ] + pending.extend(str(ref) for ref in doc_refs if str(ref) not in pending) + _update_spec(knowledge_base, **{PENDING_INDEX_CLEANUP_KEY: pending}) + logger.warning( + "[code_wiki] kb %s has %s index deletions awaiting retry", + knowledge_base.id, + len(pending), + ) + + +def retry_pending_index_cleanup( + db: Session, *, knowledge_base: Kind, effects: ProjectionSideEffects +) -> tuple[str, ...]: + """Attempt the index deletions a previous publish could not finish. + + Called at the start of every publish rather than on a timer. The cost is that a + wiki nobody regenerates keeps its orphaned chunks; the gain is that parking and + draining happen in one place, so neither can overwrite the other's view of the + list. A periodic sweeper would be a second writer of the same spec key, and the + interleaving that loses a ref needs no unusual timing to happen. + + The debt stays visible in the meantime: it is recorded on the knowledge base and + logged at warning level when it is parked. + + Returns: + The refs still outstanding. + """ + spec = (knowledge_base.json or {}).get("spec", {}) + pending = [str(ref) for ref in (spec.get(PENDING_INDEX_CLEANUP_KEY) or [])] + if not pending: + return () + + still_pending: list[str] = [] + for doc_ref in pending: + if not doc_ref.isdigit(): + # Dropped rather than retried. A ref that is not a document id can never + # be deleted, so keeping it means every sweep from here on fails on it + # and the list never drains. + logger.error( + "[code_wiki] kb %s parked an unusable index ref %r; dropping it", + knowledge_base.id, + doc_ref, + ) + continue + try: + effects.delete_rag_document(int(doc_ref)) + except Exception as exc: + logger.warning( + "[code_wiki] index cleanup still failing for %s: %s", doc_ref, exc + ) + still_pending.append(doc_ref) + + _update_spec(knowledge_base, **{PENDING_INDEX_CLEANUP_KEY: still_pending}) + db.commit() + return tuple(still_pending) diff --git a/backend/app/services/knowledge/code_wiki/registry.py b/backend/app/services/knowledge/code_wiki/registry.py new file mode 100644 index 0000000000..440ef00c4e --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/registry.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Which repository a code wiki documents. + +A code wiki is an ordinary knowledge base owned by whoever created it, so the +ordinary ACL decides who may read it. The repository is consulted once, when the +wiki is created, and never tracked afterwards. + +**A repository may have several wikis.** One per person who built one: since a wiki +created by A is invisible to B under A's ACL, refusing B a wiki of their own would +take it away on a first-come basis. ``wiki_projects`` therefore holds one row per +``(repository, wiki)`` rather than one per repository, and the composite UNIQUE is +what settles two requests racing for the same pair — a check against a JSON field on +the knowledge base could not, leaving a window between the read and the insert +exactly wide enough for the case worth preventing. + +Rows with ``kind_id = 0`` belong to the legacy wiki, at most one per repository. +""" + +import logging +from dataclasses import dataclass +from typing import Optional + +from sqlalchemy.orm import Session + +from app.models.wiki import WikiProject +from app.services.knowledge.code_wiki.source import SourceRepository + +logger = logging.getLogger(__name__) + +# Where a code wiki lands when the caller expressed no preference. Unlike every +# other decision here this one is a plain default, not a constraint: the creator may +# file a wiki in any namespace they could file a knowledge base in. +CODE_WIKI_NAMESPACE = "default" + + +def claim_repository(db: Session, source: SourceRepository, kind_id: int) -> None: + """Register that ``kind_id`` is a wiki of this repository. + + Flushed rather than committed: the caller owns the transaction, so a failure + later in the same request must take this row with it. + """ + project = WikiProject( + project_name=source.project_name, + project_type="git", + source_type=source.source_type, + source_url=source.source_url, + source_domain=source.source_domain, + description="", + ext={}, + kind_id=kind_id, + is_active=True, + ) + db.add(project) + db.flush() + logger.info( + "[code_wiki] registered repository %s as project %s for kb %s", + source.project_name, + project.id, + kind_id, + ) + + +def existing_wiki_id( + db: Session, source: SourceRepository, *, owner_id: int +) -> Optional[int]: + """This caller's own wiki of this repository, if they already have one. + + Scoped to the caller because somebody else's wiki is not an answer to "give me a + wiki of this repository" — under their ACL the caller may not even be able to + read it. + """ + from app.models.kind import Kind + + row = ( + db.query(WikiProject.kind_id) + .join(Kind, Kind.id == WikiProject.kind_id) + .filter( + WikiProject.source_url == source.source_url, + WikiProject.kind_id > 0, + Kind.user_id == owner_id, + Kind.is_active.is_(True), + ) + .first() + ) + return row[0] if row else None + + +@dataclass(frozen=True) +class ExistingWiki: + """A wiki of this repository that somebody has already built.""" + + id: int + name: str + owner_name: str + # Whether the caller can already open it. False means the useful action is to + # ask its owner for a share, which is why the owner is named at all. + accessible: bool + + +def existing_wikis_for( + db: Session, source_url: str, *, viewer_id: int +) -> list[ExistingWiki]: + """Wikis already built from this repository, whoever owns them. + + Shown before creating another so that somebody can ask for a share instead of + paying for a second generation — which needs the owner's name, not just a count. + + Listing wikis the caller cannot open is the point rather than a leak to weigh: + the caller has demonstrated they can read the repository, and the disclosure is + that a colleague documented it. Names of accessible wikis come with an id so the + client can link straight to them. + """ + from app.models.kind import Kind + from app.models.user import User + from app.services.knowledge.knowledge_service import KnowledgeService + + rows = ( + db.query(Kind, User.user_name) + .join(WikiProject, WikiProject.kind_id == Kind.id) + .outerjoin(User, User.id == Kind.user_id) + .filter( + WikiProject.source_url == source_url, + WikiProject.kind_id > 0, + Kind.is_active.is_(True), + ) + .order_by(Kind.created_at.asc()) + .all() + ) + + visible = {kind.id for kind in KnowledgeService.list_knowledge_bases(db, viewer_id)} + return [ + ExistingWiki( + id=kind.id, + name=(kind.json or {}).get("spec", {}).get("name") or kind.name, + owner_name=owner_name or "", + accessible=kind.id in visible, + ) + for kind, owner_name in rows + ] diff --git a/backend/app/services/knowledge/code_wiki/repo_state.py b/backend/app/services/knowledge/code_wiki/repo_state.py new file mode 100644 index 0000000000..137730985e --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/repo_state.py @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Reading what a repository is at right now, so a run can decide whether to happen. + +Without this, ``decide_run_mode`` never sees a HEAD commit, and two of its three +answers become unreachable: it cannot recognise an unchanged repository, and with no +diff it must assume the worst and rebuild. A weekly schedule over a quiet repository +then pays a full pass through the model every week to conclude nothing happened. + +The mode has to be chosen *before* the agent starts — it decides whether the version +is seeded and which instructions are sent — so the agent's own clone cannot answer +this. It has to be read from the provider first. + +Everything here degrades to "unknown", never to a guess. A provider that is down, an +older self-hosted instance without a compare endpoint, a diff too large to return +whole: each of those yields ``None``, which ``decide_run_mode`` reads as *the extent +of the change is unknown* and answers with a full rebuild. That is the expensive +answer, and it is the only safe one — a partial diff mistaken for a complete one +picks an incremental run for a change that reshaped the repository. +""" + +import logging +from dataclasses import dataclass +from typing import Optional + +from sqlalchemy.orm import Session + +from app.services.git_skill.utils import get_user_git_info +from app.services.knowledge.code_wiki.run_mode import ChangedPath +from app.services.knowledge.code_wiki.source import SourceRepository, provider_for + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class RepositoryState: + """What the repository looks like now, as far as could be determined.""" + + head_commit: str = "" + branch: str = "" + # ``None`` means the diff is unknown, which is different from "nothing changed". + changed_paths: Optional[tuple[ChangedPath, ...]] = None + + +def read_repository_state( + db: Session, + *, + user_id: int, + source: SourceRepository, + since_commit: str = "", +) -> RepositoryState: + """Read the default branch's HEAD, and the diff since ``since_commit``. + + Args: + db: Session, for looking up the caller's credentials. + user_id: Whose token to read the repository with. + source: The repository the wiki is bound to. + since_commit: Commit the published wiki documents. Empty on a first run, in + which case no diff is asked for — there is nothing to compare against. + + Returns: + What could be read. Every field is best-effort: a failure anywhere leaves the + state less certain, never wrong. + """ + provider = provider_for(source.source_type) + if provider is None: + logger.info( + "[code_wiki] no provider for '%s'; repository state unknown", + source.source_type, + ) + return RepositoryState() + + git_info = get_user_git_info(user_id=user_id, domain=source.source_domain, db=db) + token = (git_info or {}).get("token") + if not token: + logger.info( + "[code_wiki] user %s has no credentials for %s; repository state unknown", + user_id, + source.source_domain, + ) + return RepositoryState() + + head = _read_head(provider, token, source) + if not head.head_commit or not since_commit: + return head + + return RepositoryState( + head_commit=head.head_commit, + branch=head.branch, + changed_paths=_read_changed_paths( + provider, token, source, since_commit, head.head_commit + ), + ) + + +def _read_head(provider, token: str, source: SourceRepository) -> RepositoryState: + """The default branch and its commit, or an empty state when it cannot be read.""" + try: + result = provider.get_default_branch_head( + token=token, + git_domain=source.source_domain, + repo_name=source.project_name, + ) + except Exception as exc: + logger.warning( + "[code_wiki] could not read HEAD of %s: %s", source.project_name, exc + ) + return RepositoryState() + + return RepositoryState( + head_commit=str(result.get("commit", "") or ""), + branch=str(result.get("branch", "") or ""), + ) + + +def _read_changed_paths( + provider, token: str, source: SourceRepository, base: str, head: str +) -> Optional[tuple[ChangedPath, ...]]: + """The diff between two commits, or ``None`` when it cannot be trusted.""" + if base == head: + # Nothing to compare. Returning an empty diff rather than ``None`` matters: + # it is what lets the run be skipped instead of rebuilt. + return () + + try: + entries = provider.get_changed_files( + token=token, + git_domain=source.source_domain, + repo_name=source.project_name, + base=base, + head=head, + ) + except Exception as exc: + logger.warning( + "[code_wiki] could not diff %s..%s in %s: %s", + base, + head, + source.project_name, + exc, + ) + return None + + if entries is None: + return None + return tuple( + ChangedPath(path=entry["path"], status=entry.get("status", "M")) + for entry in entries + if entry.get("path") + ) diff --git a/backend/app/services/knowledge/code_wiki/resolution.py b/backend/app/services/knowledge/code_wiki/resolution.py new file mode 100644 index 0000000000..4988d20015 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/resolution.py @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""What a caller needs to know about a repository before binding a wiki to it. + +One call answers three questions that would otherwise be three probes: + +1. **May this caller read it?** — which decides whether creating a wiki is allowed. +2. **What is its default branch?** — so the create form does not need a branch listing, + which has no anonymous path and would be a second thing to open up. +3. **What is it called, and what is it?** — used to fill in a name and description the + caller left blank. + +Repositories can be read without a credential when they are public, and that case has +to work: the repository selector only lists repositories the caller is a member of, so +a public repository is one they can read in a browser but cannot pick from a list. A +wiki that could not be built for it would be more closed than the thing it documents. + +Anonymous requests to GitHub are limited to 60 an hour per address, so results are +cached. The cache is keyed by whether a credential was used, because the same +repository legitimately answers differently with and without one. +""" + +import logging +from dataclasses import dataclass +from typing import Optional + +from sqlalchemy.orm import Session + +from app.core.cache import cache_manager +from app.services.git_skill.utils import get_user_git_info +from app.services.knowledge.code_wiki.source import ( + SUPPORTED_SOURCE_TYPES, + SourceAccessDenied, + SourceRepository, + provider_for, +) + +logger = logging.getLogger(__name__) + +# Long enough that a form filled in over several minutes costs one request, short +# enough that a repository turned private stops being reported as public within the +# hour. Visibility is a cache here, never a stored fact. +RESOLUTION_CACHE_SECONDS = 600 + + +@dataclass(frozen=True) +class ResolvedRepository: + """A repository the caller may read, described well enough to bind.""" + + exists: bool + visibility: str + default_branch: str + name: str + description: str + access: str # "public" | "member" | "none" + + +UNREADABLE = ResolvedRepository( + exists=False, + visibility="", + default_branch="", + name="", + description="", + access="none", +) + + +def resolve_repository( + db: Session, user_id: int, source: SourceRepository +) -> ResolvedRepository: + """Describe a repository as far as this caller is entitled to see it. + + Never raises for an unreadable repository: "you cannot read this" is an answer + the form displays, not an error. Absent and private are reported identically, + since telling them apart would disclose which private repositories exist. + """ + if source.source_type not in SUPPORTED_SOURCE_TYPES: + raise SourceAccessDenied( + f"Unsupported repository type '{source.source_type}'. " + f"Supported types: {', '.join(SUPPORTED_SOURCE_TYPES)}" + ) + + provider = provider_for(source.source_type) + if provider is None: # pragma: no cover - guarded by the check above + return UNREADABLE + + git_info = get_user_git_info(user_id=user_id, domain=source.source_domain, db=db) + token = (git_info or {}).get("token") or "" + + cached = _cached(source, has_token=bool(token)) + if cached is not None: + return cached + + described = provider.describe_repository( + token=token, + git_domain=source.source_domain, + repo_name=source.project_name, + ) + if not described: + # Not cached: a repository that is about to be granted to the caller should + # not stay unreadable for ten minutes because they asked one moment early. + return UNREADABLE + + resolved = ResolvedRepository( + exists=True, + visibility=str(described.get("visibility") or "private"), + default_branch=str(described.get("default_branch") or ""), + name=str(described.get("name") or source.project_name), + description=str(described.get("description") or ""), + access="member" if token else "public", + ) + _remember(source, has_token=bool(token), resolved=resolved) + return resolved + + +def _cache_key(source: SourceRepository, *, has_token: bool) -> str: + return ( + f"code_wiki:resolve:{source.source_type}:{source.source_domain}:" + f"{source.project_name}:{'auth' if has_token else 'anon'}" + ) + + +def _cached( + source: SourceRepository, *, has_token: bool +) -> Optional[ResolvedRepository]: + try: + payload = _run(cache_manager.get(_cache_key(source, has_token=has_token))) + except Exception as exc: # pragma: no cover - cache is an optimisation + logger.debug("[code_wiki] resolution cache read failed: %s", exc) + return None + if not isinstance(payload, dict): + return None + return ResolvedRepository(**payload) + + +def _remember( + source: SourceRepository, *, has_token: bool, resolved: ResolvedRepository +) -> None: + try: + _run( + cache_manager.set( + _cache_key(source, has_token=has_token), + resolved.__dict__, + expire=RESOLUTION_CACHE_SECONDS, + ) + ) + except Exception as exc: # pragma: no cover - cache is an optimisation + logger.debug("[code_wiki] resolution cache write failed: %s", exc) + + +def _run(coroutine): + """Await a cache coroutine from this synchronous path. + + The endpoint is synchronous, so there is no running loop to attach to; a fresh + one per call is the same shape the repository providers already use. + """ + import asyncio + + return asyncio.run(coroutine) diff --git a/backend/app/services/knowledge/code_wiki/run_mode.py b/backend/app/services/knowledge/code_wiki/run_mode.py new file mode 100644 index 0000000000..26cd735617 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/run_mode.py @@ -0,0 +1,206 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Deciding how much of a code wiki a run should rebuild. + +The mode decides **how a version is built**, and nothing else: + +- ``SKIP`` — the repository has not moved, so no version is created. +- ``INCREMENTAL`` — the default. The version is seeded from the published one and the + agent revises only the pages the changes affect. +- ``FULL`` — the version starts empty and the agent writes every page. + +Both modes end with a complete snapshot, so **publishing does not depend on the mode**. +An earlier design let the mode decide whether the run's reported page set could be +used to delete pages, which needed a matching set of heuristics to keep an incremental +run from deleting every page it had not touched. Seeding removes that question: the +projection always compares complete snapshots, so orphans are a plain set difference. + +A full rebuild is still forced periodically. An incremental run reworking the page +layout can leave pages nothing points at any more, and neither the agent nor the diff +has the whole picture; starting from an empty version is what clears them. +""" + +from dataclasses import dataclass, field +from enum import Enum +from fnmatch import fnmatchcase +from typing import Optional, Sequence + + +class RunMode(str, Enum): + """How much of the wiki a single run rebuilds.""" + + SKIP = "skip" + INCREMENTAL = "incremental" + FULL = "full" + + +@dataclass(frozen=True) +class ChangedPath: + """One entry from the diff between the last documented commit and HEAD.""" + + path: str + # Git name-status letter: "A" added, "M" modified, "D" deleted, "R" renamed. + status: str + + @property + def is_structural_move(self) -> bool: + """Whether this entry adds or removes a file rather than editing one.""" + return self.status.upper().startswith(("A", "D", "R")) + + +# Files that describe how the project is built or what it depends on. A change here +# usually reshapes the architecture the wiki describes, so it earns a full rebuild +# even when only one file moved. +DEFAULT_MANIFEST_PATTERNS: tuple[str, ...] = ( + "package.json", + "pnpm-workspace.yaml", + "pyproject.toml", + "requirements*.txt", + "go.mod", + "Cargo.toml", + "pom.xml", + "build.gradle*", + "Makefile", + "Dockerfile", + "docker-compose*.yml", + "*/package.json", + "*/pyproject.toml", + "*/go.mod", + "*/Cargo.toml", +) + + +@dataclass(frozen=True) +class RunModePolicy: + """Thresholds that promote an incremental run to a full rebuild.""" + + max_changed_files: int = 50 + max_structural_moves: int = 15 + max_changed_ratio: float = 0.25 + max_incrementals_since_full: int = 10 + max_days_since_full: float = 30.0 + manifest_patterns: tuple[str, ...] = field(default=DEFAULT_MANIFEST_PATTERNS) + + +DEFAULT_POLICY = RunModePolicy() + + +@dataclass(frozen=True) +class RunModeDecision: + """The chosen mode and why, recorded on the run for troubleshooting.""" + + mode: RunMode + reason: str + + @property + def seeds_from_published(self) -> bool: + """Whether the new version starts as a copy of the published one. + + Compared by value, not identity: ``RunMode`` is a ``str`` enum so that a mode + survives a round trip through a task payload, and ``"incremental" is + RunMode.INCREMENTAL`` is false. An identity test would quietly stop seeding, + and an unseeded incremental version is a partial snapshot — the projection + would read every page the run did not touch as an orphan and delete it. + """ + return RunMode(self.mode) == RunMode.INCREMENTAL + + +def _matches_manifest(path: str, patterns: Sequence[str]) -> bool: + """Match case-sensitively, so behaviour does not depend on the host platform.""" + return any(fnmatchcase(path, pattern) for pattern in patterns) + + +def decide_run_mode( + *, + head_commit: str, + last_commit: Optional[str] = None, + changed_paths: Optional[Sequence[ChangedPath]] = None, + incrementals_since_full: int = 0, + days_since_full: Optional[float] = None, + policy: RunModePolicy = DEFAULT_POLICY, + total_source_files: Optional[int] = None, +) -> RunModeDecision: + """Choose the mode for one run. + + Args: + head_commit: Commit the repository is at now. + last_commit: Commit the wiki was last generated from; absent on a first run. + changed_paths: Diff between ``last_commit`` and ``head_commit``. When absent + the extent of the change is unknown, so a full rebuild is chosen. + incrementals_since_full: Incremental runs completed since the last full one. + days_since_full: Days since the last full run, if one has happened. + policy: Thresholds to apply. + total_source_files: Files under consideration at ``head_commit``, used for the + proportional threshold; skipped when unknown. + """ + if not last_commit: + return RunModeDecision(RunMode.FULL, "first run for this repository") + + if last_commit == head_commit: + return RunModeDecision(RunMode.SKIP, "repository unchanged since last run") + + if changed_paths is None: + return RunModeDecision( + RunMode.FULL, "extent of changes unknown, rebuilding to stay correct" + ) + + if not changed_paths: + # The commit moved but nothing we document did — treat as unchanged rather + # than paying for a rebuild. + return RunModeDecision(RunMode.SKIP, "no documented files changed") + + manifests = [ + change.path + for change in changed_paths + if _matches_manifest(change.path, policy.manifest_patterns) + ] + if manifests: + return RunModeDecision( + RunMode.FULL, f"build or dependency manifest changed: {manifests[0]}" + ) + + # Files appearing, disappearing or moving reshape what the wiki documents far more + # than edits do, so a burst of them earns a rebuild sooner than plain edits. + structural = [change for change in changed_paths if change.is_structural_move] + if len(structural) > policy.max_structural_moves: + return RunModeDecision( + RunMode.FULL, + f"{len(structural)} files added, removed or renamed, over the limit of " + f"{policy.max_structural_moves}", + ) + + if len(changed_paths) > policy.max_changed_files: + return RunModeDecision( + RunMode.FULL, + f"{len(changed_paths)} files changed, over the limit of " + f"{policy.max_changed_files}", + ) + + if total_source_files and total_source_files > 0: + ratio = len(changed_paths) / total_source_files + if ratio > policy.max_changed_ratio: + return RunModeDecision( + RunMode.FULL, + f"{ratio:.0%} of files changed, over the limit of " + f"{policy.max_changed_ratio:.0%}", + ) + + # Periodic rebuild: incremental runs cannot see pages orphaned by restructuring + # or by the agent relaying out the wiki, so those only get cleaned up here. + if incrementals_since_full >= policy.max_incrementals_since_full: + return RunModeDecision( + RunMode.FULL, + f"{incrementals_since_full} incremental runs since the last full rebuild", + ) + + if days_since_full is not None and days_since_full >= policy.max_days_since_full: + return RunModeDecision( + RunMode.FULL, + f"{days_since_full:.0f} days since the last full rebuild", + ) + + return RunModeDecision( + RunMode.INCREMENTAL, f"{len(changed_paths)} files changed since last run" + ) diff --git a/backend/app/services/knowledge/code_wiki/runner.py b/backend/app/services/knowledge/code_wiki/runner.py new file mode 100644 index 0000000000..cbcfd9f67b --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/runner.py @@ -0,0 +1,377 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Driving a code wiki run from a knowledge base to an agent and back. + +Everything either side of this module already exists and is tested on its own: the +version store decides what a run may do, the prompts say what the agent must produce, +and the publisher turns a finished version into knowledge base content. What was +missing is the part that actually starts one — resolving the repository, choosing the +mode, writing the instructions and handing them to a Task. + +Two ordering decisions here are deliberate: + +**The generation is committed before the task is created.** A crash between the two +then leaves a run with no task, which the staleness sweep reclaims after six hours. The +alternative ordering — task first — leaves a task with no run to report into, which +nothing reclaims and which would write its pages into a version that does not exist. + +**A task that cannot be created fails the run immediately** rather than leaving it +RUNNING for the sweep to find. A wiki that refuses to regenerate for six hours because +of a misconfigured team is a much worse failure than one that says so at once. +""" + +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Optional, Sequence + +from sqlalchemy.orm import Session + +from app.core.wiki_config import wiki_settings +from app.models.kind import Kind +from app.models.user import User +from app.models.wiki import WikiGeneration, WikiGenerationStatus +from app.schemas.knowledge import KnowledgeBaseType +from app.schemas.task import TaskCreate +from app.services.knowledge.code_wiki.generation import ( + SOURCE_COMMIT_KEY, + finish_generation, + published_commit, + start_generation, +) +from app.services.knowledge.code_wiki.prompts import WikiRunContext, build_prompt +from app.services.knowledge.code_wiki.publisher import PublishResult, read_version_pages +from app.services.knowledge.code_wiki.repo_state import read_repository_state +from app.services.knowledge.code_wiki.run_mode import ChangedPath, RunMode +from app.services.knowledge.code_wiki.side_effects import build_projection_side_effects +from app.services.knowledge.code_wiki.source import SourceRepository + +logger = logging.getLogger(__name__) + +# What the model is told to write in, keyed by the configured language code. The +# prompt states this in prose, so it needs a language name rather than a code. +_LANGUAGE_NAMES = {"en": "English", "zh": "Chinese (Simplified)"} + + +class CodeWikiRunError(RuntimeError): + """Raised when a run cannot be set up. The message is shown to the caller.""" + + +@dataclass(frozen=True) +class StartedRun: + """The outcome of asking a code wiki to regenerate.""" + + generation: Optional[WikiGeneration] + reason: str + mode: str = "" + task_id: int = 0 + + @property + def started(self) -> bool: + return self.generation is not None + + +def source_of(knowledge_base: Kind) -> SourceRepository: + """Read the repository a code wiki is bound to. + + Raises: + CodeWikiRunError: If this knowledge base is not a code wiki, or is one with no + usable repository recorded — which would otherwise surface much later as a + task cloning an empty URL. + """ + spec = (knowledge_base.json or {}).get("spec", {}) + if spec.get("kbType") != KnowledgeBaseType.CODE_WIKI.value: + raise CodeWikiRunError("This knowledge base is not a code wiki") + + source = SourceRepository.from_spec(spec.get("source")) + if source is None or not source.source_url or not source.project_name: + raise CodeWikiRunError( + "This code wiki has no source repository recorded and cannot be generated" + ) + return source + + +def start_run( + db: Session, + *, + knowledge_base: Kind, + user: User, + head_commit: str = "", + changed_paths: Optional[Sequence[ChangedPath]] = None, + total_source_files: Optional[int] = None, +) -> StartedRun: + """Start a run for ``knowledge_base`` and hand its instructions to a task. + + Args: + db: Session. + knowledge_base: The code wiki to regenerate. + user: Identity the run is attributed to. + head_commit: Commit the repository is at now. When empty it is read from the + provider, and only if that also fails is the extent of the change taken + as unknown — which costs a full rebuild. + changed_paths: Diff since the published commit. ``None`` asks for it to be + read from the provider alongside the commit. + total_source_files: Repository size, used by the change-ratio threshold. + + Returns: + The started run, or a reason why none was needed. + + Raises: + CodeWikiRunError: If the wiki cannot be generated, or a run is already going. + GenerationInFlight: Propagated from the version store. + """ + source = source_of(knowledge_base) + team, task_user = _resolve_execution_context(db, user) + + previous_commit = published_commit(db, knowledge_base) + if not head_commit: + # Read as the user that will clone the repository, not the one who asked. On + # a schedule there is no asker, and a token that cannot read the repository + # would answer for a run that is about to fail at checkout anyway. + state = read_repository_state( + db, + user_id=task_user.id, + source=source, + since_commit=previous_commit, + ) + head_commit = state.head_commit + if changed_paths is None: + changed_paths = state.changed_paths + + started = start_generation( + db, + knowledge_base=knowledge_base, + # The account that runs the task, not the one that asked: it is the identity + # the agent authenticates as, so anything scoped to "this run's owner" has to + # agree with it, and it is the account that owns the knowledge base being + # published into. + user=task_user, + head_commit=head_commit, + changed_paths=changed_paths, + total_source_files=total_source_files, + team_id=team.id, + ) + if not started.started: + return StartedRun( + generation=None, reason=started.decision.reason, mode=started.decision.mode + ) + + generation = started.generation + full = RunMode(started.decision.mode) is RunMode.FULL + prompt = build_prompt( + WikiRunContext( + project_name=source.project_name, + generation_id=generation.id, + head_commit=head_commit, + language=_LANGUAGE_NAMES.get( + (wiki_settings.DEFAULT_LANGUAGE or "en").lower(), "English" + ), + previous_commit=previous_commit, + changed_paths=[change.path for change in changed_paths or ()], + existing_pages=[ + page.path for page in read_version_pages(db, generation.id) + ], + ), + full=full, + ) + + # Committed before the task exists, so that a task always has a version to report + # into. See the module docstring for why the reverse ordering is worse. + db.commit() + + task_id = _create_task( + db, + source=source, + team_id=team.id, + task_user=task_user, + prompt=prompt, + generation=generation, + ) + + generation.task_id = task_id + db.commit() + logger.info( + "[code_wiki] generation %s for kb %s running under task %s", + generation.id, + knowledge_base.id, + task_id, + ) + return StartedRun( + generation=generation, + reason=started.decision.reason, + mode=started.decision.mode, + task_id=task_id, + ) + + +def finish_run( + db: Session, + *, + generation: WikiGeneration, + succeeded: bool, + error_message: str = "", + head_commit: str = "", +) -> Optional[PublishResult]: + """Conclude a run the agent has reported on, and publish it if it succeeded. + + Args: + db: Session. + generation: The run being concluded. + succeeded: Whether the agent reported success. + error_message: Why it failed, when it did. + head_commit: Commit the agent actually documented. Recorded over whatever the + run started with, because the agent read the working tree and the trigger + only knew what it was told — and this value is what the next run's mode + decision compares against. + + Returns: + The publish outcome, or ``None`` when the run failed or was not publishable. + """ + knowledge_base = _knowledge_base_of(db, generation) + if knowledge_base is None: + raise CodeWikiRunError( + f"generation {generation.id} has no knowledge base to publish into" + ) + + user = db.get(User, generation.user_id) + if user is None: + raise CodeWikiRunError( + f"generation {generation.id} has no user to publish as " + f"(user {generation.user_id})" + ) + + if head_commit: + snapshot = dict(generation.source_snapshot or {}) + snapshot[SOURCE_COMMIT_KEY] = head_commit + generation.source_snapshot = snapshot + db.flush() + + return finish_generation( + db, + knowledge_base=knowledge_base, + generation=generation, + user=user, + effects=build_projection_side_effects( + db, knowledge_base=knowledge_base, user=user + ), + succeeded=succeeded, + error_message=error_message, + ) + + +def is_code_wiki_generation(db: Session, generation: WikiGeneration) -> bool: + """Whether this run belongs to a code wiki rather than the legacy wiki path.""" + return _knowledge_base_of(db, generation) is not None + + +def _knowledge_base_of(db: Session, generation: WikiGeneration) -> Optional[Kind]: + """The code wiki a run belongs to, or ``None`` for a legacy generation.""" + if not generation.kind_id: + return None + knowledge_base = db.get(Kind, generation.kind_id) + if knowledge_base is None or knowledge_base.kind != "KnowledgeBase": + return None + return knowledge_base + + +def _resolve_execution_context(db: Session, user: User) -> tuple[Kind, User]: + """Find the team that runs code wikis, and the user it runs as. + + The run executes as the knowledge base's owner, using their Git credentials to + clone. An expired token then fails that owner's own wiki, which is attributable; + a shared account's expiry would fail everybody's at once. + + The team, by contrast, comes from configuration rather than from the request: it + carries the prompt and the tools the agent gets, so letting the caller choose it + would let them choose those. + """ + from app.services.adapters.team_kinds import team_kinds_service + + task_user = user + team_name = wiki_settings.CODE_WIKI_TEAM_NAME + if not team_name: + raise CodeWikiRunError( + "WIKI_CODE_WIKI_TEAM_NAME is not configured, so there is no team to run " + "the wiki agent" + ) + + team = team_kinds_service.get_team_by_name_and_namespace( + db=db, + team_name=team_name, + team_namespace="default", + user_id=task_user.id, + ) + if not team: + raise CodeWikiRunError( + f"Code wiki team '{team_name}' was not found for user {task_user.id}. " + "Check WIKI_CODE_WIKI_TEAM_NAME and that the default resources are loaded." + ) + return team, task_user + + +def _create_task( + db: Session, + *, + source: SourceRepository, + team_id: int, + task_user: User, + prompt: str, + generation: WikiGeneration, +) -> int: + """Create the task that runs the agent, failing the run if it cannot be.""" + from app.services.adapters.task_kinds import task_kinds_service + + try: + task_id = task_kinds_service.create_task_id(db, task_user.id) + task_kinds_service.create_task_or_append( + db=db, + obj_in=TaskCreate( + title=f"Code wiki: {source.project_name}", + team_id=team_id, + git_url=source.source_url, + git_repo=source.project_name, + git_repo_id=0, + git_domain=source.source_domain, + # Empty means the repository's default branch, resolved at clone time, + # so a wiki always documents whatever that branch currently is. + branch_name="", + prompt=prompt, + type="online", + task_type="code", + auto_delete_executor="false", + source="code_wiki", + ), + user=task_user, + task_id=task_id, + ) + return task_id + except Exception as exc: + logger.error( + "[code_wiki] could not create a task for generation %s: %s", + generation.id, + exc, + ) + _fail_without_a_task(db, generation, exc) + raise CodeWikiRunError(f"Could not start the wiki task: {exc}") from exc + + +def _fail_without_a_task( + db: Session, generation: WikiGeneration, exc: Exception +) -> None: + """Mark a run failed when nothing will ever report on it. + + Left RUNNING it would block the wiki until the staleness sweep, which is hours of + a wiki refusing to regenerate over a configuration error visible right now. + """ + db.rollback() + live = db.get(WikiGeneration, generation.id) + if live is None: + return + live.status = WikiGenerationStatus.FAILED + live.completed_at = datetime.now(timezone.utc).replace(tzinfo=None) + ext = dict(live.ext or {}) + ext["errorMessage"] = f"task creation failed: {exc}" + live.ext = ext + db.commit() diff --git a/backend/app/services/knowledge/code_wiki/side_effects.py b/backend/app/services/knowledge/code_wiki/side_effects.py new file mode 100644 index 0000000000..7436a3e663 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/side_effects.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Wiring the projection's side effects to the real services. + +The projection takes these as injected callables so its ordering rules can be +asserted on directly. This module is the other half of that arrangement: the adapters +that talk to attachment storage, the vector store and the indexing queue. + +Keeping them here rather than inside the projection means the rules and the plumbing +fail separately — a signature that turns out to be wrong shows up as an adapter fault +rather than as an ordering test that no longer proves anything. +""" + +import logging + +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import KnowledgeDocument +from app.models.user import User +from app.services.knowledge.code_wiki.projection import ProjectionSideEffects + +logger = logging.getLogger(__name__) + + +def build_projection_side_effects( + db: Session, + *, + knowledge_base: Kind, + user: User, +) -> ProjectionSideEffects: + """Assemble the real side effects for projecting into ``knowledge_base``. + + Args: + db: Session the projection runs in. Attachment writes join it; the vector + store and the queue cannot, which is why their failures are retried + rather than rolled back. + knowledge_base: The code wiki being projected into. + user: Identity attachments and indexing run under. + """ + from app.services.context import context_service + + def write_attachment(*, filename: str, content: str) -> int: + attachment, _ = context_service.upload_attachment( + db=db, + user_id=user.id, + filename=filename, + binary_data=content.encode("utf-8"), + subtask_id=0, + ) + return attachment.id + + def delete_attachment(attachment_id: int) -> None: + context_service.delete_context( + db=db, + context_id=attachment_id, + user_id=user.id, + ) + + def delete_rag_document(document_id: int) -> None: + _delete_document_index( + db, knowledge_base=knowledge_base, user=user, document_id=document_id + ) + + def enqueue_reindex(document_id: int) -> None: + _enqueue_reindex( + db, knowledge_base=knowledge_base, user=user, document_id=document_id + ) + + return ProjectionSideEffects( + write_attachment=write_attachment, + delete_attachment=delete_attachment, + delete_rag_document=delete_rag_document, + enqueue_reindex=enqueue_reindex, + ) + + +def _delete_document_index( + db: Session, *, knowledge_base: Kind, user: User, document_id: int +) -> None: + """Remove a removed page's chunks from the vector store. + + Raises on failure so the caller can park the reference and retry: chunks left + behind outlive the page, and retrieval goes on answering from it while citing an + id that no longer resolves. + """ + from app.services.knowledge.index_runtime import get_kb_index_info_by_record + from app.services.rag.gateway_factory import get_delete_gateway + from app.services.rag.runtime_resolver import RagRuntimeResolver + + spec = (knowledge_base.json or {}).get("spec", {}) + if not spec.get("retrievalConfig"): + # Nothing was ever indexed, so there is nothing to remove. + return + + index_info = get_kb_index_info_by_record( + db=db, knowledge_base=knowledge_base, current_user_id=user.id + ) + delete_spec = RagRuntimeResolver().build_delete_runtime_spec( + db=db, + knowledge_base_id=knowledge_base.id, + # The index keys documents by their id rendered as a string, which is what + # KnowledgeService.delete_document also uses. + document_ref=str(document_id), + index_owner_user_id=index_info.index_owner_user_id, + ) + result = _run(get_delete_gateway().delete_document_index(delete_spec, db=db)) + status = (result or {}).get("status") + if status not in {"success", "deleted"}: + raise RuntimeError(f"index deletion returned status '{status}'") + + +def _enqueue_reindex( + db: Session, *, knowledge_base: Kind, user: User, document_id: int +) -> None: + """Queue a written page for indexing. + + A page stays invisible until its index succeeds — the state machine turns on + ``is_active`` and ``status`` together at that point — so this is what actually + publishes a page to readers. + """ + from app.services.knowledge.orchestrator import knowledge_orchestrator + + document = db.get(KnowledgeDocument, document_id) + if document is None: + return + knowledge_orchestrator._schedule_indexing_celery( + db=db, + knowledge_base=knowledge_base, + document=document, + user=user, + # Summaries are a knowledge base level concern and would be recomputed once + # per page here; the publish path refreshes them separately. + trigger_summary=False, + replace_active=True, + ) + + +def _run(awaitable): + """Run an awaitable from synchronous code.""" + from app.services.knowledge.knowledge_service import _run_async_in_new_loop + + return _run_async_in_new_loop(awaitable) diff --git a/backend/app/services/knowledge/code_wiki/source.py b/backend/app/services/knowledge/code_wiki/source.py new file mode 100644 index 0000000000..3171bb7d8f --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/source.py @@ -0,0 +1,354 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Source repository binding for code wikis, and the gate guarding it. + +A code wiki is derived from a repository, so its pages can expose whatever that +repository contains. Access is therefore split in two: + +- **Creating** a code wiki is gated here: the requester must be able to read the + repository. This stops someone from having a wiki built for a private repository + they cannot read themselves. +- **Reading** an existing code wiki is governed purely by knowledge-base permissions + (namespace roles, resource members, organization visibility). Re-checking the + repository on every read was slow and fragile, and it is not what decides who may + see a knowledge base. + +Leak protection therefore rests on this gate plus the namespace the creator picks, +which is the same trust model as pasting private code into any other knowledge base. +""" + +import logging +from dataclasses import dataclass +from typing import Any, Dict, Optional + +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from app.services.git_skill.utils import get_user_git_info, parse_repo_url + +logger = logging.getLogger(__name__) + +SUPPORTED_SOURCE_TYPES = ("github", "gitlab", "gitea") + + +class SourceAccessDenied(Exception): + """Raised when a user cannot read the repository they asked to document.""" + + +@dataclass(frozen=True) +class SourceRepository: + """The repository a code wiki is generated from. + + Every field except the type is derived from the URL rather than accepted + separately, so the repository this gate checks is necessarily the repository that + later gets cloned. Taking the domain and project name as independent inputs would + let a caller pass a repository they can read while storing a URL pointing + somewhere else entirely — the gate would approve one repository and the wiki would + be built from another. + """ + + source_type: str + # Credentials stripped: a URL may arrive with a token embedded, and this value is + # stored on the knowledge base where anyone who can read it would see them. + source_url: str + project_name: str + source_domain: str + + @classmethod + def from_url(cls, source_type: str, source_url: str) -> "SourceRepository": + """Build from a repository URL, deriving the domain and project name. + + Raises: + SourceAccessDenied: If the type is unsupported or the URL is unusable. + Both are refusals to bind a repository, so they surface the same way. + """ + if source_type not in SUPPORTED_SOURCE_TYPES: + raise SourceAccessDenied( + f"Unsupported repository type '{source_type}'. " + f"Supported types: {', '.join(SUPPORTED_SOURCE_TYPES)}" + ) + + try: + parsed = parse_repo_url(source_url) + except HTTPException as exc: + raise SourceAccessDenied( + f"Could not read a repository from '{source_url}': {exc.detail}" + ) from exc + except Exception as exc: + raise SourceAccessDenied( + f"Could not read a repository from '{source_url}'" + ) from exc + + if not parsed.domain or not parsed.owner or not parsed.repo: + raise SourceAccessDenied( + f"Could not read a host and project from '{source_url}'" + ) + + _assert_host_is_addressable(parsed.domain) + + repo = parsed.repo[:-4] if parsed.repo.endswith(".git") else parsed.repo + return cls( + source_type=source_type, + source_url=f"https://{parsed.domain}/{parsed.owner}/{repo}.git", + project_name=f"{parsed.owner}/{repo}", + source_domain=parsed.domain, + ) + + def to_spec(self) -> Dict[str, Any]: + """Render as the ``spec.source`` object stored on the knowledge base.""" + return { + "sourceType": self.source_type, + "sourceUrl": self.source_url, + "sourceDomain": self.source_domain, + "projectName": self.project_name, + } + + @classmethod + def from_spec( + cls, spec_source: Optional[Dict[str, Any]] + ) -> Optional["SourceRepository"]: + """Read back a stored source, or ``None`` when one was never recorded.""" + if not spec_source: + return None + return cls( + source_type=str(spec_source.get("sourceType", "")), + source_url=str(spec_source.get("sourceUrl", "")), + project_name=str(spec_source.get("projectName", "")), + source_domain=str(spec_source.get("sourceDomain", "")), + ) + + +def _assert_host_is_addressable(host: str) -> None: + """Refuse hosts that are never a Git server but are a useful SSRF target. + + Binding a repository makes the server fetch from whatever host the URL names, + carrying the caller's token, so the host is attacker-chosen input. The cloud + metadata endpoint sits on a link-local address and answers unauthenticated + requests with instance credentials — that is the one worth ruling out by name. + + Private ranges are deliberately still allowed: a self-hosted GitLab or Gitea on + an internal network is the normal deployment here, and blocking those would + break the product to close a much smaller hole. + + This bounds what a URL may *name*. It does not stop a public name that resolves + inward, which needs a check at connection time rather than here. + """ + import ipaddress + + if host in {"localhost", "localhost.localdomain"}: + raise SourceAccessDenied(f"'{host}' is not a reachable repository host") + + try: + address = ipaddress.ip_address(host) + except ValueError: + # A name rather than a literal; nothing more to check without resolving it. + return + + # ``is_unspecified`` is checked alongside the other two because 0.0.0.0 and :: + # are neither loopback nor link-local, and connecting to them reaches the local + # host anyway. Python already folds IPv4-mapped forms into these properties, so + # ::ffff:127.0.0.1 and ::ffff:169.254.169.254 are covered without special casing; + # the tests pin that, since it is a property of the standard library rather than + # of anything written here. + if address.is_loopback or address.is_link_local or address.is_unspecified: + raise SourceAccessDenied(f"'{host}' is not a reachable repository host") + + +def provider_for(source_type: str): + if source_type == "github": + from app.repository.github_provider import GitHubProvider + + return GitHubProvider() + if source_type == "gitlab": + from app.repository.gitlab_provider import GitLabProvider + + return GitLabProvider() + if source_type == "gitea": + from app.repository.gitea_provider import GiteaProvider + + return GiteaProvider() + return None + + +# Where "may write" starts, on the scale both providers are mapped onto: GitLab's +# Developer is 30, and GitHub's "push" is mapped to the same number. Anything below +# it can read the repository but not change it. +WRITE_ACCESS_LEVEL = 30 + + +def _check_access(provider, source_type: str, token: str, source: SourceRepository): + """Ask the provider whether the token can read the repository. + + GitLab names its parameter ``project_id`` but accepts a full project path, which + is what is used here: a caller-supplied numeric id could point at a different + project than the URL, which is the mismatch this module exists to rule out. + """ + if source_type == "gitlab": + return provider.check_user_project_access( + token=token, + git_domain=source.source_domain, + project_id=source.project_name, + ) + return provider.check_user_project_access( + token=token, + git_domain=source.source_domain, + repo_name=source.project_name, + ) + + +def _anonymous_read_access(source: SourceRepository) -> Dict[str, Any]: + """Read access granted by the repository being public, or a refusal. + + The access level reported is read and no more: nobody has write access to a + repository they reached without a credential, so this can never satisfy the + write gate. + """ + provider = provider_for(source.source_type) + described = ( + provider.describe_repository( + token="", + git_domain=source.source_domain, + repo_name=source.project_name, + ) + if provider is not None + else None + ) + if not described or described.get("visibility") != "public": + raise SourceAccessDenied( + f"No credentials configured for {source.source_domain}, and " + f"'{source.project_name}' is not readable without one. Add a token for " + "this Git domain, or check the repository address." + ) + + logger.info( + "[code_wiki] user reached public repository %s without a credential", + source.project_name, + ) + return { + "has_access": True, + "access_level": 10, + "access_level_name": "Read", + "visibility": "public", + } + + +def _or_public(source: SourceRepository, refusal: SourceAccessDenied) -> Dict[str, Any]: + """Fall back to reading the repository as anyone would, or raise ``refusal``. + + Raising the original refusal rather than the public probe's own wording keeps + the more specific answer: "your token does not reach this" is more use than + "this is not public". + + Answering a *successful* public probe is not the same as answering an absent + one — it is positive evidence that the repository is world-readable, obtained + from the provider we just failed to reach with a credential. So this does not + turn an unreachable provider into an open door: if the provider is down, the + probe fails too and the refusal stands. + """ + try: + return _anonymous_read_access(source) + except SourceAccessDenied: + raise refusal from None + + +def assert_user_can_read_source( + db: Session, user_id: int, source: SourceRepository +) -> Dict[str, Any]: + """Verify a user can read the repository, or raise ``SourceAccessDenied``. + + Returns the provider's access details so callers can log the granted level. + """ + if source.source_type not in SUPPORTED_SOURCE_TYPES: + raise SourceAccessDenied( + f"Unsupported repository type '{source.source_type}'. " + f"Supported types: {', '.join(SUPPORTED_SOURCE_TYPES)}" + ) + + git_info: Optional[Dict[str, Any]] = get_user_git_info( + user_id=user_id, domain=source.source_domain, db=db + ) + if not git_info or not git_info.get("token"): + # No credential is not the same as no access: a public repository is readable + # by anyone, and refusing here would make its wiki more closed than the + # repository. Asked anonymously, so an unreadable one is reported as such + # without disclosing whether it exists. + return _anonymous_read_access(source) + + # The credential's own type must agree with the declared one. Asking one provider + # about a repository hosted by another produces a meaningless answer, so require + # agreement rather than guessing which side is right. + configured_type = git_info.get("type") + if configured_type and configured_type != source.source_type: + raise SourceAccessDenied( + f"Credentials for {source.source_domain} are configured as " + f"'{configured_type}', but the repository was declared as " + f"'{source.source_type}'." + ) + + provider = provider_for(source.source_type) + if provider is None: + raise SourceAccessDenied(f"Unsupported repository type '{source.source_type}'") + + try: + result = _check_access(provider, source.source_type, git_info["token"], source) + except Exception as exc: + logger.warning( + "[code_wiki] repository access check failed for %s: %s", + source.project_name, + exc, + ) + # The provider's own error text is logged above but deliberately kept out of + # the message, which reaches the client as a 403 body. It is an external + # system's wording about our internal request, not an answer to the caller. + return _or_public( + source, + SourceAccessDenied( + f"Could not verify access to '{source.project_name}'. " + "Please try again later." + ), + ) + + if not result.get("has_access", False): + # A credential that does not reach this repository is not the last word: the + # membership check both providers use answers "not a member", which is also + # the answer for a public repository the caller has never joined. Falling + # through to the public probe is what stops a stale or unrelated token from + # making a world-readable repository undocumentable. + return _or_public( + source, + SourceAccessDenied( + f"You do not have read access to '{source.project_name}'. " + f"{result.get('error', '')}".strip() + ), + ) + + logger.info( + "[code_wiki] user %s has %s access to %s", + user_id, + result.get("access_level_name", "read"), + source.project_name, + ) + return result + + +def assert_user_can_write_source( + db: Session, user_id: int, source: SourceRepository +) -> Dict[str, Any]: + """Verify a user may change the repository, or raise ``SourceAccessDenied``. + + Triggering a run rewrites every page of the wiki, so it is closer to changing + the repository than to reading it. Read access is checked first and separately, + so that somebody who cannot see the repository at all is told that rather than + being told their permissions are merely insufficient. + """ + result = assert_user_can_read_source(db, user_id, source) + + level = result.get("access_level") + if not isinstance(level, int) or level < WRITE_ACCESS_LEVEL: + raise SourceAccessDenied( + f"You have read access to '{source.project_name}' but not write access, " + "which is what regenerating its wiki requires." + ) + return result diff --git a/backend/app/services/knowledge/code_wiki/version_store.py b/backend/app/services/knowledge/code_wiki/version_store.py new file mode 100644 index 0000000000..013d23cc58 --- /dev/null +++ b/backend/app/services/knowledge/code_wiki/version_store.py @@ -0,0 +1,337 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""The version store behind a code wiki. + +A generation run writes here, not into the knowledge base. The knowledge base is a +projection of whichever version is published, so everything a run does — including +crashing halfway through — stays invisible until a complete version passes its publish +gate. + +That containment is the point: it puts the atomicity boundary around a deterministic +projection measured in seconds instead of around an LLM run measured in hours. + +Three invariants hold here: + +- **A version is a complete snapshot.** An incremental run is seeded from the + published version and revises it in place, so the projection can always treat a + version as the whole truth and compute orphans as a plain set difference. +- **Seeding is the server's job.** The agent is told what changed and what pages + exist, but is never responsible for carrying unchanged pages forward. +- **The published version is never collected.** Retention would otherwise eat the + rollback baseline exactly when it is needed most. +""" + +import logging +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Optional, Sequence + +from sqlalchemy import func, insert, literal, select +from sqlalchemy.orm import Session + +from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.services.knowledge.code_wiki.page_path import ( + collation_key, + normalize_page_path, +) + +logger = logging.getLogger(__name__) + +# Key under which a page's stable path lives in ``WikiContent.ext``. +PATH_EXT_KEY = "path" + +# How long a run may sit in flight before it is treated as abandoned. Generous, because +# a large repository legitimately takes a long time; the cost of being wrong is one +# duplicated run, whereas never expiring costs a wiki that can never be regenerated +# again — the lock below would refuse every later run forever. +STALE_RUN_AFTER_HOURS = 6.0 + +IN_FLIGHT_STATUSES = ( + WikiGenerationStatus.PENDING, + WikiGenerationStatus.RUNNING, +) + +# Retention defaults. Successful versions are the rollback material; failed ones are +# kept only long enough to be looked at. +DEFAULT_KEEP_SUCCESSFUL = 10 +DEFAULT_MAX_AGE_DAYS = 90 +DEFAULT_FAILED_RETENTION_DAYS = 7 + + +def _utcnow() -> datetime: + """Return a timezone-naive UTC timestamp, matching the wiki tables.""" + return datetime.now(timezone.utc).replace(tzinfo=None) + + +def _as_naive_utc(value: Optional[datetime]) -> datetime: + """Normalize a caller-supplied instant to the naive UTC the wiki tables store. + + An aware value is converted before its offset is dropped. Dropping it outright + would shift the instant by that offset, and both callers compare the result + against stored timestamps to decide what to reclaim or delete. + """ + if value is None: + return _utcnow() + if value.tzinfo is None: + return value + return value.astimezone(timezone.utc).replace(tzinfo=None) + + +@dataclass(frozen=True) +class SeedOutcome: + """What seeding did.""" + + copied_pages: int + skipped_reason: str = "" + + @property + def seeded(self) -> bool: + return not self.skipped_reason + + +def page_path_of(content: WikiContent) -> str: + """Return the stable path recorded on a version entry, or an empty string.""" + ext = content.ext or {} + return str(ext.get(PATH_EXT_KEY, "") or "") + + +def set_page_path(content: WikiContent, path: str) -> None: + """Record a normalized path on a version entry.""" + ext = dict(content.ext or {}) + ext[PATH_EXT_KEY] = path + content.ext = ext + + +def seed_from_published( + db: Session, + *, + target_generation_id: int, + published_generation_id: int, +) -> SeedOutcome: + """Copy the published version's pages into a new generation. + + Runs before the agent starts, so an incremental run revises a complete snapshot + rather than producing a partial one. A run that then fails leaves the seed as + unreferenced rows under a failed generation, which retention collects. + + Idempotent: a generation that already holds pages is left alone, so a retried + scheduling attempt cannot double the version. + """ + if published_generation_id <= 0: + return SeedOutcome(0, skipped_reason="no published version to seed from") + + if target_generation_id == published_generation_id: + return SeedOutcome(0, skipped_reason="target is the published version") + + existing = ( + db.query(func.count(WikiContent.id)) + .filter(WikiContent.generation_id == target_generation_id) + .scalar() + ) + if existing: + return SeedOutcome(0, skipped_reason="generation already holds pages") + + now = _utcnow() + # parent_id is deliberately not carried over: it refers to row ids inside the + # source generation, so copying it would point every seeded page at a row in a + # different version. Hierarchy comes from the page path, which is copied intact. + source = select( + literal(target_generation_id), + WikiContent.type, + WikiContent.title, + WikiContent.content, + literal(0), + WikiContent.ext, + literal(now), + literal(now), + ).where(WikiContent.generation_id == published_generation_id) + + db.execute( + insert(WikiContent).from_select( + [ + "generation_id", + "type", + "title", + "content", + "parent_id", + "ext", + "created_at", + "updated_at", + ], + source, + ) + ) + db.flush() + + copied = ( + db.query(func.count(WikiContent.id)) + .filter(WikiContent.generation_id == target_generation_id) + .scalar() + or 0 + ) + logger.info( + "[code_wiki] seeded generation %s with %s pages from %s", + target_generation_id, + copied, + published_generation_id, + ) + return SeedOutcome(copied) + + +def remove_page(db: Session, *, generation_id: int, path: str) -> bool: + """Drop a page from an in-flight version at the agent's request. + + Only the agent knows which page covered a module that no longer exists: phase one + records no provenance, so the server sees changed file paths but cannot infer the + page they belong to. The risk this creates is contained rather than avoided — the + removal lands in an unpublished version, the previous version still has the page, + and the publish gate checks how far the page count dropped. + + Returns: + Whether a page was removed. + """ + normalized = normalize_page_path(path) + wanted = collation_key(normalized) + + for content in ( + db.query(WikiContent).filter(WikiContent.generation_id == generation_id).all() + ): + if collation_key(page_path_of(content)) == wanted: + db.delete(content) + db.flush() + logger.info( + "[code_wiki] removed page '%s' from generation %s", + normalized, + generation_id, + ) + return True + return False + + +def reclaim_stale_generations( + db: Session, + *, + kind_id: int, + now: Optional[datetime] = None, + stale_after_hours: float = STALE_RUN_AFTER_HOURS, +) -> tuple[int, ...]: + """Fail in-flight generations whose worker is gone. + + Without this a single lost worker blocks the wiki permanently: scheduling takes + the generation row's lock and refuses to start while one is in flight, and a + crashed run never leaves that state on its own. + + Returns: + Ids of the generations that were failed. + """ + cutoff = _as_naive_utc(now) - timedelta(hours=stale_after_hours) + + stale: Sequence[WikiGeneration] = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == kind_id, + WikiGeneration.status.in_(IN_FLIGHT_STATUSES), + WikiGeneration.updated_at < cutoff, + ) + .all() + ) + if not stale: + return () + + for generation in stale: + generation.status = WikiGenerationStatus.FAILED + generation.completed_at = _as_naive_utc(now) + db.flush() + + reclaimed = tuple(generation.id for generation in stale) + logger.warning( + "[code_wiki] reclaimed %s abandoned generation(s) for kb %s: %s", + len(reclaimed), + kind_id, + reclaimed, + ) + return reclaimed + + +def apply_retention( + db: Session, + *, + kind_id: int, + published_generation_id: int, + keep_successful: int = DEFAULT_KEEP_SUCCESSFUL, + max_age_days: int = DEFAULT_MAX_AGE_DAYS, + failed_retention_days: int = DEFAULT_FAILED_RETENTION_DAYS, + now: Optional[datetime] = None, +) -> tuple[int, ...]: + """Collect versions that are no longer worth keeping. + + The published version is exempt unconditionally. Two ordinary situations would + otherwise collect it and leave the wiki with no version to roll back to: a run of + consecutive failures pushing it out of the newest ``keep_successful``, and a + repository quiet for long enough that every version ages out. + + Returns: + Ids of the generations that were deleted. + """ + reference = _as_naive_utc(now) + doomed: list[int] = [] + + successful = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == kind_id, + WikiGeneration.status == WikiGenerationStatus.COMPLETED, + ) + .order_by(WikiGeneration.created_at.desc(), WikiGeneration.id.desc()) + .all() + ) + age_cutoff = reference - timedelta(days=max_age_days) + for position, generation in enumerate(successful): + if generation.id == published_generation_id: + continue + over_count = position >= keep_successful + over_age = (generation.created_at or reference) < age_cutoff + if over_count or over_age: + doomed.append(generation.id) + + # Only terminal generations are collected. Excluding COMPLETED alone would sweep + # up PENDING and RUNNING rows once they are old enough, deleting the pages of a + # run still being written to. Reclamation normally fails an abandoned run long + # before that, but retention must not depend on it having done so. + failed_cutoff = reference - timedelta(days=failed_retention_days) + unsuccessful = ( + db.query(WikiGeneration) + .filter( + WikiGeneration.kind_id == kind_id, + WikiGeneration.status.notin_( + [WikiGenerationStatus.COMPLETED, *IN_FLIGHT_STATUSES] + ), + WikiGeneration.created_at < failed_cutoff, + ) + .all() + ) + for generation in unsuccessful: + if generation.id != published_generation_id: + doomed.append(generation.id) + + if not doomed: + return () + + # Contents are removed explicitly rather than through the foreign key, because + # SQLite does not enforce ON DELETE CASCADE unless asked to and the tests run there. + db.query(WikiContent).filter(WikiContent.generation_id.in_(doomed)).delete( + synchronize_session=False + ) + db.query(WikiGeneration).filter(WikiGeneration.id.in_(doomed)).delete( + synchronize_session=False + ) + db.flush() + + logger.info( + "[code_wiki] retention removed %s version(s) from kb %s", + len(doomed), + kind_id, + ) + return tuple(doomed) diff --git a/backend/app/services/knowledge/content_scope.py b/backend/app/services/knowledge/content_scope.py new file mode 100644 index 0000000000..7a6ed3519e --- /dev/null +++ b/backend/app/services/knowledge/content_scope.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Query scopes for knowledge content, separated by ownership and target kind. + +A knowledge base can hold content that serves different purposes: + +- **Wiki pages** — documents a reader browses in the folder tree. +- **Code targets** (planned) — one hidden target per indexed source file. These are + retrieval artifacts, not pages: they carry no folder, never appear in the page + tree, and are only reachable through retrieval. + +Every read that means "the pages of this wiki" must exclude code targets, and every +write or cleanup driven by generation must be limited to agent-owned content. Leaving +that to each call site has already proven unreliable — the filter was missed twice +during design review, once for folder listing and once for the generated-content sweep, +where it would have deleted the whole code index. + +So the filters live here instead of at the call sites. Callers pick a scope by name +and cannot express "no filter" by accident; reaching code targets requires asking for +them explicitly. +""" + +from sqlalchemy import or_ +from sqlalchemy.orm import Query + +from app.models.kind import Kind +from app.models.knowledge import ( + ContentOrigin, + DocumentSourceType, + KnowledgeDocument, + KnowledgeFolder, +) +from app.schemas.knowledge import KnowledgeBaseType + +# Source type marking a document as an indexed source file rather than a wiki page. +CODE_TARGET_SOURCE_TYPE = DocumentSourceType.CODE.value + +# folder_id for targets that are deliberately outside the folder tree. Distinct from +# 0 (root level) so that listing a folder's children — including the root's — cannot +# return them even if a caller bypasses these scopes. +NO_FOLDER = -1 + + +def wiki_pages(query: Query) -> Query: + """Restrict a ``KnowledgeDocument`` query to browsable wiki pages. + + Excludes code targets. Use for the folder tree, document listing, page + navigation, and anything else a reader sees. + """ + return query.filter(KnowledgeDocument.source_type != CODE_TARGET_SOURCE_TYPE) + + +def generated_wiki_pages(query: Query) -> Query: + """Restrict a ``KnowledgeDocument`` query to agent-owned wiki pages. + + This is the scope the projection owns: the only documents it may create, update or + delete. It excludes both user-owned content — which is not regenerable, so a + mistaken delete is unrecoverable — and code targets, which no wiki version + produces and which a set difference would therefore treat as orphans. + """ + return wiki_pages(query).filter( + KnowledgeDocument.origin == ContentOrigin.GENERATED.value + ) + + +def generated_folders(query: Query) -> Query: + """Restrict a ``KnowledgeFolder`` query to agent-owned folders.""" + return query.filter(KnowledgeFolder.origin == ContentOrigin.GENERATED.value) + + +def code_targets(query: Query) -> Query: + """Restrict a ``KnowledgeDocument`` query to indexed source files. + + Separate from every reader-facing scope, so retrieval and index maintenance have + to name code targets deliberately. + """ + return query.filter(KnowledgeDocument.source_type == CODE_TARGET_SOURCE_TYPE) + + +def only_code_wikis(query: Query) -> Query: + """A ``Kind`` query restricted to code wikis. + + For listings that render repository fields, not for access control: a code wiki + is visible exactly as far as its own ACL says, the same as any other knowledge + base. There is deliberately no ``exclude_code_wikis`` counterpart — hiding them + from the general listing would also hide them from chat and the MCP tool, where + being citable is the point. + """ + return query.filter( + Kind.json["spec"]["kbType"].as_string() == KnowledgeBaseType.CODE_WIKI.value + ) diff --git a/backend/app/services/knowledge/knowledge_service.py b/backend/app/services/knowledge/knowledge_service.py index 123e4accd0..66544ce21e 100644 --- a/backend/app/services/knowledge/knowledge_service.py +++ b/backend/app/services/knowledge/knowledge_service.py @@ -44,6 +44,7 @@ BatchOperationResult, KnowledgeBaseCreate, KnowledgeBaseResponse, + KnowledgeBaseType, KnowledgeBaseUpdate, KnowledgeBaseWithGroupInfo, KnowledgeDocumentCreate, @@ -59,6 +60,7 @@ get_user_groups, get_view_role_in_group, ) +from app.services.knowledge.content_scope import wiki_pages from app.services.knowledge.folder_policy import assert_document_can_be_placed_in_folder from app.services.knowledge.knowledge_access_policy import ( can_directly_access_knowledge_base as evaluate_direct_knowledge_base_access, @@ -292,11 +294,18 @@ def create_knowledge_base( "name": data.name, "description": data.description or "", "directAccessRequirement": data.direct_access_requirement, - "kbType": data.kb_type - or "notebook", # Default to 'notebook' if not provided + # A code wiki is fixed here; there is deliberately no code path that turns + # an existing knowledge base into one, or out of one. + "kbType": KnowledgeBaseType( + data.kb_type or KnowledgeBaseType.NOTEBOOK + ).value, "retrievalConfig": _to_json_dict(data.retrieval_config), "summaryEnabled": data.summary_enabled, } + # A code wiki records the repository it is generated from + if data.source: + spec_kwargs["source"] = data.source + # Add summaryModelRef if provided if data.summary_model_ref: spec_kwargs["summaryModelRef"] = data.summary_model_ref @@ -1040,7 +1049,15 @@ def update_knowledge_base_type( # Get current default view kb_json = kb.json spec = kb_json.get("spec", {}) - current_type = spec.get("kbType", "notebook") + + current_type = spec.get("kbType", KnowledgeBaseType.NOTEBOOK.value) + + # A code wiki is a repository binding, not a view preference, so the toggle does + # not apply. Rejecting it here also keeps this endpoint from being used to + # reinterpret a code wiki as an ordinary knowledge base, which would orphan its + # repository and version history. + if current_type == KnowledgeBaseType.CODE_WIKI.value: + raise ValueError("Cannot change the opening view of a code wiki") # If same type, return current kb if current_type == new_type: @@ -1472,8 +1489,12 @@ def list_documents( if not kb or not has_access: return [] - query = db.query(KnowledgeDocument).filter( - KnowledgeDocument.kind_id == knowledge_base_id, + # Scoped to browsable pages: code targets are retrieval artifacts and must + # never surface in a listing. See content_scope. + query = wiki_pages( + db.query(KnowledgeDocument).filter( + KnowledgeDocument.kind_id == knowledge_base_id, + ) ) if folder_id is not None: @@ -1502,8 +1523,11 @@ def list_documents_paginated( if not kb or not has_access: return [], 0 - query = db.query(KnowledgeDocument).filter( - KnowledgeDocument.kind_id == knowledge_base_id, + # Scoped to browsable pages; see list_documents. + query = wiki_pages( + db.query(KnowledgeDocument).filter( + KnowledgeDocument.kind_id == knowledge_base_id, + ) ) if folder_id is not None: diff --git a/backend/app/services/knowledge/orchestrator.py b/backend/app/services/knowledge/orchestrator.py index 77abcd3825..8f5e760a24 100644 --- a/backend/app/services/knowledge/orchestrator.py +++ b/backend/app/services/knowledge/orchestrator.py @@ -36,11 +36,13 @@ KnowledgeBaseCreate, KnowledgeBaseListResponse, KnowledgeBaseResponse, + KnowledgeBaseType, KnowledgeDocumentCreate, KnowledgeDocumentListResponse, KnowledgeDocumentResponse, ResourceScope, ) +from app.services.knowledge.code_wiki.source import SourceRepository from app.services.knowledge.document_read_service import ( DOCUMENT_READ_ERROR_NOT_FOUND, document_read_service, @@ -1066,7 +1068,8 @@ def create_knowledge_base( description: Optional[str] = None, namespace: str = "default", direct_access_requirement: Literal["read", "edit"] = "read", - kb_type: str = "notebook", + kb_type: str = KnowledgeBaseType.NOTEBOOK.value, + source: Optional[SourceRepository] = None, summary_enabled: bool = False, rag_config_mode: Literal["auto", "disabled"] = "auto", # REST API scenario: pass complete config @@ -1191,7 +1194,8 @@ def create_knowledge_base( description=description, namespace=namespace, direct_access_requirement=direct_access_requirement, - kb_type=kb_type, + kb_type=KnowledgeBaseType(kb_type), + source=source.to_spec() if source else None, retrieval_config=resolved_retrieval_config, summary_enabled=summary_enabled, summary_model_ref=resolved_summary_model_ref, diff --git a/backend/app/services/wiki_service.py b/backend/app/services/wiki_service.py index 6db519d0aa..e491963fde 100644 --- a/backend/app/services/wiki_service.py +++ b/backend/app/services/wiki_service.py @@ -23,12 +23,26 @@ ) from app.schemas.task import TaskCreate from app.schemas.wiki import ( + WikiContentSummary, WikiContentWriteRequest, WikiGenerationCreate, + WikiPageRead, WikiProjectCreate, ) from app.services.adapters.task_kinds import task_kinds_service from app.services.adapters.team_kinds import team_kinds_service +from app.services.knowledge.code_wiki.page_path import ( + InvalidPagePath, + assert_unique_within_version, + collation_key, + normalize_page_path, +) +from app.services.knowledge.code_wiki.runner import finish_run, is_code_wiki_generation +from app.services.knowledge.code_wiki.version_store import ( + page_path_of, + remove_page, + set_page_path, +) from app.services.user import user_service from shared.utils.url_util import domains_match @@ -562,10 +576,10 @@ def save_generation_contents( - Resilient writes regardless of current generation status so reruns can overwrite results """ has_sections = bool(payload.sections) - if not has_sections and not payload.summary: + if not has_sections and not payload.summary and not payload.removed_paths: raise HTTPException( status_code=400, - detail="No sections or summary provided", + detail="No sections, removals or summary provided", ) total_payload_size = ( @@ -595,34 +609,72 @@ def save_generation_contents( existing_contents: List[WikiContent] = [] if has_sections: + # Pages are identified by path when one is given. Normalising up front + # means a malformed path fails this write rather than the publish of the + # whole version, and keeps two spellings of one path from becoming two + # pages that the projection could not both honour. + normalized_paths: Dict[int, str] = {} + try: + for index, section in enumerate(payload.sections): + if section.path: + normalized_paths[index] = normalize_page_path(section.path) + assert_unique_within_version(normalized_paths.values()) + except InvalidPagePath as exc: + # The generation row is held with_for_update; release it before + # raising, as the other failure branches in this method do. + wiki_db.rollback() + raise HTTPException(status_code=400, detail=str(exc)) from exc + titles = [section.title for section in payload.sections] + # Everything in the generation is loaded, not just the titles being + # written: a page whose title changed has to be found by its path. existing_contents = ( wiki_db.query(WikiContent) - .filter( - WikiContent.generation_id == generation.id, - WikiContent.title.in_(titles), - ) + .filter(WikiContent.generation_id == generation.id) .with_for_update() .all() ) + existing_by_path: Dict[str, WikiContent] = {} + path_less: List[WikiContent] = [] + for content in existing_contents: + content_path = page_path_of(content) + if content_path: + existing_by_path[collation_key(content_path)] = content + else: + path_less.append(content) + + # The title-based indices deliberately exclude anything that already has a + # page path. Moving a page keeps its title, so a title can name several + # path-identified pages; a legacy write resolving through it would pick one + # of them by query order rather than by any rule, and overwrite it. existing_by_key: Dict[Tuple[str, str], WikiContent] = { - (content.type, content.title): content for content in existing_contents + (content.type, content.title): content for content in path_less } existing_by_title: Dict[str, WikiContent] = { - content.title: content for content in existing_contents + content.title: content + for content in path_less + if content.title in titles } - for section in payload.sections: - content_item = existing_by_key.get( - (section.type, section.title) - ) or existing_by_title.get(section.title) + for index, section in enumerate(payload.sections): + path = normalized_paths.get(index) + if path: + content_item = existing_by_path.get(collation_key(path)) + else: + # Legacy write path: no page identity was supplied, so fall back + # to matching on the title as this API originally did. + content_item = existing_by_key.get( + (section.type, section.title) + ) or existing_by_title.get(section.title) if content_item: content_item.type = section.type content_item.title = section.title content_item.content = section.content content_item.ext = section.ext or None + if path: + set_page_path(content_item, path) content_item.updated_at = now updated_sections += 1 else: @@ -638,6 +690,9 @@ def save_generation_contents( created_at=now, updated_at=now, ) + if path: + set_page_path(content_record, path) + existing_by_path[collation_key(path)] = content_record wiki_db.add(content_record) created_sections += 1 @@ -654,6 +709,10 @@ def save_generation_contents( status_code=400, detail="Failed to persist wiki contents" ) + # Applied after the writes, so that a payload both writing and removing a path + # ends with it removed regardless of the order the agent listed them in. + removed_paths = self._apply_removals(wiki_db, generation, payload.removed_paths) + summary = payload.summary previous_status = generation.status ext = generation.ext.copy() if isinstance(generation.ext, dict) else {} @@ -662,6 +721,8 @@ def save_generation_contents( content_meta["last_write_titles"] = titles content_meta["created_sections"] = created_sections content_meta["updated_sections"] = updated_sections + if removed_paths: + content_meta["removed_paths"] = removed_paths content_meta["status_before_write"] = ( previous_status.value if isinstance(previous_status, WikiGenerationStatus) @@ -685,6 +746,12 @@ def save_generation_contents( generation.ext = ext generation.updated_at = now + # A code wiki does not simply record its outcome: a successful version has to + # pass the publish gate and be projected into the knowledge base, and that runs + # after this write is committed rather than inside it. Deferring the status too + # keeps the two from disagreeing if the projection is refused. + finishes_a_code_wiki = False + if summary and summary.status: try: status_enum = WikiGenerationStatus(summary.status) @@ -698,8 +765,12 @@ def save_generation_contents( status_code=400, detail=f"Unsupported summary status: {summary.status}", ) from exc - generation.status = status_enum - if status_enum in { + finishes_a_code_wiki = is_code_wiki_generation(wiki_db, generation) + if finishes_a_code_wiki: + generation.status = WikiGenerationStatus.RUNNING + else: + generation.status = status_enum + if not finishes_a_code_wiki and status_enum in { WikiGenerationStatus.COMPLETED, WikiGenerationStatus.FAILED, WikiGenerationStatus.CANCELLED, @@ -748,6 +819,106 @@ def save_generation_contents( content_meta.get("status_after_write"), ) + if finishes_a_code_wiki: + self._finish_code_wiki(wiki_db, generation, summary) + + def get_generation_page( + self, + wiki_db: Session, + generation_id: int, + path: str, + ) -> Optional[WikiPageRead]: + """Read one page of a version by its path. + + Matched the same way a write is — normalised, then compared case-insensitively + — so that a path which would update a page also reads it. Resolving them + differently would let the agent read one page and overwrite another. + + Returns: + The page, or ``None`` when the version holds none at that path. + """ + try: + normalized = normalize_page_path(path) + except InvalidPagePath as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + wanted = collation_key(normalized) + for content in ( + wiki_db.query(WikiContent) + .filter(WikiContent.generation_id == generation_id) + .all() + ): + content_path = page_path_of(content) + if content_path and collation_key(content_path) == wanted: + return WikiPageRead( + path=content_path, + title=content.title, + content=content.content, + ) + return None + + def _apply_removals( + self, + wiki_db: Session, + generation: WikiGeneration, + paths: List[str], + ) -> List[str]: + """Drop pages the agent declared gone, returning the ones that existed. + + A path that names no page is reported rather than refused: an agent listing a + page it already removed on a retry has nothing to correct, and failing the + whole write would lose the sections alongside it. + """ + removed: List[str] = [] + for raw_path in paths: + try: + normalized = normalize_page_path(raw_path) + except InvalidPagePath as exc: + wiki_db.rollback() + raise HTTPException(status_code=400, detail=str(exc)) from exc + if remove_page(wiki_db, generation_id=generation.id, path=normalized): + removed.append(normalized) + else: + logger.info( + "[wiki] removal of '%s' from generation %s matched no page", + normalized, + generation.id, + ) + return removed + + def _finish_code_wiki( + self, + wiki_db: Session, + generation: WikiGeneration, + summary: Optional[WikiContentSummary], + ) -> None: + """Conclude a code wiki run once its final write is safely committed. + + Failures here are reported to the agent as a 5xx rather than swallowed. The + version is committed either way, so a retried submission republishes it; a + silent failure would leave a run stuck RUNNING with content nobody projects. + """ + succeeded = summary is not None and summary.status == "COMPLETED" + try: + finish_run( + wiki_db, + generation=generation, + succeeded=succeeded, + error_message=(summary.error_message if summary else "") or "", + head_commit=(summary.head_commit if summary else "") or "", + ) + except Exception as exc: + wiki_db.rollback() + logger.error( + "[wiki] failed to conclude code wiki generation %s: %s", + generation.id, + exc, + ) + raise HTTPException( + status_code=500, + detail=f"Failed to publish the wiki version: {exc}", + ) from exc + def get_generations( self, db: Session, diff --git a/backend/init_data/02-public-resources.yaml b/backend/init_data/02-public-resources.yaml index eff96d9635..a4f6ed109c 100644 --- a/backend/init_data/02-public-resources.yaml +++ b/backend/init_data/02-public-resources.yaml @@ -328,6 +328,110 @@ status: --- apiVersion: agent.wecode.io/v1 kind: Ghost +metadata: + name: code-wiki-ghost + namespace: default +spec: + mcpServers: {} + skills: + - wiki_submit + systemPrompt: | + You write and maintain code wikis for a knowledge base. Each run is given a + repository, a generation id and a set of instructions; this prompt covers only how + to hand your work back, which is the same every time. + + ## Submitting + + Use the wiki_submit skill (wiki_submit.js). Submit each page as you finish it + rather than batching them at the end: a run that stops early then still leaves + behind the pages it completed. + + ```bash + node wiki_submit.js submit \ + --generation-id \ + --path architecture/backend \ + --title "Backend Architecture" \ + --file ./page.md + ``` + + ## Paths are identities + + `--path` is what identifies a page: `index`, `architecture/backend`, + `modules/indexing`. Lowercase, `/`-separated, no file extension, at most 4 folders + deep, and two paths may not differ only by case. + + Keep a page's path stable across runs. It is what lets an unchanged page keep its + place, its links and its search index. Changing it republishes the page as a + deletion plus an insertion, so reword titles freely and move paths rarely. + + Send a page's complete content every time. There is no patch format; what you send + replaces the page. + + ## Removing + + Only an incremental run removes pages by declaring them, because its version starts + as a copy of the published wiki: + + ```bash + node wiki_submit.js remove --generation-id --path modules/legacy-sync + ``` + + In a full rebuild the version starts empty, so a page you do not write is already + absent and declaring it would do nothing. + + ## Finishing + + Always finish the run, in success or failure. Nothing is published until you do, + and a run left unreported blocks the wiki until it is reclaimed hours later. + + ```bash + node wiki_submit.js complete --generation-id --head-commit "$(git rev-parse HEAD)" + node wiki_submit.js fail --generation-id --error-message "why it failed" + ``` + + Report the commit you actually documented. The next run compares against it to + decide what has changed, so a missing or wrong value costs a needless full rebuild + or, worse, skips changes nobody has written up. +status: + state: Available +--- +apiVersion: agent.wecode.io/v1 +kind: Bot +metadata: + name: code-wiki-bot + namespace: default +spec: + ghostRef: + name: code-wiki-ghost + namespace: default + shellRef: + name: ClaudeCode + namespace: default +status: + state: Available +--- +apiVersion: agent.wecode.io/v1 +kind: Team +metadata: + name: code-wiki-team + namespace: default +spec: + description: Generates and maintains a knowledge base code wiki from a repository. + members: + - role: leader + botRef: + name: code-wiki-bot + namespace: default + prompt: "" + collaborationModel: solo + bind_mode: [] + workflow: + mode: solo +status: + state: Available +--- +apiVersion: agent.wecode.io/v1 +kind: Ghost metadata: name: chat-ghost namespace: default diff --git a/backend/init_data/skills/wiki_submit/SKILL.md b/backend/init_data/skills/wiki_submit/SKILL.md index bc4ead8119..c6620d9d83 100644 --- a/backend/init_data/skills/wiki_submit/SKILL.md +++ b/backend/init_data/skills/wiki_submit/SKILL.md @@ -1,6 +1,6 @@ --- -description: "Submit wiki documentation sections to Wegent backend API. Simplifies the HTTP POST process for wiki content submission." -version: "1.1.0" +description: "Submit wiki documentation pages to Wegent backend API. Simplifies the HTTP POST process for wiki content submission." +version: "1.2.0" author: "Wegent Team" tags: ["wiki", "documentation", "api", "submission"] bindShells: ["ClaudeCode"] @@ -8,37 +8,80 @@ bindShells: ["ClaudeCode"] # Wiki Submit Skill -This skill provides a simple command-line tool to submit wiki documentation sections to the Wegent backend. +This skill provides a simple command-line tool to submit wiki documentation pages to the Wegent backend. -## Usage +## Page paths + +A page is identified by its `--path`: `index`, `architecture/backend`, `modules/indexing`. +Lowercase, `/`-separated, no file extension, at most 4 folders deep. Two paths may not +differ only by case — the projection matches them case-insensitively, so they would +collapse into one page. + +**Keep a path stable across runs.** It is what lets an unchanged page keep its place, +its links and its search index. Changing it republishes the page as a deletion plus an +insertion, so reword titles freely and move paths rarely. +Send a page's **complete content** every time. There is no patch format; what you send +replaces the page. + +## Usage -### Submit a single section from a markdown file +### Submit a page from a markdown file ```bash node wiki_submit.js submit \ --generation-id 123 \ - --type overview \ - --title "Project Overview" \ - --file /path/to/overview.md + --path architecture/backend \ + --title "Backend Architecture" \ + --file /path/to/page.md ``` -### Submit section content directly +### Submit page content directly + +Note the `$'...'` quoting: in a plain double-quoted string `\n` stays a backslash and +an `n`, and the page arrives as one long line. For anything beyond a few lines, write +the markdown to a file and use `--file`. ```bash node wiki_submit.js submit \ --generation-id 123 \ - --type architecture \ - --title "System Architecture" \ - --content "# Architecture\n\nYour markdown content here..." + --path index \ + --title "Overview" \ + --content $'# Overview\n\nYour markdown content here...' +``` + +### Read what a page currently says + +Only your own generation is readable, which in an incremental run is a complete copy +of the published wiki — so this is how you see a page before revising it. + +```bash +node wiki_submit.js read --generation-id 123 --path architecture/backend > current.md +``` + +Exits 0 with no output when the page does not exist yet. In an incremental run that +means the page is new. + +### Remove pages that no longer have a subject + +Only meaningful in an incremental run, where your version starts as a copy of the +published wiki and not writing a page therefore does *not* remove it. + +```bash +node wiki_submit.js remove \ + --generation-id 123 \ + --path modules/legacy-sync \ + --path guides/old-setup ``` ### Complete the wiki generation +Report the commit you documented, so the next run knows what has already been covered. + ```bash node wiki_submit.js complete \ --generation-id 123 \ - --structure-order "overview: Project Overview" "architecture: System Architecture" "module: Core Modules" + --head-commit "$(git rev-parse HEAD)" ``` ### Mark generation as failed @@ -49,14 +92,12 @@ node wiki_submit.js fail \ --error-message "Failed to analyze repository structure" ``` -## Section Types +## Section types + +`--type` defaults to `chapter` and can be left out. The legacy wiki used it to group +pages; a code wiki organises them by path instead. -- `overview`: Project overview and objectives -- `architecture`: System architecture and design -- `module`: Module documentation -- `api`: API documentation -- `guide`: User guides and tutorials -- `deep`: In-depth technical analysis +Accepted values: `overview`, `architecture`, `module`, `api`, `guide`, `deep`, `chapter`. ## Authentication diff --git a/backend/init_data/skills/wiki_submit/wiki_submit.js b/backend/init_data/skills/wiki_submit/wiki_submit.js index 5b4c96492a..8d27abdab1 100644 --- a/backend/init_data/skills/wiki_submit/wiki_submit.js +++ b/backend/init_data/skills/wiki_submit/wiki_submit.js @@ -148,6 +148,34 @@ function makeRequest(url, options, body) { }) } +/** + * Read one page of the generation being written. + * @param {string} endpoint - Write endpoint URL, used to derive the read URL + * @param {string} token - Authorization token + * @param {number} generationId - Wiki generation ID + * @param {string} pagePath - Stable page path + * @returns {Promise} + */ +async function readPage(endpoint, token, generationId, pagePath) { + // The read endpoint sits beside the write one under /generations. A custom + // endpoint that does not end in that suffix would make this a no-op and build a + // wrong URL, whose 404 would then read as "page does not exist". + const suffix = /\/generations\/contents\/?$/ + if (!suffix.test(endpoint)) { + console.error(`Error: cannot derive the read URL from endpoint '${endpoint}'.`) + console.error("It must end in '/generations/contents'.") + process.exit(1) + } + const base = endpoint.replace(suffix, '') + const url = `${base}/generations/${generationId}/pages?path=${encodeURIComponent(pagePath)}` + + return makeRequest( + url, + { method: 'GET', headers: { Authorization: `Bearer ${token}` } }, + null + ) +} + /** * Submit wiki sections to the backend API. * @param {string} endpoint - API endpoint URL @@ -157,7 +185,7 @@ function makeRequest(url, options, body) { * @param {object|null} summary - Optional summary for completion * @returns {Promise} */ -async function submitSections(endpoint, token, generationId, sections, summary = null) { +async function submitSections(endpoint, token, generationId, sections, summary = null, removedPaths = []) { const headers = { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', @@ -172,6 +200,10 @@ async function submitSections(endpoint, token, generationId, sections, summary = payload.summary = summary } + if (removedPaths && removedPaths.length > 0) { + payload.removed_paths = removedPaths + } + return makeRequest(endpoint, { method: 'POST', headers }, JSON.stringify(payload)) } @@ -210,11 +242,17 @@ async function cmdSubmit(args) { } const section = { - type: args.type, + // Defaults to a plain chapter: a code wiki identifies pages by path, so the + // section type carries no meaning there and asking for one every time is noise. + type: args.type || 'chapter', title: args.title, content: content, } + if (args.path) { + section.path = args.path + } + if (args.ext) { try { section.ext = JSON.parse(args.ext) @@ -231,7 +269,82 @@ async function cmdSubmit(args) { return 1 } - console.log(`✅ Section '${args.title}' submitted successfully`) + console.log(`✅ Page '${args.path || args.title}' submitted successfully`) + return 0 +} + +/** + * Handle read command: print a page's current content. + * @param {object} args - Command arguments + * @returns {Promise} + */ +async function cmdRead(args) { + const endpoint = getWikiEndpoint(args.endpoint) + const token = getAuthToken(args.token) + if (!token) { + console.error('Error: Authorization token is required. It can be obtained from TASK_INFO, WIKI_TOKEN env var, or --token argument.') + process.exit(1) + } + if (!args.generationId) { + console.error('Error: --generation-id is required.') + process.exit(1) + } + if (!args.path) { + console.error('Error: --path is required for read command') + return 1 + } + const generationId = parseInt(args.generationId, 10) + + const result = await readPage(endpoint, token, generationId, args.path) + + if (result.status === 'error') { + // A page that does not exist yet is an answer, not a failure: in an incremental + // run it means this page is new. Reported on stderr so it cannot be mistaken + // for content when stdout is redirected to a file. + // The backend's phrase, not a bare status code: any other 404 means something + // is misconfigured, and calling that "a new page" hides it. + if (/has no page at/i.test(result.message || '')) { + console.error(`Page '${args.path}' does not exist yet`) + return 0 + } + console.error(`❌ Error: ${result.message}`) + return 1 + } + + process.stdout.write(result.content || '') + return 0 +} + +/** + * Handle remove command: declare pages as gone. + * @param {object} args - Command arguments + * @returns {Promise} + */ +async function cmdRemove(args) { + const endpoint = getWikiEndpoint(args.endpoint) + const token = getAuthToken(args.token) + if (!token) { + console.error('Error: Authorization token is required. It can be obtained from TASK_INFO, WIKI_TOKEN env var, or --token argument.') + process.exit(1) + } + if (!args.generationId) { + console.error('Error: --generation-id is required.') + process.exit(1) + } + if (!args.paths.length) { + console.error('Error: --path is required for remove command') + return 1 + } + const generationId = parseInt(args.generationId, 10) + + const result = await submitSections(endpoint, token, generationId, [], null, args.paths) + + if (result.status === 'error') { + console.error(`❌ Error: ${result.message}`) + return 1 + } + + console.log(`✅ Removed ${args.paths.length} page(s): ${args.paths.join(', ')}`) return 0 } @@ -258,6 +371,9 @@ async function cmdComplete(args) { structure_order: args.structureOrder || [], } + if (args.headCommit) { + summary.head_commit = args.headCommit + } if (args.model) { summary.model = args.model } @@ -323,6 +439,8 @@ function parseArgs(argv) { generationId: null, type: null, title: null, + path: null, + paths: [], file: null, content: null, ext: null, @@ -330,6 +448,7 @@ function parseArgs(argv) { model: null, tokensUsed: null, errorMessage: null, + headCommit: null, } let i = 2 // Skip 'node' and script name @@ -359,6 +478,13 @@ function parseArgs(argv) { case '--title': args.title = argv[++i] break + case '--path': + args.path = argv[++i] + args.paths.push(args.path) + break + case '--head-commit': + args.headCommit = argv[++i] + break case '--file': case '-f': args.file = argv[++i] @@ -413,7 +539,9 @@ Wiki Submit Skill - Submit wiki documentation to Wegent backend Usage: node wiki_submit.js [options] Commands: - submit Submit a wiki section + submit Submit a wiki page + read Print a page's current content + remove Declare wiki pages as gone complete Mark wiki generation as completed fail Mark wiki generation as failed @@ -428,13 +556,23 @@ environment variable when running inside an executor container. You don't need to specify it manually in most cases. Submit Options: - --type Section type (overview|architecture|module|api|guide|deep) - --title Section title - --file, -f Path to markdown file containing section content - --content, -c Section content (alternative to --file) + --path Stable page path, e.g. "architecture/backend". This is the + page's identity: keep it the same across runs. + --title Page title (required) + --type Section type, defaults to "chapter" + --file, -f Path to markdown file containing page content + --content, -c Page content (alternative to --file) --ext Extension data as JSON string +Read Options: + --path Page path to read. Exits 0 with no output when the page + does not exist yet. + +Remove Options: + --path Page path to remove. Repeat for several pages. + Complete Options: + --head-commit Commit that was documented, from \`git rev-parse HEAD\` --structure-order Ordered list of section identifiers --model Model name used for generation --tokens-used Number of tokens used @@ -443,8 +581,10 @@ Fail Options: --error-message, -m Error message describing the failure Examples: - node wiki_submit.js submit --generation-id 123 --type overview --title "Project Overview" --file ./overview.md - node wiki_submit.js complete --generation-id 123 --structure-order "overview: Project Overview" "architecture: System Architecture" + node wiki_submit.js submit --generation-id 123 --path architecture/backend --title "Backend Architecture" --file ./page.md + node wiki_submit.js read --generation-id 123 --path architecture/backend > current.md + node wiki_submit.js remove --generation-id 123 --path modules/legacy-sync + node wiki_submit.js complete --generation-id 123 --head-commit $(git rev-parse HEAD) node wiki_submit.js fail --generation-id 123 --error-message "Failed to analyze repository" `) } @@ -463,8 +603,11 @@ async function main() { let exitCode switch (args.command) { case 'submit': - if (!args.type) { - console.error('Error: --type is required for submit command') + if (!args.path) { + // The path is the page's identity. Without one the write is accepted and the + // page is then skipped at publish time, so it would report success and + // silently produce nothing. + console.error('Error: --path is required for submit command') process.exit(1) } if (!args.title) { @@ -473,6 +616,12 @@ async function main() { } exitCode = await cmdSubmit(args) break + case 'read': + exitCode = await cmdRead(args) + break + case 'remove': + exitCode = await cmdRemove(args) + break case 'complete': exitCode = await cmdComplete(args) break diff --git a/backend/scripts/seed_code_wiki.py b/backend/scripts/seed_code_wiki.py new file mode 100644 index 0000000000..f8981233c9 --- /dev/null +++ b/backend/scripts/seed_code_wiki.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Fill a code wiki with pages without running an agent. + +Everything after the agent — the publish gate, the projection, attachment storage, +the index queue and the reader — can be exercised without a model or an executor, +because the agent's only interface is the write API. This plays that interface. + +What it does *not* cover: choosing a run mode, cloning the repository, and whether +the model produces anything worth reading. Those need the real thing. + + uv run python scripts/seed_code_wiki.py --kb-id 12 + uv run python scripts/seed_code_wiki.py --kb-id 12 --pages index architecture \\ + architecture/backend +""" + +import argparse +import sys +from datetime import datetime + +from app.db.session import SessionLocal +from app.models.kind import Kind +from app.models.user import User +from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.schemas.wiki import ( + WikiContentSection, + WikiContentSummary, + WikiContentWriteRequest, +) +from app.services.knowledge.code_wiki.generation import start_generation +from app.services.knowledge.code_wiki.registry import wiki_owner +from app.services.wiki_service import WikiService + +DEFAULT_PAGES = ["index", "architecture", "architecture/backend", "guides/setup"] + + +def _body(path: str) -> str: + """Content with a heading tree and a diagram, so the outline and Mermaid show.""" + return f"""# {path} + +Seeded page for `{path}`. + +## What this covers + +Placeholder prose so the outline on the right has something to list. + +## How it fits + +```mermaid +flowchart TD + A[Request] --> B[{path}] + B --> C[(Storage)] +``` + +### A deeper heading + +The outline folds anything below level three, so this is the last level shown. +""" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kb-id", type=int, required=True, help="Code wiki to fill") + parser.add_argument( + "--pages", nargs="*", default=DEFAULT_PAGES, help="Page paths to write" + ) + parser.add_argument( + "--commit", + default="seed" + datetime.now().strftime("%H%M%S"), + help="Commit the seeded version claims to document", + ) + args = parser.parse_args() + + with SessionLocal() as db: + knowledge_base = db.get(Kind, args.kb_id) + if knowledge_base is None or knowledge_base.kind != "KnowledgeBase": + print(f"No knowledge base {args.kb_id}", file=sys.stderr) + return 1 + + requester = db.get(User, knowledge_base.user_id) + if requester is None: + print(f"Knowledge base {args.kb_id} has no owner", file=sys.stderr) + return 1 + owner: User = wiki_owner(db, requester) + started = start_generation( + db, + knowledge_base=knowledge_base, + user=owner, + head_commit=args.commit, + ) + if not started.started: + print(f"No run needed: {started.decision.reason}", file=sys.stderr) + return 1 + + generation: WikiGeneration = started.generation + db.commit() + print(f"generation {generation.id} ({started.decision.mode})") + + # Written through the same API the agent uses, so the path matching, the + # removal channel and the completion handoff are all the real ones. + service = WikiService() + service.save_generation_contents( + db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[ + WikiContentSection( + type="chapter", + title=path.rsplit("/", 1)[-1].replace("-", " ").title(), + content=_body(path), + path=path, + ) + for path in args.pages + ], + ), + ) + + service.save_generation_contents( + db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[], + summary=WikiContentSummary( + status="COMPLETED", + head_commit=args.commit, + structure_order=list(args.pages), + ), + ), + ) + + db.refresh(generation) + published = ( + (knowledge_base.json or {}).get("spec", {}).get("publishedGenerationId") + ) + pages = ( + db.query(WikiContent) + .filter(WikiContent.generation_id == generation.id) + .count() + ) + print( + f"status={generation.status} pages={pages} published={published}", + file=( + sys.stderr + if generation.status != WikiGenerationStatus.COMPLETED + else sys.stdout + ), + ) + return 0 if published == generation.id else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/api/test_knowledge_code_wiki.py b/backend/tests/api/test_knowledge_code_wiki.py new file mode 100644 index 0000000000..2212f9d043 --- /dev/null +++ b/backend/tests/api/test_knowledge_code_wiki.py @@ -0,0 +1,715 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""API tests for creating a code wiki.""" + +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session, sessionmaker + +from app.core.security import create_access_token +from app.models.user import User +from app.schemas.knowledge import KnowledgeBaseType +from app.services.knowledge.code_wiki.source import SourceAccessDenied + +CREATE_URL = "/api/knowledge-bases/code-wikis" + +PAYLOAD = { + "name": "Wegent Wiki", + "namespace": "default", + "source_type": "github", + "source_url": "https://github.com/wecode-ai/Wegent.git", +} + + +@pytest.fixture +def auth_headers(test_user: User) -> dict[str, str]: + token = create_access_token(data={"sub": test_user.user_name}) + return {"Authorization": f"Bearer {token}"} + + +@pytest.fixture +def kind_services_use_test_db(test_db: Session, monkeypatch: pytest.MonkeyPatch): + """Point ``KindBaseService``'s own session at the test database. + + Creating a knowledge base resolves a default embedding model, which reaches + ``KindBaseService.list_resources``. That opens its own ``SessionLocal`` instead of + using the request's session, so the FastAPI dependency override does not reach it + and it connects to whatever database is configured. On a developer machine with + MySQL running the test then passes for the wrong reason; in CI there is no MySQL + and it fails. + """ + factory = sessionmaker( + autocommit=False, + autoflush=False, + bind=test_db.get_bind(), + expire_on_commit=False, + ) + monkeypatch.setattr("app.services.kind_base.SessionLocal", factory) + + +def test_creating_a_code_wiki_records_its_type_and_source( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + response = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + + assert response.status_code == 201, response.text + body = response.json() + assert body["kb_type"] == KnowledgeBaseType.CODE_WIKI.value + assert body["name"] == "Wegent Wiki" + + +def test_creation_is_refused_without_repository_access( + test_client: TestClient, auth_headers: dict[str, str] +): + """A wiki must not be built for a repository the requester cannot read.""" + from app.services.knowledge.code_wiki.source import SourceAccessDenied + + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + side_effect=SourceAccessDenied("You do not have read access to 'x/y'."), + ): + response = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + + assert response.status_code == 403 + assert "read access" in response.json()["detail"] + + +def test_creation_requires_authentication(test_client: TestClient): + response = test_client.post(CREATE_URL, json=PAYLOAD) + + assert response.status_code in (401, 403) + + +def test_unsupported_source_type_is_rejected_by_validation( + test_client: TestClient, auth_headers: dict[str, str] +): + payload = {**PAYLOAD, "source_type": "svn"} + + response = test_client.post(CREATE_URL, json=payload, headers=auth_headers) + + assert response.status_code == 422 + + +def test_the_general_endpoint_refuses_to_create_a_code_wiki( + test_client: TestClient, auth_headers: dict[str, str] +): + """Only the code wiki endpoint may create one, because only it checks the repo.""" + response = test_client.post( + "/api/knowledge-bases", + json={"name": "sneaky", "kb_type": "code_wiki"}, + headers=auth_headers, + ) + + assert response.status_code == 400 + assert "code-wikis" in response.json()["detail"] + + +def test_ordinary_knowledge_bases_are_still_created_as_notebooks( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + response = test_client.post( + "/api/knowledge-bases", json={"name": "plain notes"}, headers=auth_headers + ) + + assert response.status_code == 201, response.text + assert response.json()["kb_type"] == KnowledgeBaseType.NOTEBOOK.value + + +# --- triggering a run ------------------------------------------------------- + + +def _create_wiki(test_client: TestClient, auth_headers: dict[str, str]) -> int: + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + response = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + assert response.status_code == 201, response.text + return response.json()["id"] + + +def _run_url(knowledge_base_id: int) -> str: + return f"/api/knowledge-bases/{knowledge_base_id}/code-wiki/generations" + + +@pytest.fixture +def caller_can_write(): + """Grant repository write access for tests that are about something else.""" + with patch( + "app.api.endpoints.knowledge.assert_user_can_write_source", + return_value={"has_access": True, "access_level": 30}, + ): + yield + + +def test_a_run_can_be_triggered_without_waiting_for_a_schedule( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, + caller_can_write, +): + kb_id = _create_wiki(test_client, auth_headers) + + with patch("app.api.endpoints.knowledge.start_run") as start: + start.return_value.started = True + start.return_value.mode = "full" + start.return_value.reason = "first run for this repository" + start.return_value.generation.id = 7 + start.return_value.task_id = 42 + + response = test_client.post( + _run_url(kb_id), json={"head_commit": "abc1234"}, headers=auth_headers + ) + + assert response.status_code == 202, response.text + body = response.json() + assert body["started"] is True + assert body["generation_id"] == 7 + assert body["task_id"] == 42 + + +def test_a_run_that_was_not_needed_is_a_success_not_a_failure( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, + caller_can_write, +): + """ "Nothing changed" is the answer the caller asked for, not an error.""" + kb_id = _create_wiki(test_client, auth_headers) + + with patch("app.api.endpoints.knowledge.start_run") as start: + start.return_value.started = False + start.return_value.mode = "skip" + start.return_value.reason = "repository unchanged since last run" + start.return_value.generation = None + start.return_value.task_id = 0 + + response = test_client.post(_run_url(kb_id), json={}, headers=auth_headers) + + assert response.status_code == 202, response.text + body = response.json() + assert body["started"] is False + assert body["mode"] == "skip" + assert body["generation_id"] == 0 + + +def test_a_second_run_while_one_is_live_is_a_conflict( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, + caller_can_write, +): + from app.services.knowledge.code_wiki.generation import GenerationInFlight + + kb_id = _create_wiki(test_client, auth_headers) + + with patch( + "app.api.endpoints.knowledge.start_run", + side_effect=GenerationInFlight("generation 3 is already running"), + ): + response = test_client.post(_run_url(kb_id), json={}, headers=auth_headers) + + assert response.status_code == 409 + assert "already running" in response.json()["detail"] + + +def test_a_knowledge_base_that_is_not_a_code_wiki_cannot_be_generated( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + response = test_client.post( + "/api/knowledge-bases", json={"name": "plain notes"}, headers=auth_headers + ) + notebook_id = response.json()["id"] + + response = test_client.post(_run_url(notebook_id), json={}, headers=auth_headers) + + assert response.status_code == 400 + assert "not a code wiki" in response.json()["detail"] + + +def test_a_missing_knowledge_base_is_not_found( + test_client: TestClient, auth_headers: dict[str, str] +): + response = test_client.post(_run_url(999999), json={}, headers=auth_headers) + + assert response.status_code == 404 + + +def test_triggering_a_run_requires_authentication(test_client: TestClient): + response = test_client.post(_run_url(1), json={}) + + assert response.status_code in (401, 403) + + +# --- who owns a code wiki, and how many there are --------------------------- + + +# --- ownership and listing -------------------------------------------------- + + +LIST_URL = "/api/knowledge-bases/code-wikis" + + +def test_a_code_wiki_belongs_to_whoever_created_it( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + test_user: User, + kind_services_use_test_db, +): + """Ownership is what makes the ordinary ACL apply. Filing it under a shared wiki + account instead left the knowledge base visible to nobody, which forced a second + authorisation rule that only the newly written endpoints ever consulted.""" + from app.models.kind import Kind + + kb_id = _create_wiki(test_client, auth_headers) + + kind = test_db.get(Kind, kb_id) + assert kind.user_id == test_user.id + + +def test_the_creator_can_read_the_wiki_they_just_created( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """The failure this pins is not hypothetical: under the previous owner the + creator was refused their own wiki's navigation immediately after creating it.""" + kb_id = _create_wiki(test_client, auth_headers) + + response = test_client.get( + f"/api/knowledge-bases/{kb_id}/code-wiki/pages", headers=auth_headers + ) + + assert response.status_code == 200, response.text + + +def test_a_code_wiki_appears_in_the_general_knowledge_base_list( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """It is a knowledge base. Excluding it from the general list would also take it + out of chat citation and the MCP tool, where being citable is the point.""" + kb_id = _create_wiki(test_client, auth_headers) + + grouped = test_client.get( + "/api/knowledge-bases/all-grouped", headers=auth_headers + ).json() + + listed = [ + kb["id"] + for bucket in ( + grouped["personal"]["created_by_me"], + grouped["personal"]["shared_with_me"], + ) + for kb in bucket + ] + assert kb_id in listed + + +def test_the_list_shows_a_wiki_the_caller_owns( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + kb_id = _create_wiki(test_client, auth_headers) + + response = test_client.get(LIST_URL, headers=auth_headers) + + assert response.status_code == 200, response.text + body = response.json() + assert [item["id"] for item in body["items"]] == [kb_id] + assert body["items"][0]["project_name"] == "wecode-ai/Wegent" + + +def test_the_list_hides_another_user_s_wiki( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """Nothing special to code wikis — this is the knowledge-base ACL doing its job, + and the point of the change is that it is the only rule in play.""" + from app.core.security import create_access_token, get_password_hash + + _create_wiki(test_client, auth_headers) + stranger = User( + user_name="stranger", + password_hash=get_password_hash("irrelevant"), + email="stranger@example.com", + is_active=True, + ) + test_db.add(stranger) + test_db.commit() + token = create_access_token(data={"sub": stranger.user_name}) + + response = test_client.get(LIST_URL, headers={"Authorization": f"Bearer {token}"}) + + assert response.json() == {"items": [], "total": 0} + + +def test_the_repository_is_registered_against_the_wiki_that_documents_it( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """One row per (repository, wiki). The composite UNIQUE is what settles two + requests racing for the same pair, which a check on a JSON field could not.""" + from app.models.wiki import WikiProject + + kb_id = _create_wiki(test_client, auth_headers) + + project = test_db.query(WikiProject).one() + assert project.kind_id == kb_id + assert project.source_url == "https://github.com/wecode-ai/Wegent.git" + + +def test_asking_twice_for_the_same_repository_returns_the_caller_s_own_wiki( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """Answering with the existing one, 200 rather than 201, says which happened.""" + first_id = _create_wiki(test_client, auth_headers) + + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + again = test_client.post(CREATE_URL, json=PAYLOAD, headers=auth_headers) + + assert again.status_code == 200, again.text + assert again.json()["id"] == first_id + + +def test_another_user_may_build_their_own_wiki_of_the_same_repository( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """The first wiki is invisible to the second caller under its owner's ACL, so + refusing them one of their own would take it away on a first-come basis.""" + from app.core.security import create_access_token, get_password_hash + + first_id = _create_wiki(test_client, auth_headers) + other = User( + user_name="colleague", + password_hash=get_password_hash("irrelevant"), + email="colleague@example.com", + is_active=True, + ) + test_db.add(other) + test_db.commit() + token = create_access_token(data={"sub": other.user_name}) + + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + response = test_client.post( + CREATE_URL, json=PAYLOAD, headers={"Authorization": f"Bearer {token}"} + ) + + assert response.status_code == 201, response.text + assert response.json()["id"] != first_id + + +# --- who may trigger a run -------------------------------------------------- + + +def test_regenerating_requires_write_access_to_the_repository( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """A run rewrites every page, so reading the wiki is not enough. Without this a + wiki shared with a reader lets them spend a generation on somebody else's + knowledge base.""" + kb_id = _create_wiki(test_client, auth_headers) + + with patch( + "app.api.endpoints.knowledge.assert_user_can_write_source", + side_effect=SourceAccessDenied("read access but not write access"), + ): + response = test_client.post(_run_url(kb_id), json={}, headers=auth_headers) + + assert response.status_code == 403 + assert "write access" in response.json()["detail"] + + +def test_a_reader_of_the_repository_cannot_regenerate( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """The threshold, not just the presence of a gate: read access reports an access + level below Developer, and that has to be refused rather than rounded up.""" + from app.services.knowledge.code_wiki import source as source_module + + kb_id = _create_wiki(test_client, auth_headers) + + with patch.object( + source_module, + "assert_user_can_read_source", + return_value={"has_access": True, "access_level": 10}, + ): + response = test_client.post(_run_url(kb_id), json={}, headers=auth_headers) + + assert response.status_code == 403 + + +def test_creating_a_wiki_starts_its_first_run( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """Otherwise a new wiki sits empty until somebody finds the regenerate button, + which is not a flow anyone would guess.""" + with patch("app.api.endpoints.knowledge.start_run") as start: + _create_wiki(test_client, auth_headers) + + assert start.call_count == 1 + + +def test_a_first_run_that_cannot_start_still_leaves_the_wiki_created( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """The knowledge base is already committed. Reporting the creation as failed + would be untrue, and the reader's own button still starts a run.""" + from app.services.knowledge.code_wiki.runner import CodeWikiRunError + + with patch( + "app.api.endpoints.knowledge.start_run", + side_effect=CodeWikiRunError("no team configured"), + ): + kb_id = _create_wiki(test_client, auth_headers) + + assert kb_id > 0 + assert ( + test_client.get( + f"/api/knowledge-bases/{kb_id}/code-wiki/pages", headers=auth_headers + ).status_code + == 200 + ) + + +# --- resolving a repository before binding ---------------------------------- + + +RESOLVE_URL = "/api/knowledge-bases/code-wikis/resolve" +RESOLVE_PAYLOAD = { + "source_type": "github", + "source_url": "https://github.com/wecode-ai/Wegent.git", +} + + +def test_resolving_answers_with_what_the_create_form_needs( + test_client: TestClient, auth_headers: dict[str, str] +): + from app.services.knowledge.code_wiki.resolution import ResolvedRepository + + with patch( + "app.api.endpoints.knowledge.resolve_repository", + return_value=ResolvedRepository( + exists=True, + visibility="public", + default_branch="main", + name="wecode-ai/Wegent", + description="An agent operating system", + access="public", + ), + ): + response = test_client.post( + RESOLVE_URL, json=RESOLVE_PAYLOAD, headers=auth_headers + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["default_branch"] == "main" + assert body["access"] == "public" + assert body["name"] == "wecode-ai/Wegent" + + +def test_an_unreadable_repository_resolves_to_200_not_404( + test_client: TestClient, auth_headers: dict[str, str] +): + """This assists a form; it does not assert that something is missing. 404 would + also make private and absent distinguishable, which they must not be.""" + from app.services.knowledge.code_wiki.resolution import UNREADABLE + + with patch( + "app.api.endpoints.knowledge.resolve_repository", return_value=UNREADABLE + ): + response = test_client.post( + RESOLVE_URL, json=RESOLVE_PAYLOAD, headers=auth_headers + ) + + assert response.status_code == 200 + assert response.json()["exists"] is False + + +def test_resolving_names_the_wikis_that_already_exist( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """A count is not actionable: asking for a share needs somebody to ask. One the + caller can already open is reported as accessible, so the client links to it + instead.""" + from app.services.knowledge.code_wiki.resolution import UNREADABLE + + kb_id = _create_wiki(test_client, auth_headers) + + with patch( + "app.api.endpoints.knowledge.resolve_repository", return_value=UNREADABLE + ): + response = test_client.post( + RESOLVE_URL, json=RESOLVE_PAYLOAD, headers=auth_headers + ) + + (existing,) = response.json()["existing_wikis"] + assert existing["id"] == kb_id + assert existing["accessible"] is True + assert existing["owner_name"] == "testuser" + + +def test_another_user_s_wiki_is_named_with_its_owner( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """The owner is the whole point of naming it: the caller cannot open this one, + so the useful action is to ask.""" + from app.core.security import create_access_token, get_password_hash + from app.services.knowledge.code_wiki.resolution import UNREADABLE + + _create_wiki(test_client, auth_headers) + stranger = User( + user_name="stranger", + password_hash=get_password_hash("irrelevant"), + email="stranger@example.com", + is_active=True, + ) + test_db.add(stranger) + test_db.commit() + token = create_access_token(data={"sub": stranger.user_name}) + + with patch( + "app.api.endpoints.knowledge.resolve_repository", return_value=UNREADABLE + ): + response = test_client.post( + RESOLVE_URL, + json=RESOLVE_PAYLOAD, + headers={"Authorization": f"Bearer {token}"}, + ) + + (existing,) = response.json()["existing_wikis"] + assert existing["accessible"] is False + assert existing["owner_name"] == "testuser" + + +def test_a_code_wiki_can_be_created_without_a_name( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """Left blank, the repository's name is used. The client sends what the form + already resolved, so this does not ask the provider a second time — the access + gate has asked it once already.""" + from app.models.kind import Kind + + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + response = test_client.post( + CREATE_URL, + # Deliberately not what the URL parses to: the provider is the authority + # on what a repository is called, renames included. + json={**PAYLOAD, "name": "wecode-ai/Wegent-Renamed"}, + headers=auth_headers, + ) + + assert response.status_code == 201, response.text + kind = test_db.get(Kind, response.json()["id"]) + assert kind.json["spec"]["name"] == "wecode-ai/Wegent-Renamed" + + +def test_an_unnamed_wiki_falls_back_to_the_url( + test_client: TestClient, + auth_headers: dict[str, str], + test_db: Session, + kind_services_use_test_db, +): + """A caller that skipped the probe, or called the API directly, would otherwise + create a knowledge base with no name at all, which no listing can render.""" + from app.models.kind import Kind + + with patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ): + response = test_client.post( + CREATE_URL, json={**PAYLOAD, "name": ""}, headers=auth_headers + ) + + assert response.status_code == 201, response.text + kind = test_db.get(Kind, response.json()["id"]) + assert kind.json["spec"]["name"] == "wecode-ai/Wegent" + + +def test_creating_does_not_describe_the_repository_again( + test_client: TestClient, + auth_headers: dict[str, str], + kind_services_use_test_db, +): + """The access gate already asked the provider. Resolving again here worked only + because the form had just warmed the cache — an implicit coupling that fails + whenever it has not.""" + with ( + patch( + "app.api.endpoints.knowledge.assert_user_can_read_source", + return_value={"has_access": True}, + ), + patch("app.api.endpoints.knowledge.resolve_repository") as resolve, + ): + test_client.post(CREATE_URL, json={**PAYLOAD, "name": ""}, headers=auth_headers) + + resolve.assert_not_called() + + +def test_resolving_a_malformed_url_is_a_bad_request( + test_client: TestClient, auth_headers: dict[str, str] +): + response = test_client.post( + RESOLVE_URL, + json={"source_type": "github", "source_url": "not-a-url"}, + headers=auth_headers, + ) + + assert response.status_code == 400 + + +def test_resolving_requires_authentication(test_client: TestClient): + assert test_client.post(RESOLVE_URL, json=RESOLVE_PAYLOAD).status_code == 401 diff --git a/backend/tests/repository/test_gitlab_provider.py b/backend/tests/repository/test_gitlab_provider.py new file mode 100644 index 0000000000..475efa75ac --- /dev/null +++ b/backend/tests/repository/test_gitlab_provider.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for how GitLabProvider resolves the credential it calls GitLab with. + +Tokens are stored encrypted on the user, and nothing between the session user and +the provider decrypts them. So what is pinned here is that the provider does it +itself, at the single place entries are built, and that a domain whose credential +is refused says so rather than dropping out of the result in silence. +""" + +from unittest.mock import AsyncMock, Mock, patch + +import pytest +import requests + +from app.repository.gitlab_provider import GitLabProvider +from shared.utils.crypto import encrypt_git_token + +PLAIN_TOKEN = "glpat-000000000000000000" +DOMAIN = "gitlab.example.com" + + +@pytest.fixture(autouse=True) +def _crypto_env(monkeypatch): + monkeypatch.setenv("GIT_TOKEN_AES_KEY", "12345678901234567890123456789012") + monkeypatch.setenv("GIT_TOKEN_AES_IV", "1234567890123456") + # The module caches the key on first use, so a test that runs after one which + # left a different key would otherwise read the stale one. + import shared.utils.crypto as crypto + + monkeypatch.setattr(crypto, "_aes_key", None) + monkeypatch.setattr(crypto, "_aes_iv", None) + + +def _user(token: str) -> Mock: + user = Mock() + user.id = 1 + user.user_name = "testuser" + user.git_info = [{"type": "gitlab", "git_domain": DOMAIN, "git_token": token}] + return user + + +@pytest.fixture +def provider() -> GitLabProvider: + return GitLabProvider() + + +@pytest.fixture +def _no_cache(): + """Keep the result-caching side of get_repositories out of the way. + + ``set`` is awaited, so it has to be an async double; ``generate_full_cache_key`` + is not. + """ + cache = Mock(set=AsyncMock(), generate_full_cache_key=Mock(return_value="key")) + with patch("app.repository.gitlab_provider.cache_manager", cache): + yield cache + + +@pytest.mark.unit +class TestTokenResolution: + def test_a_stored_token_is_decrypted_before_it_is_used(self, provider): + """The ciphertext is not a credential. Sent as one it yields a 401, which + reads as a bad token rather than as the provider never decrypting it.""" + user = _user(encrypt_git_token(PLAIN_TOKEN)) + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == PLAIN_TOKEN + + def test_a_plaintext_token_is_left_alone(self, provider): + """Deployments exist that put a usable token straight into git_info.""" + user = _user(PLAIN_TOKEN) + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == PLAIN_TOKEN + + def test_the_placeholder_is_left_alone(self, provider): + """'***' means the token lives elsewhere and is substituted before the call. + Decrypting it would destroy the marker the substitution looks for.""" + user = _user("***") + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == "***" + + @pytest.mark.asyncio + async def test_the_decrypted_token_is_what_reaches_gitlab( + self, provider, _no_cache + ): + """Covers the whole path rather than the helper alone: a caller between + _get_git_infos and the request could still pass the raw entry through.""" + user = _user(encrypt_git_token(PLAIN_TOKEN)) + response = Mock(status_code=200) + response.json.return_value = [] + + with ( + patch.object( + provider, "_get_all_repositories_from_cache", return_value=None + ), + patch.object( + provider, "_make_request_with_auth_retry", return_value=response + ) as request, + ): + await provider.get_repositories(user, page=1, limit=10) + + assert request.call_args.kwargs["token"] == PLAIN_TOKEN + + +@pytest.mark.unit +class TestFailedDomainsAreReported: + @pytest.mark.asyncio + async def test_a_refused_domain_is_logged_rather_than_silently_skipped( + self, provider, caplog + ): + """An empty list and a rejected credential must not look alike. This is the + signal that turns 'this user has no repositories' back into a diagnosis.""" + user = _user(encrypt_git_token(PLAIN_TOKEN)) + refused = requests.exceptions.HTTPError("401 Unauthorized") + refused.response = Mock(status_code=401) + + with ( + patch.object( + provider, "_get_all_repositories_from_cache", return_value=None + ), + patch.object( + provider, "_make_request_with_auth_retry", side_effect=refused + ), + caplog.at_level("WARNING"), + ): + result = await provider.get_repositories(user, page=1, limit=10) + + assert result == [] + assert DOMAIN in caplog.text + assert "401" in caplog.text + + @pytest.mark.asyncio + async def test_one_bad_domain_does_not_lose_a_good_one(self, provider, _no_cache): + """Reporting the failure must not turn a partial result into no result.""" + user = _user(encrypt_git_token(PLAIN_TOKEN)) + user.git_info.append( + { + "type": "gitlab", + "git_domain": "gitlab.other.com", + "git_token": PLAIN_TOKEN, + } + ) + good = Mock(status_code=200) + good.json.return_value = [ + { + "id": 7, + "name": "repo", + "path_with_namespace": "group/repo", + "http_url_to_repo": "https://gitlab.other.com/group/repo.git", + "visibility": "private", + } + ] + refused = requests.exceptions.HTTPError("401 Unauthorized") + refused.response = Mock(status_code=401) + + with ( + patch.object( + provider, "_get_all_repositories_from_cache", return_value=None + ), + patch.object( + provider, "_make_request_with_auth_retry", side_effect=[refused, good] + ), + ): + result = await provider.get_repositories(user, page=1, limit=10) + + assert [repo["full_name"] for repo in result] == ["group/repo"] diff --git a/backend/tests/repository/test_provider_token_resolution.py b/backend/tests/repository/test_provider_token_resolution.py new file mode 100644 index 0000000000..1754960646 --- /dev/null +++ b/backend/tests/repository/test_provider_token_resolution.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Every provider must decrypt the stored credential before it calls out. + +Tokens are written to ``users.git_info`` encrypted, and nothing between the session +user and a provider decrypts them: ``get_current_user`` deliberately does not, so +that an unrelated endpoint never depends on Git crypto configuration. Each provider +therefore has to do it itself, and each one builds its entries in its own copy of +``_get_git_infos`` — which is exactly how the same omission ended up in all five. + +This is parametrized rather than written per provider so a sixth one cannot be added +with the omission intact. +""" + +import pytest + +from app.repository.gerrit_provider import GerritProvider +from app.repository.gitea_provider import GiteaProvider +from app.repository.gitee_provider import GiteeProvider +from app.repository.github_provider import GitHubProvider +from app.repository.gitlab_provider import GitLabProvider +from shared.utils.crypto import encrypt_git_token + +PLAIN_TOKEN = "glpat-000000000000000000" + +PROVIDERS = [ + pytest.param(GitHubProvider, "github", id="github"), + pytest.param(GitLabProvider, "gitlab", id="gitlab"), + pytest.param(GiteaProvider, "gitea", id="gitea"), + pytest.param(GiteeProvider, "gitee", id="gitee"), + pytest.param(GerritProvider, "gerrit", id="gerrit"), +] + + +@pytest.fixture(autouse=True) +def _crypto_env(monkeypatch): + monkeypatch.setenv("GIT_TOKEN_AES_KEY", "12345678901234567890123456789012") + monkeypatch.setenv("GIT_TOKEN_AES_IV", "1234567890123456") + import shared.utils.crypto as crypto + + monkeypatch.setattr(crypto, "_aes_key", None) + monkeypatch.setattr(crypto, "_aes_iv", None) + + +def _user(provider_type: str, token: str): + from unittest.mock import Mock + + user = Mock() + user.id = 1 + user.user_name = "testuser" + user.git_info = [ + { + "type": provider_type, + "git_domain": f"{provider_type}.example.com", + "git_token": token, + # Gerrit refuses an entry without one; harmless to the others. + "user_name": "testuser", + } + ] + return user + + +@pytest.mark.unit +@pytest.mark.parametrize("provider_class,provider_type", PROVIDERS) +def test_a_stored_token_is_decrypted(provider_class, provider_type): + """Sent as-is the ciphertext yields a 401, which reads as a bad credential + rather than as the provider never having decrypted it.""" + provider = provider_class() + user = _user(provider_type, encrypt_git_token(PLAIN_TOKEN)) + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == PLAIN_TOKEN + + +@pytest.mark.unit +@pytest.mark.parametrize("provider_class,provider_type", PROVIDERS) +def test_a_plaintext_token_is_left_alone(provider_class, provider_type): + """Deployments exist that put a usable token straight into git_info.""" + provider = provider_class() + user = _user(provider_type, PLAIN_TOKEN) + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == PLAIN_TOKEN + + +@pytest.mark.unit +@pytest.mark.parametrize("provider_class,provider_type", PROVIDERS) +def test_the_placeholder_survives(provider_class, provider_type): + """'***' means the real token is substituted in at call time by a deployment + overlay. Rewriting it would destroy the marker that substitution looks for.""" + provider = provider_class() + user = _user(provider_type, "***") + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == "***" + + +@pytest.mark.unit +@pytest.mark.parametrize("provider_class,provider_type", PROVIDERS) +def test_an_empty_token_stays_empty(provider_class, provider_type): + """Callers test the token for emptiness to raise "not configured". Turning it + into anything else would send them past that check.""" + provider = provider_class() + user = _user(provider_type, "") + + (entry,) = provider._get_git_infos(user) + + assert entry["git_token"] == "" diff --git a/backend/tests/repository/test_repository_state_reads.py b/backend/tests/repository/test_repository_state_reads.py new file mode 100644 index 0000000000..505969cc41 --- /dev/null +++ b/backend/tests/repository/test_repository_state_reads.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the two repository reads a code wiki run depends on. + +Each platform words its compare API differently, and each has a way of telling you +less than you asked for: GitHub caps the file list, GitLab gives up on a large diff, +and an older self-hosted Gitea has no compare endpoint at all. All three have to +surface as ``None`` — the caller reads that as "the extent of the change is unknown" +and rebuilds — because an incomplete diff mistaken for a complete one picks an +incremental run for a change that reshaped the repository. +""" + +from unittest.mock import Mock, patch + +from app.repository.gitea_provider import GiteaProvider +from app.repository.github_provider import GITHUB_COMPARE_FILE_LIMIT, GitHubProvider +from app.repository.gitlab_provider import GitLabProvider + +DOMAIN_ARGS = dict(token="t0ken", repo_name="wecode-ai/Wegent") + + +def _response(payload: dict, status_code: int = 200) -> Mock: + response = Mock() + response.status_code = status_code + response.json.return_value = payload + response.raise_for_status.return_value = None + return response + + +# --- GitHub ----------------------------------------------------------------- + + +def test_github_reads_the_default_branch_and_its_commit(): + provider = GitHubProvider() + responses = [ + _response({"default_branch": "develop"}), + _response({"commit": {"sha": "abc123"}}), + ] + + with patch("app.repository.github_provider.requests.get", side_effect=responses): + result = provider.get_default_branch_head( + git_domain="github.com", **DOMAIN_ARGS + ) + + assert result == {"branch": "develop", "commit": "abc123"} + + +def test_github_maps_compare_statuses_to_name_status_letters(): + """The run-mode rules are written against git's letters, not GitHub's words.""" + provider = GitHubProvider() + payload = { + "files": [ + {"filename": "a.py", "status": "added"}, + {"filename": "b.py", "status": "modified"}, + {"filename": "c.py", "status": "removed"}, + {"filename": "d.py", "status": "renamed"}, + ] + } + + with patch( + "app.repository.github_provider.requests.get", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="github.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed == [ + {"path": "a.py", "status": "A"}, + {"path": "b.py", "status": "M"}, + {"path": "c.py", "status": "D"}, + {"path": "d.py", "status": "R"}, + ] + + +def test_github_reports_a_truncated_compare_as_unknown(): + """GitHub caps the list at 300 and does not say so; a capped list read as + complete would understate a change big enough to need a rebuild.""" + provider = GitHubProvider() + payload = { + "files": [ + {"filename": f"file{index}.py", "status": "modified"} + for index in range(GITHUB_COMPARE_FILE_LIMIT) + ] + } + + with patch( + "app.repository.github_provider.requests.get", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="github.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed is None + + +def test_github_reports_a_compare_below_the_cap_normally(): + provider = GitHubProvider() + payload = { + "files": [ + {"filename": f"file{index}.py", "status": "modified"} + for index in range(GITHUB_COMPARE_FILE_LIMIT - 1) + ] + } + + with patch( + "app.repository.github_provider.requests.get", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="github.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed is not None + assert len(changed) == GITHUB_COMPARE_FILE_LIMIT - 1 + + +# --- GitLab ----------------------------------------------------------------- + + +def test_gitlab_reads_the_default_branch_and_its_commit(): + provider = GitLabProvider() + responses = [ + _response({"default_branch": "main"}), + _response({"commit": {"id": "def456"}}), + ] + + with patch.object( + GitLabProvider, "_make_request_with_auth_retry", side_effect=responses + ): + result = provider.get_default_branch_head( + git_domain="gitlab.com", **DOMAIN_ARGS + ) + + assert result == {"branch": "main", "commit": "def456"} + + +def test_gitlab_derives_status_from_the_diff_flags(): + provider = GitLabProvider() + payload = { + "diffs": [ + {"new_path": "a.py", "new_file": True}, + {"new_path": "b.py"}, + {"old_path": "c.py", "deleted_file": True}, + {"new_path": "d.py", "renamed_file": True}, + ] + } + + with patch.object( + GitLabProvider, "_make_request_with_auth_retry", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="gitlab.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed == [ + {"path": "a.py", "status": "A"}, + {"path": "b.py", "status": "M"}, + {"path": "c.py", "status": "D"}, + {"path": "d.py", "status": "R"}, + ] + + +def test_gitlab_reports_a_timed_out_compare_as_unknown(): + """GitLab answers 200 with a partial diff and a flag, so the flag is the only + thing separating "small change" from "gave up".""" + provider = GitLabProvider() + payload = { + "compare_timeout": True, + "diffs": [{"new_path": "a.py"}], + } + + with patch.object( + GitLabProvider, "_make_request_with_auth_retry", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="gitlab.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed is None + + +# --- Gitea ------------------------------------------------------------------ + + +def test_gitea_reads_the_default_branch_and_its_commit(): + provider = GiteaProvider() + responses = [ + _response({"default_branch": "master"}), + _response({"commit": {"id": "789abc"}}), + ] + + with patch("app.repository.gitea_provider.requests.get", side_effect=responses): + result = provider.get_default_branch_head(git_domain="gitea.com", **DOMAIN_ARGS) + + assert result == {"branch": "master", "commit": "789abc"} + + +def test_gitea_maps_compare_statuses_to_name_status_letters(): + provider = GiteaProvider() + payload = { + "files": [ + {"filename": "a.py", "status": "added"}, + {"filename": "b.py", "status": "changed"}, + {"filename": "c.py", "status": "deleted"}, + ] + } + + with patch( + "app.repository.gitea_provider.requests.get", return_value=_response(payload) + ): + changed = provider.get_changed_files( + git_domain="gitea.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed == [ + {"path": "a.py", "status": "A"}, + {"path": "b.py", "status": "M"}, + {"path": "c.py", "status": "D"}, + ] + + +def test_an_older_gitea_without_a_compare_endpoint_reports_unknown(): + """Self-hosted instances lag; a 404 here must degrade, not raise.""" + provider = GiteaProvider() + + with patch( + "app.repository.gitea_provider.requests.get", + return_value=_response({}, status_code=404), + ): + changed = provider.get_changed_files( + git_domain="gitea.example.com", base="aaa", head="bbb", **DOMAIN_ARGS + ) + + assert changed is None diff --git a/backend/tests/services/knowledge/__init__.py b/backend/tests/services/knowledge/__init__.py new file mode 100644 index 0000000000..fd017d8796 --- /dev/null +++ b/backend/tests/services/knowledge/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/backend/tests/services/knowledge/code_wiki/__init__.py b/backend/tests/services/knowledge/code_wiki/__init__.py new file mode 100644 index 0000000000..fd017d8796 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/backend/tests/services/knowledge/code_wiki/test_content_write.py b/backend/tests/services/knowledge/code_wiki/test_content_write.py new file mode 100644 index 0000000000..a0c97e7f69 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_content_write.py @@ -0,0 +1,347 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for writing pages into a version by path. + +Matching on path rather than title is what keeps a page's document id — and with it +its RAG index entry and any stored citation — stable when the agent rewords a heading. +""" + +from datetime import datetime + +import pytest +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from app.models.wiki import ( + WikiContent, + WikiGeneration, + WikiGenerationStatus, + WikiGenerationType, +) +from app.schemas.wiki import WikiContentSection, WikiContentWriteRequest +from app.services.knowledge.code_wiki.version_store import page_path_of +from app.services.wiki_service import WikiService + +KIND_ID = 91 + + +@pytest.fixture +def generation(test_db: Session) -> WikiGeneration: + record = WikiGeneration( + project_id=1, + kind_id=KIND_ID, + user_id=1, + task_id=0, + team_id=1, + generation_type=WikiGenerationType.INCREMENTAL, + source_snapshot={}, + status=WikiGenerationStatus.RUNNING, + # The column defaults to a string literal, which SQLite refuses. + completed_at=datetime(1970, 1, 1), + ) + test_db.add(record) + test_db.flush() + return record + + +def _write(db: Session, generation: WikiGeneration, *sections: WikiContentSection): + WikiService().save_generation_contents( + db, + WikiContentWriteRequest(generation_id=generation.id, sections=list(sections)), + ) + + +def _section(path: str | None, title: str, content: str = "body") -> WikiContentSection: + return WikiContentSection(type="chapter", title=title, content=content, path=path) + + +def _pages(db: Session, generation_id: int) -> list[WikiContent]: + return ( + db.query(WikiContent).filter(WikiContent.generation_id == generation_id).all() + ) + + +def test_a_page_is_written_with_its_path(test_db: Session, generation: WikiGeneration): + _write(test_db, generation, _section("architecture/backend", "Backend")) + + (page,) = _pages(test_db, generation.id) + assert page_path_of(page) == "architecture/backend" + assert page.title == "Backend" + + +def test_rewording_a_title_revises_the_same_page( + test_db: Session, generation: WikiGeneration +): + """The point of path identity: no delete-and-recreate, so the id survives.""" + _write(test_db, generation, _section("architecture/backend", "Backend", "v1")) + (original,) = _pages(test_db, generation.id) + + _write( + test_db, + generation, + _section("architecture/backend", "The Backend Service", "v2"), + ) + + pages = _pages(test_db, generation.id) + assert len(pages) == 1 + assert pages[0].id == original.id + assert pages[0].title == "The Backend Service" + assert pages[0].content == "v2" + + +def test_moving_a_page_to_a_new_path_creates_a_new_page( + test_db: Session, generation: WikiGeneration +): + _write(test_db, generation, _section("architecture/backend", "Backend")) + + _write(test_db, generation, _section("services/backend", "Backend")) + + assert {page_path_of(page) for page in _pages(test_db, generation.id)} == { + "architecture/backend", + "services/backend", + } + + +def test_the_path_survives_a_rewrite_that_supplies_no_ext( + test_db: Session, generation: WikiGeneration +): + """``ext`` is replaced wholesale on update, which previously dropped the path.""" + _write(test_db, generation, _section("architecture/backend", "Backend", "v1")) + + _write(test_db, generation, _section("architecture/backend", "Backend", "v2")) + + (page,) = _pages(test_db, generation.id) + assert page_path_of(page) == "architecture/backend" + + +def test_a_path_is_normalized_before_it_is_stored( + test_db: Session, generation: WikiGeneration +): + _write(test_db, generation, _section(" architecture//backend.md ", "Backend")) + + (page,) = _pages(test_db, generation.id) + assert page_path_of(page) == "architecture/backend" + + +def test_a_malformed_path_fails_the_write_not_the_publish( + test_db: Session, generation: WikiGeneration +): + with pytest.raises(HTTPException) as exc: + _write(test_db, generation, _section("../escape", "Escape")) + + assert exc.value.status_code == 400 + + +def test_two_paths_colliding_by_case_are_refused( + test_db: Session, generation: WikiGeneration +): + """The knowledge tables collate case-insensitively; both cannot be honoured.""" + with pytest.raises(HTTPException) as exc: + _write( + test_db, + generation, + _section("architecture/backend", "One"), + _section("Architecture/Backend", "Two"), + ) + + assert exc.value.status_code == 400 + assert "case" in exc.value.detail + + +def test_a_rejected_write_leaves_the_version_unchanged( + test_db: Session, generation: WikiGeneration +): + _write(test_db, generation, _section("index", "Index")) + + with pytest.raises(HTTPException): + _write(test_db, generation, _section("bad\\path", "Bad")) + + assert {page_path_of(page) for page in _pages(test_db, generation.id)} == {"index"} + + +def test_writes_without_a_path_still_match_on_title( + test_db: Session, generation: WikiGeneration +): + """The legacy write path predates page identity and must keep working.""" + _write(test_db, generation, _section(None, "Overview", "v1")) + + _write(test_db, generation, _section(None, "Overview", "v2")) + + (page,) = _pages(test_db, generation.id) + assert page.content == "v2" + + +def test_a_legacy_write_never_resolves_to_a_path_identified_page( + test_db: Session, generation: WikiGeneration +): + """Moving a page keeps its title, so a title can name several pages. + + If the title fallback could reach them, which one a path-less write overwrote + would depend on query order rather than on any rule. + """ + _write(test_db, generation, _section("architecture/backend", "Backend", "kept")) + _write(test_db, generation, _section("services/backend", "Backend", "kept too")) + + _write(test_db, generation, _section(None, "Backend", "legacy")) + + by_path = { + page_path_of(page): page.content for page in _pages(test_db, generation.id) + } + assert by_path["architecture/backend"] == "kept" + assert by_path["services/backend"] == "kept too" + # The legacy write created its own path-less entry instead of hijacking one. + assert by_path[""] == "legacy" + + +# --- removals --------------------------------------------------------------- + + +def _remove(db: Session, generation: WikiGeneration, *paths: str): + WikiService().save_generation_contents( + db, + WikiContentWriteRequest( + generation_id=generation.id, sections=[], removed_paths=list(paths) + ), + ) + + +def test_a_page_the_agent_declares_gone_is_dropped_from_the_version( + test_db: Session, generation: WikiGeneration +): + """An incremental version is a copy of the published one, so not writing a page + cannot mean removing it. Declaring it is the only channel there is.""" + _write(test_db, generation, _section("index", "Index"), _section("legacy", "Old")) + + _remove(test_db, generation, "legacy") + + assert {page_path_of(page) for page in _pages(test_db, generation.id)} == {"index"} + + +def test_a_removal_is_matched_the_same_way_a_write_is( + test_db: Session, generation: WikiGeneration +): + """A path spelled differently must still name the same page, or a removal + silently does nothing and the page outlives its subject.""" + _write(test_db, generation, _section("modules/sync", "Sync")) + + _remove(test_db, generation, " Modules//Sync.md ") + + assert _pages(test_db, generation.id) == [] + + +def test_removing_a_page_that_is_already_gone_is_not_an_error( + test_db: Session, generation: WikiGeneration +): + """A retried submission would otherwise lose the sections sent alongside it.""" + _write(test_db, generation, _section("index", "Index")) + + WikiService().save_generation_contents( + test_db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[_section("guide", "Guide")], + removed_paths=["never-existed"], + ), + ) + + assert {page_path_of(page) for page in _pages(test_db, generation.id)} == { + "index", + "guide", + } + + +def test_a_malformed_removal_path_is_refused( + test_db: Session, generation: WikiGeneration +): + with pytest.raises(HTTPException) as exc: + _remove(test_db, generation, "../escape") + + assert exc.value.status_code == 400 + + +def test_a_page_written_and_removed_in_one_payload_ends_up_removed( + test_db: Session, generation: WikiGeneration +): + """Order within a payload must not decide the outcome.""" + WikiService().save_generation_contents( + test_db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[_section("doomed", "Doomed")], + removed_paths=["doomed"], + ), + ) + + assert _pages(test_db, generation.id) == [] + + +def test_a_payload_with_nothing_in_it_is_still_refused( + test_db: Session, generation: WikiGeneration +): + with pytest.raises(HTTPException) as exc: + WikiService().save_generation_contents( + test_db, + WikiContentWriteRequest(generation_id=generation.id, sections=[]), + ) + + assert exc.value.status_code == 400 + + +# --- reading a page back ---------------------------------------------------- + + +def test_a_written_page_can_be_read_back(test_db: Session, generation: WikiGeneration): + """Without this the instruction to revise a page cannot be followed: the agent + knows the path and cannot see a word of what the page says.""" + _write(test_db, generation, _section("architecture/backend", "Backend", "v1")) + + page = WikiService().get_generation_page( + test_db, generation.id, "architecture/backend" + ) + + assert page is not None + assert page.path == "architecture/backend" + assert page.title == "Backend" + assert page.content == "v1" + + +def test_a_page_is_read_by_the_same_path_that_would_write_it( + test_db: Session, generation: WikiGeneration +): + """Resolving reads and writes differently would let the agent read one page and + overwrite another.""" + _write(test_db, generation, _section("modules/sync", "Sync", "body")) + + page = WikiService().get_generation_page( + test_db, generation.id, " Modules//Sync.md " + ) + + assert page is not None and page.content == "body" + + +def test_a_path_holding_no_page_reads_as_absent( + test_db: Session, generation: WikiGeneration +): + """An answer, not a failure: in an incremental run it means the page is new.""" + assert WikiService().get_generation_page(test_db, generation.id, "nothing") is None + + +def test_a_malformed_path_is_refused_rather_than_read_as_absent( + test_db: Session, generation: WikiGeneration +): + with pytest.raises(HTTPException) as exc: + WikiService().get_generation_page(test_db, generation.id, "../escape") + + assert exc.value.status_code == 400 + + +def test_a_pathless_legacy_entry_is_not_readable( + test_db: Session, generation: WikiGeneration +): + """It has no identity to ask for, and guessing would hand back a page the caller + did not name.""" + _write(test_db, generation, _section(None, "Overview", "legacy")) + + assert WikiService().get_generation_page(test_db, generation.id, "overview") is None diff --git a/backend/tests/services/knowledge/code_wiki/test_generation.py b/backend/tests/services/knowledge/code_wiki/test_generation.py new file mode 100644 index 0000000000..8bf1aa1a37 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_generation.py @@ -0,0 +1,388 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for starting and finishing a code wiki run. + +This is where the separate decisions meet, so these tests are about the combinations: +a run that must not start, a version that must be seeded before the agent sees it, and +a failure that must leave the published wiki exactly as it was. +""" + +from dataclasses import dataclass, field +from datetime import datetime, timedelta + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import KnowledgeDocument +from app.models.user import User +from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.services.knowledge.code_wiki.generation import ( + GenerationInFlight, + finish_generation, + published_commit, + start_generation, +) +from app.services.knowledge.code_wiki.projection import ProjectionSideEffects +from app.services.knowledge.code_wiki.publisher import published_generation_id +from app.services.knowledge.code_wiki.run_mode import ChangedPath, RunMode +from app.services.knowledge.code_wiki.version_store import ( + STALE_RUN_AFTER_HOURS, + set_page_path, +) + +HEAD = "aaaaaaa" +NEXT_HEAD = "bbbbbbb" +NOW = datetime(2026, 7, 31, 12, 0, 0) + + +@dataclass +class FakeEffects: + written: list[str] = field(default_factory=list) + next_id: int = 7000 + + def build(self) -> ProjectionSideEffects: + return ProjectionSideEffects( + write_attachment=self._write, + delete_attachment=lambda _: None, + delete_rag_document=lambda _: None, + enqueue_reindex=lambda _: None, + ) + + def _write(self, *, filename: str, content: str) -> int: + self.next_id += 1 + self.written.append(filename) + return self.next_id + + +@pytest.fixture +def effects() -> FakeEffects: + return FakeEffects() + + +@pytest.fixture +def knowledge_base(test_db: Session, test_user: User) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-generation", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def _write_page(test_db: Session, generation: WikiGeneration, path: str, body: str): + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path, + content=body, + parent_id=0, + ) + set_page_path(entry, path) + test_db.add(entry) + test_db.flush() + + +def _publish_a_first_wiki(test_db, knowledge_base, test_user, effects, *, pages=3): + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + for index in range(pages): + _write_page(test_db, started.generation, f"page-{index}", "body") + finish_generation( + test_db, + knowledge_base=knowledge_base, + generation=started.generation, + user=test_user, + effects=effects.build(), + succeeded=True, + now=NOW, + ) + return started.generation + + +def test_a_first_run_rebuilds_everything( + test_db: Session, knowledge_base: Kind, test_user: User +): + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + + assert started.started + assert RunMode(started.decision.mode) is RunMode.FULL + assert started.seeded_pages == 0 + + +def test_an_unchanged_repository_starts_nothing( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + _publish_a_first_wiki(test_db, knowledge_base, test_user, effects) + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + + assert not started.started + assert RunMode(started.decision.mode) is RunMode.SKIP + + +def test_an_incremental_run_is_seeded_before_the_agent_sees_it( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + """An unseeded incremental version would be projected as a mass deletion.""" + _publish_a_first_wiki(test_db, knowledge_base, test_user, effects, pages=3) + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + now=NOW, + ) + + assert RunMode(started.decision.mode) is RunMode.INCREMENTAL + assert started.seeded_pages == 3 + + +def test_a_full_run_starts_from_an_empty_version( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + _publish_a_first_wiki(test_db, knowledge_base, test_user, effects, pages=3) + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=None, # unknown diff forces a rebuild + now=NOW, + ) + + assert RunMode(started.decision.mode) is RunMode.FULL + assert started.seeded_pages == 0 + + +def test_a_second_run_is_refused_while_one_is_live( + test_db: Session, knowledge_base: Kind, test_user: User +): + start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + + with pytest.raises(GenerationInFlight): + start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + now=NOW, + ) + + +def test_an_abandoned_run_does_not_block_the_wiki_forever( + test_db: Session, knowledge_base: Kind, test_user: User +): + """A crashed worker would otherwise make the wiki unregenerable.""" + stuck = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ).generation + + # Aged explicitly: the row's updated_at comes from the database default, so + # leaving it alone would measure staleness against the wall clock and make this + # test pass or fail depending on the day it runs. + stuck.updated_at = NOW + test_db.flush() + + later = NOW + timedelta(hours=STALE_RUN_AFTER_HOURS + 1) + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + now=later, + ) + + assert started.started + test_db.refresh(stuck) + assert stuck.status == WikiGenerationStatus.FAILED + + +def test_a_successful_run_publishes_its_version( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + _write_page(test_db, started.generation, "index", "overview") + + result = finish_generation( + test_db, + knowledge_base=knowledge_base, + generation=started.generation, + user=test_user, + effects=effects.build(), + succeeded=True, + now=NOW, + ) + + assert result is not None and result.published + assert published_generation_id(knowledge_base) == started.generation.id + assert published_commit(test_db, knowledge_base) == HEAD + + +def test_a_failed_run_leaves_the_published_wiki_untouched( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + first = _publish_a_first_wiki(test_db, knowledge_base, test_user, effects, pages=3) + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + now=NOW, + ) + result = finish_generation( + test_db, + knowledge_base=knowledge_base, + generation=started.generation, + user=test_user, + effects=effects.build(), + succeeded=False, + error_message="model timed out", + now=NOW, + ) + + assert result is None + assert published_generation_id(knowledge_base) == first.id + assert started.generation.status == WikiGenerationStatus.FAILED + assert ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .count() + == 3 + ) + + +def test_a_failure_leaves_the_work_to_be_redone_not_skipped( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + """The published commit must not advance, or the next run skips the changes.""" + _publish_a_first_wiki(test_db, knowledge_base, test_user, effects, pages=3) + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + now=NOW, + ) + finish_generation( + test_db, + knowledge_base=knowledge_base, + generation=started.generation, + user=test_user, + effects=effects.build(), + succeeded=False, + now=NOW, + ) + + assert published_commit(test_db, knowledge_base) == HEAD + + retry = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + now=NOW, + ) + + assert retry.started + + +def test_the_run_mode_reason_is_kept_for_troubleshooting( + test_db: Session, knowledge_base: Kind, test_user: User +): + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=HEAD, + now=NOW, + ) + + assert started.generation.ext["runModeReason"] + + +def test_a_wiki_running_on_increments_forever_is_eventually_rebuilt( + test_db: Session, knowledge_base: Kind, test_user: User, effects: FakeEffects +): + """decide_run_mode has had this branch from the start and was never given the + inputs, so it defaulted to "no drift yet" and the rebuild never came however + long the wiki ran on increments.""" + from app.models.wiki import WikiGenerationType + from app.services.knowledge.code_wiki.run_mode import DEFAULT_POLICY + + _publish_a_first_wiki(test_db, knowledge_base, test_user, effects) + + for index in range(DEFAULT_POLICY.max_incrementals_since_full + 1): + done = WikiGeneration( + project_id=0, + kind_id=knowledge_base.id, + user_id=test_user.id, + task_id=0, + team_id=0, + generation_type=WikiGenerationType.INCREMENTAL, + source_snapshot={"commit": f"c{index}"}, + status=WikiGenerationStatus.COMPLETED, + completed_at=NOW + timedelta(minutes=index + 1), + ) + test_db.add(done) + test_db.flush() + + started = start_generation( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + now=NOW + timedelta(days=1), + ) + + assert RunMode(started.decision.mode) is RunMode.FULL + assert "incremental" in started.decision.reason.lower() diff --git a/backend/tests/services/knowledge/code_wiki/test_mermaid_check.py b/backend/tests/services/knowledge/code_wiki/test_mermaid_check.py new file mode 100644 index 0000000000..1f49266a28 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_mermaid_check.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for structural checks on Mermaid diagrams.""" + +import pytest + +from app.services.knowledge.code_wiki.mermaid_check import ( + check_mermaid_blocks, + describe_warnings, +) + + +def _fence(body: str, info: str = "mermaid") -> str: + return f"```{info}\n{body}\n```" + + +def test_a_valid_diagram_produces_no_warnings(): + markdown = "# Architecture\n\n" + _fence("flowchart TD\n A[Start] --> B[End]") + + assert check_mermaid_blocks(markdown) == [] + + +def test_documents_without_diagrams_produce_no_warnings(): + assert check_mermaid_blocks("# Just prose\n\nNo diagrams here.") == [] + + +def test_a_misspelled_diagram_type_is_reported(): + markdown = _fence("flowchat TD\n A --> B") + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "flowchat" in warnings[0].message + assert "not render" in warnings[0].message + + +def test_an_unclosed_fence_is_reported(): + markdown = "```mermaid\nflowchart TD\n A --> B\n" + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "never closed" in warnings[0].message + + +def test_an_empty_diagram_is_reported(): + markdown = _fence("") + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "empty" in warnings[0].message + + +def test_an_unbalanced_bracket_is_reported(): + markdown = _fence("flowchart TD\n A[Start --> B[End]") + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "unclosed '['" in warnings[0].message + + +def test_a_mismatched_closing_bracket_is_reported(): + markdown = _fence("flowchart TD\n A[Start) --> B") + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "mismatched" in warnings[0].message + + +def test_brackets_inside_quoted_labels_are_not_misread(): + markdown = _fence('flowchart TD\n A["build (release)"] --> B["ship"]') + + assert check_mermaid_blocks(markdown) == [] + + +def test_an_unclosed_quote_is_reported(): + markdown = _fence('flowchart TD\n A["Start --> B') + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "unclosed quote" in warnings[0].message + + +def test_comments_before_the_diagram_type_are_skipped(): + markdown = _fence("%% layout notes\nflowchart LR\n A --> B") + + assert check_mermaid_blocks(markdown) == [] + + +def test_diagram_type_with_a_direction_or_colon_is_accepted(): + assert check_mermaid_blocks(_fence("graph LR\n A --> B")) == [] + assert check_mermaid_blocks(_fence("stateDiagram-v2\n [*] --> Idle")) == [] + assert check_mermaid_blocks(_fence("pie title Share\n 'a' : 10")) == [] + + +def test_a_mermaid_example_quoted_inside_another_fence_is_not_checked(): + """A code sample showing broken Mermaid is documentation, not a diagram.""" + markdown = "````markdown\n```mermaid\nflowchat TD\n```\n````" + + assert check_mermaid_blocks(markdown) == [] + + +def test_entity_relationship_cardinality_is_not_mistaken_for_a_bracket_error(): + """ER diagrams write cardinality as ``||--o{``, which only looks unbalanced. + + Reporting it would send the agent off to fix a diagram that already renders. + """ + markdown = _fence( + "erDiagram\n" + " CUSTOMER ||--o{ ORDER : places\n" + " ORDER }|..|{ LINE_ITEM : contains" + ) + + assert check_mermaid_blocks(markdown) == [] + + +def test_text_heavy_diagrams_may_contain_stray_brackets_in_labels(): + assert check_mermaid_blocks(_fence("gantt\n title Release [Q3\n")) == [] + assert check_mermaid_blocks(_fence("journey\n Browse [start: 3")) == [] + + +def test_every_diagram_in_a_page_is_checked(): + markdown = "\n\n".join( + [ + _fence("flowchart TD\n A --> B"), + _fence("nonsense\n A --> B"), + _fence("sequenceDiagram\n Alice->>Bob: hi"), + _fence("erDiagram\n A ||--o{ B : has"), + ] + ) + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "nonsense" in warnings[0].message + + +def test_warnings_carry_the_line_of_their_diagram(): + markdown = "# Title\n\nProse.\n\n" + _fence("flowchat TD\n A --> B") + + warnings = check_mermaid_blocks(markdown) + + assert warnings[0].line == 5 + + +def test_warnings_render_as_an_instruction_for_the_agent(): + warnings = check_mermaid_blocks(_fence("flowchat TD\n A --> B")) + + described = describe_warnings(warnings) + + assert "write the page again" in described + assert "flowchat" in described + + +def test_no_warnings_render_as_nothing(): + assert describe_warnings([]) == "" + + +def test_a_shorter_fence_does_not_close_a_longer_one(): + """A closing fence must be at least as long as the one it opens. + + Comparing only the fence character lets an inner ``` end an outer ````, which is + exactly how a diagram gets quoted rather than declared. Here the document is + genuinely unterminated, and treating the short fence as a close would hide that. + """ + markdown = "````mermaid\nflowchart TD\n A --> B\n```\n" + + warnings = check_mermaid_blocks(markdown) + + assert len(warnings) == 1 + assert "never closed" in warnings[0].message + + +def test_a_longer_fence_closes_normally(): + markdown = "````mermaid\nflowchart TD\n A --> B\n````\n" + + assert check_mermaid_blocks(markdown) == [] + + +@pytest.mark.parametrize( + "declaration", + ["kanban", "radar", "treemap", "classDiagram-v2", "flowchart-elk", "C4Container"], +) +def test_declarations_the_pinned_mermaid_supports_are_not_reported(declaration: str): + """The frontend resolves Mermaid 11.15, and a type missing from the allow-list is + sent back to the agent as a broken diagram — a wasted round spent "fixing" one + that renders.""" + assert check_mermaid_blocks(_fence(f"{declaration}\n A --> B")) == [] + + +def test_frontmatter_does_not_hide_the_diagram_type(): + """Mermaid allows `--- ... ---` before the type, and several of the types the + pinned version added use it. Read as the diagram, the opening `---` becomes the + type and something that renders is reported as broken.""" + body = "---\nconfig:\n theme: dark\n---\nradar-beta\n axis a, b" + + assert check_mermaid_blocks(_fence(body)) == [] + + +def test_an_unterminated_dash_line_is_not_treated_as_frontmatter(): + """Otherwise the whole diagram would be swallowed and nothing checked.""" + warnings = check_mermaid_blocks(_fence("---\nflowchat TD")) + + assert warnings diff --git a/backend/tests/services/knowledge/code_wiki/test_navigation.py b/backend/tests/services/knowledge/code_wiki/test_navigation.py new file mode 100644 index 0000000000..2103be10b8 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_navigation.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the navigation a reader is given. + +The hierarchy lives in the paths and the order on the knowledge base, and the client +renders what it is handed rather than merging the two itself. So what is pinned here +is that the two arrive already reconciled — including the case neither of them +describes on its own: a section that holds pages but is not one. +""" + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import ContentOrigin, KnowledgeDocument +from app.services.knowledge.code_wiki.navigation import page_tree +from app.services.knowledge.code_wiki.projection_plan import PAGE_PATH_KEY +from app.services.knowledge.code_wiki.publisher import PAGE_ORDER_KEY + +KIND_ID = 771 + + +@pytest.fixture +def knowledge_base(test_db: Session) -> Kind: + kind = Kind( + id=KIND_ID, + kind="KnowledgeBase", + name="kb-nav", + namespace="default", + user_id=1, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def _page(test_db: Session, path: str, title: str) -> KnowledgeDocument: + document = KnowledgeDocument( + kind_id=KIND_ID, + attachment_id=1, + name=title, + file_extension="md", + file_size=1, + user_id=1, + folder_id=0, + origin=ContentOrigin.GENERATED.value, + source_config={PAGE_PATH_KEY: path}, + ) + test_db.add(document) + test_db.flush() + return document + + +def _order(test_db: Session, knowledge_base: Kind, paths: list[str]) -> None: + payload = dict(knowledge_base.json or {}) + spec = dict(payload.get("spec", {})) + spec[PAGE_ORDER_KEY] = paths + payload["spec"] = spec + knowledge_base.json = payload + test_db.flush() + + +def _shape(nodes) -> list: + return [(node.path, _shape(node.children)) for node in nodes] + + +def test_a_page_nests_under_the_page_whose_path_prefixes_it( + test_db: Session, knowledge_base: Kind +): + _page(test_db, "architecture", "Architecture") + _page(test_db, "architecture/backend", "Backend") + + assert _shape(page_tree(test_db, knowledge_base)) == [ + ("architecture", [("architecture/backend", [])]) + ] + + +def test_a_flat_wiki_needs_no_special_case(test_db: Session, knowledge_base: Kind): + """A simple repository has one level, and that has to work as it stands.""" + _page(test_db, "index", "Overview") + _page(test_db, "setup", "Setup") + _order(test_db, knowledge_base, ["index", "setup"]) + + assert _shape(page_tree(test_db, knowledge_base)) == [("index", []), ("setup", [])] + + +def test_the_declared_order_is_what_the_reader_gets( + test_db: Session, knowledge_base: Kind +): + """Alphabetically the API reference precedes the overview, and a wiki read in + that order reads wrong.""" + for path in ("api", "index", "architecture"): + _page(test_db, path, path) + _order(test_db, knowledge_base, ["index", "architecture", "api"]) + + assert [node.path for node in page_tree(test_db, knowledge_base)] == [ + "index", + "architecture", + "api", + ] + + +def test_children_are_ordered_the_same_way_as_the_top_level( + test_db: Session, knowledge_base: Kind +): + _page(test_db, "guide", "Guide") + _page(test_db, "guide/second", "Second") + _page(test_db, "guide/first", "First") + _order(test_db, knowledge_base, ["guide", "guide/first", "guide/second"]) + + (guide,) = page_tree(test_db, knowledge_base) + assert [child.path for child in guide.children] == ["guide/first", "guide/second"] + + +def test_a_section_with_no_page_of_its_own_still_holds_its_pages( + test_db: Session, knowledge_base: Kind +): + """The publish gate allows this, so the reader has to render it. Without the + node its pages would surface at the top level, reading as though unrelated.""" + _page(test_db, "architecture/backend", "Backend") + + tree = page_tree(test_db, knowledge_base) + + assert _shape(tree) == [("architecture", [("architecture/backend", [])])] + assert tree[0].has_content is False + assert tree[0].children[0].has_content is True + + +def test_a_missing_section_several_levels_up_is_filled_in( + test_db: Session, knowledge_base: Kind +): + _page(test_db, "a/b/c", "Deep") + + assert _shape(page_tree(test_db, knowledge_base)) == [ + ("a", [("a/b", [("a/b/c", [])])]) + ] + + +def test_an_unranked_page_follows_the_ranked_ones( + test_db: Session, knowledge_base: Kind +): + """A page added without updating the order must still appear, somewhere + predictable rather than at a position nobody chose.""" + _page(test_db, "index", "Overview") + _page(test_db, "stray", "Stray") + _order(test_db, knowledge_base, ["index"]) + + assert [node.path for node in page_tree(test_db, knowledge_base)] == [ + "index", + "stray", + ] + + +def test_a_document_without_a_page_path_is_not_navigable( + test_db: Session, knowledge_base: Kind +): + """It has no place in a tree built from paths, and guessing one would put it + somewhere the next publish disagrees with.""" + document = _page(test_db, "index", "Overview") + stray = _page(test_db, "other", "Other") + stray.source_config = {} + test_db.flush() + + assert [node.document_id for node in page_tree(test_db, knowledge_base)] == [ + document.id + ] diff --git a/backend/tests/services/knowledge/code_wiki/test_page_path.py b/backend/tests/services/knowledge/code_wiki/test_page_path.py new file mode 100644 index 0000000000..c06dcc975a --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_page_path.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for wiki page path identity. + +The path is what keeps a page's document id — and therefore its RAG index entry and +any stored citation — stable across regenerations, so these tests are mostly about +refusing input that would silently resolve to a different page. +""" + +import pytest + +from app.services.knowledge.code_wiki.page_path import ( + MAX_DIRECTORY_DEPTH, + MAX_PATH_LENGTH, + MAX_SEGMENT_LENGTH, + InvalidPagePath, + assert_unique_within_version, + collation_key, + normalize_page_path, + split_page_path, +) + + +def test_a_plain_path_is_kept_as_is(): + assert normalize_page_path("architecture/backend") == "architecture/backend" + + +def test_surrounding_whitespace_and_repeated_separators_are_meaningless(): + assert normalize_page_path(" architecture//backend/ ") == "architecture/backend" + + +def test_a_trailing_markdown_suffix_is_accepted_and_dropped(): + """A model writing markdown will include it; the projection appends it itself.""" + assert normalize_page_path("architecture/backend.md") == "architecture/backend" + assert normalize_page_path("architecture/backend.MD") == "architecture/backend" + + +def test_a_suffix_inside_the_name_is_left_alone(): + assert normalize_page_path("notes/readme.md.draft") == "notes/readme.md.draft" + + +def test_an_empty_path_is_rejected(): + with pytest.raises(InvalidPagePath): + normalize_page_path(" ") + + +def test_an_absolute_path_is_rejected_rather_than_stripped(): + """Stripping the slash would quietly turn one page into another.""" + with pytest.raises(InvalidPagePath, match="relative"): + normalize_page_path("/architecture/backend") + + +def test_relative_segments_are_rejected(): + with pytest.raises(InvalidPagePath, match="relative segments"): + normalize_page_path("architecture/../../etc/passwd") + + +@pytest.mark.parametrize("raw", ["a/...md", "a/..md", "...md", "..md"]) +def test_a_suffix_strip_cannot_reintroduce_a_relative_segment(raw: str): + """Stripping ".md" from "...md" yields "..", which must not survive. + + Validation therefore runs after the suffix is removed. Checking first and + stripping afterwards let a leaf pass the relative-segment check and then become + the exact value that check exists to reject. + """ + with pytest.raises(InvalidPagePath, match="relative segments"): + normalize_page_path(raw) + + +def test_a_backslash_separator_is_rejected_as_ambiguous(): + with pytest.raises(InvalidPagePath, match="separator"): + normalize_page_path("architecture\\backend") + + +def test_control_characters_are_rejected(): + with pytest.raises(InvalidPagePath, match="control characters"): + normalize_page_path("architecture/back\x00end") + + +def test_reserved_characters_are_rejected(): + with pytest.raises(InvalidPagePath, match="reserved character"): + normalize_page_path('architecture/back"end') + + +def test_an_over_long_segment_is_rejected(): + with pytest.raises(InvalidPagePath, match="segment exceeds"): + normalize_page_path("x" * (MAX_SEGMENT_LENGTH + 1)) + + +def test_an_over_long_path_is_rejected(): + # Stay within the segment and depth limits so the length check is what fires. + segments = ["y" * 120] * 4 + with pytest.raises(InvalidPagePath, match="exceeds"): + normalize_page_path("/".join(segments) + "/" + "z" * 120) + + +def test_a_path_at_the_folder_depth_limit_is_accepted(): + path = "/".join(["d"] * MAX_DIRECTORY_DEPTH) + "/page" + + assert normalize_page_path(path) == path + + +def test_a_path_deeper_than_the_folder_tree_allows_is_rejected(): + """Rejected on write, so one bad path cannot fail an entire version at publish.""" + path = "/".join(["d"] * (MAX_DIRECTORY_DEPTH + 1)) + "/page" + + with pytest.raises(InvalidPagePath, match="folder tree allows"): + normalize_page_path(path) + + +def test_a_root_level_page_has_no_folders(): + assert split_page_path("index") == ((), "index") + + +def test_splitting_separates_folders_from_the_document_name(): + assert split_page_path("a/b/page") == (("a", "b"), "page") + + +def test_paths_differing_only_by_case_are_the_same_page(): + """The knowledge tables collate case-insensitively, so the database agrees.""" + assert collation_key("Architecture/Backend") == collation_key( + "architecture/backend" + ) + + +def test_a_version_may_not_contain_two_paths_that_collide_by_case(): + with pytest.raises(InvalidPagePath, match="differ only by case"): + assert_unique_within_version(["architecture/backend", "Architecture/Backend"]) + + +def test_a_version_may_not_repeat_a_path(): + with pytest.raises(InvalidPagePath, match="more than once"): + assert_unique_within_version(["architecture/backend", "architecture/backend"]) + + +def test_distinct_paths_pass(): + assert_unique_within_version(["index", "architecture/backend", "modules/api"]) + + +def test_the_path_limit_leaves_room_for_a_realistic_layout(): + """Guards against a limit tightened to the point of rejecting ordinary pages.""" + path = "architecture/backend/services/knowledge/document-indexing-pipeline" + + assert len(path) < MAX_PATH_LENGTH + assert normalize_page_path(path) == path diff --git a/backend/tests/services/knowledge/code_wiki/test_projection.py b/backend/tests/services/knowledge/code_wiki/test_projection.py new file mode 100644 index 0000000000..ebc621cca6 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_projection.py @@ -0,0 +1,500 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for applying a projection plan. + +Attachment bytes live outside the transaction, so the order in which they are written +and deleted decides whether a failed publish leaves litter or destroys content. These +tests record the sequence of side effects and assert on it directly. +""" + +from dataclasses import dataclass, field + +import pytest +from sqlalchemy.orm import Session + +from app.models.knowledge import ContentOrigin, KnowledgeDocument, KnowledgeFolder +from app.services.knowledge.code_wiki.projection import ( + ProjectionSideEffects, + apply_projection_plan, + finish_projection, +) +from app.services.knowledge.code_wiki.projection_plan import ( + CONTENT_HASH_KEY, + PAGE_PATH_KEY, + PageSource, + ProjectedPage, + compute_projection_plan, + content_fingerprint, +) + +KIND_ID = 501 +USER_ID = 9 + + +@dataclass +class RecordingEffects: + """Records every side effect in order, so ordering can be asserted on.""" + + calls: list[tuple[str, object]] = field(default_factory=list) + next_attachment_id: int = 1000 + failing_rag_refs: set[int] = field(default_factory=set) + + def as_side_effects(self) -> ProjectionSideEffects: + return ProjectionSideEffects( + write_attachment=self._write, + delete_attachment=self._delete_attachment, + delete_rag_document=self._delete_rag, + enqueue_reindex=self._enqueue, + ) + + def _write(self, *, filename: str, content: str) -> int: + self.next_attachment_id += 1 + self.calls.append(("write_attachment", filename)) + return self.next_attachment_id + + def _delete_attachment(self, attachment_id: int) -> None: + self.calls.append(("delete_attachment", attachment_id)) + + def _delete_rag(self, doc_ref: int) -> None: + if doc_ref in self.failing_rag_refs: + raise RuntimeError("vector store unavailable") + self.calls.append(("delete_rag", doc_ref)) + + def _enqueue(self, document_id: int) -> None: + self.calls.append(("enqueue_reindex", document_id)) + + def names(self) -> list[str]: + return [name for name, _ in self.calls] + + +@pytest.fixture +def effects() -> RecordingEffects: + return RecordingEffects() + + +def _source(path: str, content: str = "body") -> PageSource: + return PageSource(path=path, title=path.rsplit("/", 1)[-1], content=content) + + +def _existing_document( + db: Session, path: str, content: str, *, attachment_id: int = 1 +) -> KnowledgeDocument: + folders = path.split("/")[:-1] + parent_id = 0 + for segment in folders: + folder = KnowledgeFolder( + kind_id=KIND_ID, + parent_id=parent_id, + name=segment, + origin=ContentOrigin.GENERATED.value, + ) + db.add(folder) + db.flush() + parent_id = folder.id + + # Mirrors _source: the title is the leaf, so an existing row and a fresh source + # for the same path fingerprint identically and count as unchanged. + title = path.rsplit("/", 1)[-1] + document = KnowledgeDocument( + kind_id=KIND_ID, + attachment_id=attachment_id, + name=title, + file_extension="md", + file_size=len(content), + user_id=USER_ID, + folder_id=parent_id, + origin=ContentOrigin.GENERATED.value, + source_config={ + PAGE_PATH_KEY: path, + CONTENT_HASH_KEY: content_fingerprint(title, content), + }, + ) + db.add(document) + db.flush() + return document + + +def _projected(document: KnowledgeDocument) -> ProjectedPage: + config = document.source_config or {} + return ProjectedPage( + document_id=document.id, + path=config[PAGE_PATH_KEY], + content_hash=config[CONTENT_HASH_KEY], + ) + + +def _apply(db: Session, plan, effects: RecordingEffects): + return apply_projection_plan( + db, + kind_id=KIND_ID, + user_id=USER_ID, + plan=plan, + effects=effects.as_side_effects(), + ) + + +# --- ordering -------------------------------------------------------------- + + +def test_attachments_are_written_before_any_row_changes( + test_db: Session, effects: RecordingEffects +): + """Writing first costs an orphaned object on failure; the reverse costs data.""" + plan = compute_projection_plan([_source("index")], []) + + _apply(test_db, plan, effects) + + assert effects.names() == ["write_attachment"] + + +def test_no_attachment_is_deleted_before_the_transaction_commits( + test_db: Session, effects: RecordingEffects +): + """A rollback after deleting bytes cannot bring them back.""" + existing = _existing_document(test_db, "index", "old", attachment_id=77) + plan = compute_projection_plan([_source("index", "new")], [_projected(existing)]) + + _apply(test_db, plan, effects) + + assert "delete_attachment" not in effects.names() + + +def test_cleanup_and_reindex_only_run_after_the_commit( + test_db: Session, effects: RecordingEffects +): + existing = _existing_document(test_db, "index", "old", attachment_id=77) + plan = compute_projection_plan([_source("index", "new")], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + before_commit = list(effects.names()) + finish_projection( + outcome, superseded_attachment_ids=[77], effects=effects.as_side_effects() + ) + + assert before_commit == ["write_attachment"] + assert effects.names()[1:] == ["delete_attachment", "enqueue_reindex"] + + +# --- what each case does --------------------------------------------------- + + +def test_an_added_page_becomes_an_inactive_document_in_its_folder( + test_db: Session, effects: RecordingEffects +): + plan = compute_projection_plan([_source("architecture/backend")], []) + + outcome = _apply(test_db, plan, effects) + + document = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]) + assert document.name == "backend" + assert document.origin == ContentOrigin.GENERATED.value + # Left off until indexing succeeds, exactly as any other document is. + assert document.is_active is False + folder = test_db.get(KnowledgeFolder, document.folder_id) + assert folder.name == "architecture" + assert folder.origin == ContentOrigin.GENERATED.value + + +def test_an_updated_page_keeps_its_document_id( + test_db: Session, effects: RecordingEffects +): + """The reason path identity exists: the RAG index is keyed on this id.""" + existing = _existing_document(test_db, "index", "old", attachment_id=77) + original_id = existing.id + plan = compute_projection_plan([_source("index", "new")], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + + assert outcome.updated_document_ids == (original_id,) + + +def test_an_updated_page_repoints_instead_of_overwriting( + test_db: Session, effects: RecordingEffects +): + """Overwriting happens before the commit, so a rollback would lose both versions.""" + existing = _existing_document(test_db, "index", "old", attachment_id=77) + plan = compute_projection_plan([_source("index", "new")], [_projected(existing)]) + + _apply(test_db, plan, effects) + + assert existing.attachment_id != 77 + + +def test_a_page_whose_row_vanished_is_added_back_rather_than_lost( + test_db: Session, effects: RecordingEffects +): + """The plan counted it as an update, so skipping it would leave the published + version short a page and strand the attachment already written for it.""" + existing = _existing_document(test_db, "index", "old", attachment_id=77) + plan = compute_projection_plan([_source("index", "new")], [_projected(existing)]) + test_db.delete(existing) + test_db.flush() + + outcome = _apply(test_db, plan, effects) + + assert outcome.updated_document_ids == () + assert len(outcome.created_document_ids) == 1 + restored = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]) + assert restored.name == "index" + + +def test_a_removed_page_records_its_doc_ref_before_the_row_is_gone( + test_db: Session, effects: RecordingEffects +): + """After the commit there is nothing left to derive the RAG key from.""" + existing = _existing_document(test_db, "modules/legacy", "body") + doomed_id = existing.id + plan = compute_projection_plan([], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + + assert outcome.deleted_document_ids == (doomed_id,) + assert outcome.unfinished_index_cleanup == (str(doomed_id),) + assert test_db.get(KnowledgeDocument, doomed_id) is None + + +def test_an_unchanged_page_causes_no_side_effects_at_all( + test_db: Session, effects: RecordingEffects +): + existing = _existing_document(test_db, "index", "same") + plan = compute_projection_plan([_source("index", "same")], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + + assert effects.calls == [] + assert outcome.created_document_ids == () + assert outcome.updated_document_ids == () + + +def test_the_fingerprint_is_stamped_so_the_next_run_can_skip( + test_db: Session, effects: RecordingEffects +): + plan = compute_projection_plan([_source("index", "v1")], []) + + outcome = _apply(test_db, plan, effects) + + document = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]) + assert document.source_config[CONTENT_HASH_KEY] == content_fingerprint( + "index", "v1" + ) + assert document.source_config[PAGE_PATH_KEY] == "index" + + +# --- folders --------------------------------------------------------------- + + +def test_a_folder_emptied_by_deletion_is_removed( + test_db: Session, effects: RecordingEffects +): + existing = _existing_document(test_db, "modules/legacy", "body") + plan = compute_projection_plan([], [_projected(existing)]) + + _apply(test_db, plan, effects) + + assert ( + test_db.query(KnowledgeFolder) + .filter(KnowledgeFolder.kind_id == KIND_ID) + .count() + == 0 + ) + + +def test_a_whole_branch_emptied_by_deletion_is_removed( + test_db: Session, effects: RecordingEffects +): + """Emptying a leaf empties its parent, and that one its parent. Judging a folder + before its children would keep every level above the last page alive.""" + existing = _existing_document(test_db, "architecture/backend/api", "body") + plan = compute_projection_plan([], [_projected(existing)]) + + _apply(test_db, plan, effects) + + assert ( + test_db.query(KnowledgeFolder) + .filter(KnowledgeFolder.kind_id == KIND_ID) + .count() + == 0 + ) + + +def test_a_branch_is_kept_while_any_page_below_it_survives( + test_db: Session, effects: RecordingEffects +): + """The other half of the same rule: a deep survivor keeps its whole ancestry.""" + doomed = _existing_document(test_db, "architecture/legacy", "body") + kept = _existing_document(test_db, "architecture/backend/api", "body") + plan = compute_projection_plan( + [_source("architecture/backend/api", "body")], + [_projected(doomed), _projected(kept)], + ) + + _apply(test_db, plan, effects) + + names = { + folder.name + for folder in test_db.query(KnowledgeFolder) + .filter(KnowledgeFolder.kind_id == KIND_ID) + .all() + } + assert names == {"architecture", "backend"} + + +def test_an_empty_user_folder_is_left_alone( + test_db: Session, effects: RecordingEffects +): + """Tidying it away would be the projection reaching outside what it owns.""" + user_folder = KnowledgeFolder( + kind_id=KIND_ID, + parent_id=0, + name="my notes", + origin=ContentOrigin.USER.value, + ) + test_db.add(user_folder) + test_db.flush() + + _apply(test_db, compute_projection_plan([], []), effects) + + assert test_db.get(KnowledgeFolder, user_folder.id) is not None + + +def test_folders_are_reused_rather_than_duplicated( + test_db: Session, effects: RecordingEffects +): + plan = compute_projection_plan( + [_source("architecture/backend"), _source("architecture/frontend")], [] + ) + + _apply(test_db, plan, effects) + + assert ( + test_db.query(KnowledgeFolder) + .filter( + KnowledgeFolder.kind_id == KIND_ID, KnowledgeFolder.name == "architecture" + ) + .count() + == 1 + ) + + +# --- retriable cleanup ----------------------------------------------------- + + +def test_a_failed_rag_deletion_is_reported_for_retry_not_raised( + test_db: Session, effects: RecordingEffects +): + """The pages are already correct; failing the publish would regenerate them all.""" + existing = _existing_document(test_db, "modules/legacy", "body") + doomed_id = existing.id + effects.failing_rag_refs.add(doomed_id) + plan = compute_projection_plan([], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + unfinished = finish_projection( + outcome, superseded_attachment_ids=[], effects=effects.as_side_effects() + ) + + assert unfinished == (str(doomed_id),) + + +def test_successful_cleanup_leaves_nothing_to_retry( + test_db: Session, effects: RecordingEffects +): + existing = _existing_document(test_db, "modules/legacy", "body") + plan = compute_projection_plan([], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + unfinished = finish_projection( + outcome, superseded_attachment_ids=[], effects=effects.as_side_effects() + ) + + assert unfinished == () + + +def test_user_content_is_never_touched(test_db: Session, effects: RecordingEffects): + """It is not regenerable, so a mistaken delete here is unrecoverable.""" + user_document = KnowledgeDocument( + kind_id=KIND_ID, + attachment_id=5, + name="my note", + file_extension="md", + file_size=4, + user_id=USER_ID, + folder_id=0, + origin=ContentOrigin.USER.value, + ) + test_db.add(user_document) + test_db.flush() + + # The caller scopes ``existing`` to generated pages, so user content simply is + # not in the comparison — this asserts the projection does not find it anyway. + _apply(test_db, compute_projection_plan([], []), effects) + + assert test_db.get(KnowledgeDocument, user_document.id) is not None + + +# --- what a page is called -------------------------------------------------- + + +def test_a_page_is_named_after_its_title_not_its_path( + test_db: Session, effects: RecordingEffects +): + """The title used to be discarded, so "Backend Architecture" at + architecture/backend became a document called "backend" — and two pages at + architecture/backend and services/backend were indistinguishable anywhere the + path was not also shown.""" + source = PageSource( + path="architecture/backend", title="Backend Architecture", content="body" + ) + plan = compute_projection_plan([source], []) + + outcome = _apply(test_db, plan, effects) + + document = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]) + assert document.name == "Backend Architecture" + # The path stays the identity, untouched by what the page is called. + assert document.source_config[PAGE_PATH_KEY] == "architecture/backend" + + +def test_a_page_without_a_title_falls_back_to_its_path( + test_db: Session, effects: RecordingEffects +): + """A page has to be called something, and refusing the publish over a missing + heading would discard a version that is otherwise fine.""" + source = PageSource(path="architecture/backend", title=" ", content="body") + plan = compute_projection_plan([source], []) + + outcome = _apply(test_db, plan, effects) + + assert test_db.get(KnowledgeDocument, outcome.created_document_ids[0]).name == ( + "backend" + ) + + +def test_an_overlong_title_is_truncated_rather_than_failing_the_insert( + test_db: Session, effects: RecordingEffects +): + """The column is 255 wide; a long heading must not take the whole publish down.""" + source = PageSource(path="index", title="x" * 400, content="body") + plan = compute_projection_plan([source], []) + + outcome = _apply(test_db, plan, effects) + + name = test_db.get(KnowledgeDocument, outcome.created_document_ids[0]).name + assert len(name) == 255 + + +def test_rewording_a_title_renames_the_document_in_place( + test_db: Session, effects: RecordingEffects +): + """It keeps its id, so the RAG index entry and any stored citation survive.""" + existing = _existing_document(test_db, "index", "body") + original_id = existing.id + renamed = PageSource(path="index", title="A Better Heading", content="body") + plan = compute_projection_plan([renamed], [_projected(existing)]) + + outcome = _apply(test_db, plan, effects) + + assert outcome.updated_document_ids == (original_id,) + assert test_db.get(KnowledgeDocument, original_id).name == "A Better Heading" diff --git a/backend/tests/services/knowledge/code_wiki/test_projection_plan.py b/backend/tests/services/knowledge/code_wiki/test_projection_plan.py new file mode 100644 index 0000000000..b576f880d2 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_projection_plan.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the projection plan. + +The plan decides what publishing a version does to the knowledge base, including what +it deletes, so these tests are mostly about it not reaching further than it should. +""" + +from app.services.knowledge.code_wiki.projection_plan import ( + PageSource, + ProjectedPage, + compute_projection_plan, + content_fingerprint, +) + + +def _source(path: str, content: str = "body") -> PageSource: + return PageSource(path=path, title=path.rsplit("/", 1)[-1], content=content) + + +def _projected( + document_id: int, path: str, content: str = "body", title: str | None = None +) -> ProjectedPage: + return ProjectedPage( + document_id=document_id, + path=path, + content_hash=content_fingerprint(title or path.rsplit("/", 1)[-1], content), + ) + + +def test_a_page_only_the_version_has_is_added(): + plan = compute_projection_plan([_source("index")], []) + + assert [page.path for page in plan.adds] == ["index"] + assert plan.updates == () + assert plan.deletes == () + + +def test_a_page_whose_content_changed_is_updated(): + plan = compute_projection_plan( + [_source("index", "new")], [_projected(1, "index", "old")] + ) + + assert [update.existing.document_id for update in plan.updates] == [1] + assert plan.adds == () + + +def test_an_unchanged_page_is_skipped_entirely(): + """Where an incremental run's savings come from: no write, no reindex, no row.""" + plan = compute_projection_plan( + [_source("index", "same")], [_projected(1, "index", "same")] + ) + + assert plan.skips == ("index",) + assert plan.adds == () + assert plan.updates == () + assert plan.deletes == () + assert plan.is_empty + + +def test_a_page_only_the_knowledge_base_has_is_deleted(): + plan = compute_projection_plan([], [_projected(7, "modules/legacy")]) + + assert [page.document_id for page in plan.deletes] == [7] + + +def test_a_moved_page_is_an_add_and_a_delete(): + """Identity is the path, so a move is not an in-place rename.""" + plan = compute_projection_plan( + [_source("services/backend")], [_projected(3, "architecture/backend")] + ) + + assert [page.path for page in plan.adds] == ["services/backend"] + assert [page.document_id for page in plan.deletes] == [3] + + +def test_matching_ignores_case_because_the_database_does(): + plan = compute_projection_plan( + [_source("Architecture/Backend", "same")], + # Same title on both sides: this is about the path matching, and letting the + # helper derive two differently-cased titles would test something else. + [_projected(1, "architecture/backend", "same", title="Backend")], + ) + + assert plan.skips == ("Architecture/Backend",) + assert plan.deletes == () + + +def test_a_rewritten_title_alone_still_updates_the_page(): + """The title is what the document is named, so a reworded heading has to reach + the knowledge base. It used to be discarded, and comparing content alone was + right then; now that trade-off is inverted — the cost is rewriting an attachment + whose bytes did not change, and a title almost always moves with its body.""" + existing = _projected(1, "index", "unchanged body", title="Index") + renamed = PageSource( + path="index", title="A Better Heading", content="unchanged body" + ) + + plan = compute_projection_plan([renamed], [existing]) + + assert plan.skips == () + assert [update.source.title for update in plan.updates] == ["A Better Heading"] + + +def test_an_empty_version_removes_everything_it_owns(): + """Deliberate: the publish gate, not the plan, decides whether that is acceptable.""" + plan = compute_projection_plan( + [], [_projected(1, "a"), _projected(2, "b"), _projected(3, "c")] + ) + + assert len(plan.deletes) == 3 + + +def test_a_mixed_version_is_reported_in_full(): + plan = compute_projection_plan( + [ + _source("index", "changed"), + _source("architecture/backend", "same"), + _source("modules/new"), + ], + [ + _projected(1, "index", "original"), + _projected(2, "architecture/backend", "same"), + _projected(3, "modules/gone"), + ], + ) + + assert [page.path for page in plan.adds] == ["modules/new"] + assert [update.existing.document_id for update in plan.updates] == [1] + assert plan.skips == ("architecture/backend",) + assert [page.document_id for page in plan.deletes] == [3] + assert plan.touched_pages == 3 + assert plan.describe() == "1 added, 1 updated, 1 removed, 1 unchanged" + + +def test_an_identical_snapshot_produces_no_work(): + pages = [_projected(1, "index", "x"), _projected(2, "a/b", "y")] + sources = [_source("index", "x"), _source("a/b", "y")] + + assert compute_projection_plan(sources, pages).is_empty + + +def test_the_fingerprint_distinguishes_content(): + assert content_fingerprint("T", "a") != content_fingerprint("T", "b") + assert content_fingerprint("T", "a") == content_fingerprint("T", "a") diff --git a/backend/tests/services/knowledge/code_wiki/test_prompts.py b/backend/tests/services/knowledge/code_wiki/test_prompts.py new file mode 100644 index 0000000000..7fc979d06f --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_prompts.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the wiki generation instructions. + +Prompt quality cannot be asserted, but its contract can: the projection depends on +pages arriving with stable paths and complete content, and the two modes differ in +whether an unwritten page means "unchanged" or "gone". Getting either wrong is a data +loss, so those are pinned here. +""" + +from app.services.knowledge.code_wiki.prompts import ( + REQUIRED_COVERAGE, + WikiRunContext, + build_diagram_correction, + build_full_prompt, + build_incremental_prompt, + build_prompt, +) + + +def _context(**overrides) -> WikiRunContext: + defaults = dict(project_name="wecode-ai/Wegent", generation_id=42) + defaults.update(overrides) + return WikiRunContext(**defaults) + + +# --- the write contract ---------------------------------------------------- + + +def test_both_modes_require_a_stable_path(): + """The path is the page's identity; a shifting one republishes the page.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "path" in prompt + assert "stable" in prompt + + +def test_both_modes_ask_for_complete_content(): + """There is no patch format, so a partial page would replace a whole one.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "complete content" in prompt + + +def test_a_full_rebuild_says_an_unwritten_page_is_absent(): + prompt = build_full_prompt(_context()) + + assert "begins empty" in prompt + assert "every page the wiki should contain" in prompt + + +def test_an_incremental_run_says_an_unwritten_page_is_kept(): + """The opposite of a full run, and confusing them deletes or resurrects pages.""" + prompt = build_incremental_prompt(_context()) + + assert "copy of the published" in prompt + assert "does **not** remove it" in prompt + + +def test_only_the_incremental_mode_offers_explicit_removal(): + """A full run removes a page by not writing it; declaring it would be redundant.""" + assert "declare its removal" in build_incremental_prompt(_context()) + assert "declare its removal" not in build_full_prompt(_context()) + + +def test_the_path_limits_match_what_the_validator_enforces(): + """A prompt promising more than the validator accepts fails at write time.""" + prompt = build_full_prompt(_context()) + + assert "4 folders deep" in prompt + assert "differ only by case" in prompt + + +# --- content discipline ---------------------------------------------------- + + +def test_every_mode_forbids_inventing_evidence(): + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "Do not invent" in prompt + + +def test_every_mode_bounds_how_much_is_read(): + """Reading everything is how a run exhausts its budget before writing.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "Do not read every file" in prompt + + +def test_every_mode_asks_history_to_explain_why(): + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "why" in prompt + + +def test_every_mode_carries_the_relevance_test(): + """The one rule aimed at the generic, padded output the old wiki produced.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "would this change what someone does here" in prompt + + +def test_a_full_rebuild_plans_pages_and_links_before_writing(): + prompt = build_full_prompt(_context()) + + assert "Before writing any page" in prompt + assert "relationship" in prompt + + +def test_a_full_rebuild_states_the_expected_coverage(): + prompt = build_full_prompt(_context()) + + for item in REQUIRED_COVERAGE: + assert item in prompt + + +# --- run context ----------------------------------------------------------- + + +def test_the_incremental_prompt_lists_what_changed(): + prompt = build_incremental_prompt( + _context( + previous_commit="aaaa", + head_commit="bbbb", + changed_paths=["backend/app/main.py", "frontend/src/app.tsx"], + ) + ) + + assert "aaaa" in prompt and "bbbb" in prompt + assert "backend/app/main.py" in prompt + + +def test_the_incremental_prompt_lists_the_pages_that_exist(): + """Without it the agent cannot tell an update from a new page.""" + prompt = build_incremental_prompt(_context(existing_pages=["index", "arch/api"])) + + assert "arch/api" in prompt + + +def test_an_empty_change_list_is_stated_rather_than_left_blank(): + prompt = build_incremental_prompt(_context()) + + assert "no file list was available" in prompt + + +def test_an_empty_wiki_is_stated_rather_than_left_blank(): + prompt = build_incremental_prompt(_context()) + + assert "currently empty" in prompt + + +def test_the_generation_id_travels_with_the_run(): + assert "42" in build_full_prompt(_context()) + + +def test_the_mode_selects_the_prompt(): + assert build_prompt(_context(), full=True) == build_full_prompt(_context()) + assert build_prompt(_context(), full=False) == build_incremental_prompt(_context()) + + +# --- diagram feedback ------------------------------------------------------ + + +def test_diagram_problems_come_back_as_a_correction(): + correction = build_diagram_correction(["index: line 4 is not a diagram type"]) + + assert correction is not None + assert "index: line 4" in correction + assert "paths unchanged" in correction + + +def test_no_diagram_problems_means_no_follow_up(): + assert build_diagram_correction([]) is None + + +def test_both_modes_ask_for_a_page_at_every_section(): + """A section with no page becomes a heading a reader cannot open. Asking for it + here is what keeps the publish gate's warning rare.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "section that holds pages needs a page of its own" in prompt + + +def test_both_modes_explain_that_the_declared_order_is_what_readers_see(): + """Otherwise the field looks like bookkeeping and gets filled in arbitrarily.""" + for prompt in (build_full_prompt(_context()), build_incremental_prompt(_context())): + assert "--structure-order" in prompt + assert "order readers see" in prompt diff --git a/backend/tests/services/knowledge/code_wiki/test_publish_end_to_end.py b/backend/tests/services/knowledge/code_wiki/test_publish_end_to_end.py new file mode 100644 index 0000000000..4ca89ab85e --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_publish_end_to_end.py @@ -0,0 +1,322 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Publishing a version through the real attachment and indexing services. + +Every other test for this pipeline injects fakes, which proves the logic but not the +assumptions it rests on. This one uses the real attachment store and asserts on what +a reader would actually get, so that a wrong signature or a document that never +becomes visible fails here rather than in production. +""" + +from datetime import datetime +from unittest.mock import patch + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import ( + ContentOrigin, + DocumentIndexStatus, + DocumentStatus, + KnowledgeDocument, +) +from app.models.subtask_context import SubtaskContext +from app.models.user import User +from app.models.wiki import ( + WikiContent, + WikiGeneration, + WikiGenerationStatus, + WikiGenerationType, +) +from app.services.knowledge.code_wiki.projection_plan import PAGE_PATH_KEY +from app.services.knowledge.code_wiki.publisher import ( + publish_generation, + published_generation_id, +) +from app.services.knowledge.code_wiki.side_effects import ( + build_projection_side_effects, +) +from app.services.knowledge.code_wiki.version_store import set_page_path +from app.services.knowledge.index_state_machine import ( + mark_document_index_succeeded, +) + + +@pytest.fixture +def knowledge_base(test_db: Session, test_user: User) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-wiki-e2e", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +@pytest.fixture +def generation(test_db: Session, knowledge_base: Kind, test_user: User): + def build() -> WikiGeneration: + record = WikiGeneration( + project_id=1, + kind_id=knowledge_base.id, + user_id=test_user.id, + task_id=0, + team_id=1, + generation_type=WikiGenerationType.FULL, + source_snapshot={}, + status=WikiGenerationStatus.COMPLETED, + completed_at=datetime(1970, 1, 1), + ) + test_db.add(record) + test_db.flush() + return record + + return build + + +def _page(test_db: Session, generation: WikiGeneration, path: str, content: str): + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path.rsplit("/", 1)[-1], + content=content, + parent_id=0, + ) + set_page_path(entry, path) + test_db.add(entry) + test_db.flush() + + +def _publish(test_db, knowledge_base, generation, test_user, enqueued): + """Publish with real attachment storage; only the queue is intercepted.""" + effects = build_projection_side_effects( + test_db, knowledge_base=knowledge_base, user=test_user + ) + with patch( + "app.services.knowledge.code_wiki.side_effects._enqueue_reindex", + side_effect=lambda db, **kw: enqueued.append(kw["document_id"]), + ): + return publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=test_user.id, + effects=effects, + ) + + +def test_a_published_page_has_real_content_behind_it( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + """The document must point at an attachment that actually holds the page.""" + version = generation() + _page(test_db, version, "architecture/backend", "# Backend\n\nHow it works.") + enqueued: list[int] = [] + + result = _publish(test_db, knowledge_base, version, test_user, enqueued) + + assert result.published + document = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + ) + attachment = test_db.get(SubtaskContext, document.attachment_id) + assert attachment is not None + assert attachment.extracted_text.strip().startswith("# Backend") + assert document.file_size == len("# Backend\n\nHow it works.".encode("utf-8")) + + +def test_a_published_page_is_queued_for_indexing( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + version = generation() + _page(test_db, version, "index", "overview") + enqueued: list[int] = [] + + _publish(test_db, knowledge_base, version, test_user, enqueued) + + document = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + ) + assert enqueued == [document.id] + + +def test_a_page_stays_invisible_until_its_index_succeeds( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + """The assumption the whole publish path rests on. + + Pages are created switched off and the indexing state machine turns them on. If + that did not hold, a published wiki would be silently unreadable while every unit + test still passed. + """ + from app.services.knowledge.index_state_machine import ( + prepare_document_index_enqueue, + ) + + version = generation() + _page(test_db, version, "index", "overview") + enqueued: list[int] = [] + + _publish(test_db, knowledge_base, version, test_user, enqueued) + + document = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + ) + assert document.is_active is False + assert document.status == DocumentStatus.DISABLED + + # The real path, not a shortcut: the state machine only accepts a success for a + # document it queued, so jumping straight to it proves nothing. + decision = prepare_document_index_enqueue(test_db, document_id=document.id) + assert decision.should_enqueue + mark_document_index_succeeded( + test_db, document_id=document.id, generation=decision.generation + ) + test_db.refresh(document) + + assert document.is_active is True + assert document.status == DocumentStatus.ENABLED + + +def test_indexing_is_not_blocked_by_the_page_being_switched_off( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + """Enqueueing looks at the index status, not at whether the page is visible.""" + version = generation() + _page(test_db, version, "index", "overview") + enqueued: list[int] = [] + _publish(test_db, knowledge_base, version, test_user, enqueued) + + document = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + ) + + from app.services.knowledge.index_state_machine import ( + prepare_document_index_enqueue, + ) + + decision = prepare_document_index_enqueue(test_db, document_id=document.id) + + assert decision.should_enqueue is True + + +def test_a_second_publish_replaces_the_content_and_keeps_the_document( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + first = generation() + _page(test_db, first, "index", "first draft") + enqueued: list[int] = [] + _publish(test_db, knowledge_base, first, test_user, enqueued) + document_id = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + .id + ) + original_attachment = test_db.get(KnowledgeDocument, document_id).attachment_id + + second = generation() + _page(test_db, second, "index", "second draft") + _publish(test_db, knowledge_base, second, test_user, enqueued) + + document = test_db.get(KnowledgeDocument, document_id) + assert document is not None, "the document id must survive a rewrite" + assert document.attachment_id != original_attachment + attachment = test_db.get(SubtaskContext, document.attachment_id) + assert "second draft" in attachment.extracted_text + assert published_generation_id(knowledge_base) == second.id + + +def test_an_unchanged_page_writes_no_new_attachment( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + first = generation() + _page(test_db, first, "index", "identical") + enqueued: list[int] = [] + _publish(test_db, knowledge_base, first, test_user, enqueued) + attachment_count = test_db.query(SubtaskContext).count() + + second = generation() + _page(test_db, second, "index", "identical") + _publish(test_db, knowledge_base, second, test_user, enqueued) + + assert test_db.query(SubtaskContext).count() == attachment_count + + +def test_a_removed_page_takes_its_attachment_with_it( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + first = generation() + _page(test_db, first, "index", "kept") + _page(test_db, first, "doomed", "removed later") + enqueued: list[int] = [] + _publish(test_db, knowledge_base, first, test_user, enqueued) + doomed = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.name == "doomed") + .one() + ) + doomed_attachment = doomed.attachment_id + + second = generation() + _page(test_db, second, "index", "kept") + result = _publish(test_db, knowledge_base, second, test_user, enqueued) + + # Stated rather than assumed: removing one of two pages sits exactly on the + # gate's threshold, and everything below only means something if it published. + assert result.published + assert test_db.get(KnowledgeDocument, doomed.id) is None + assert test_db.get(SubtaskContext, doomed_attachment) is None + + +def test_the_folder_tree_is_built_from_the_page_paths( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + version = generation() + _page(test_db, version, "index", "root page") + _page(test_db, version, "architecture/backend", "nested page") + enqueued: list[int] = [] + + _publish(test_db, knowledge_base, version, test_user, enqueued) + + documents = { + (document.source_config or {}).get(PAGE_PATH_KEY): document + for document in test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .all() + } + assert documents["index"].folder_id == 0 + assert documents["architecture/backend"].folder_id != 0 + assert documents["architecture/backend"].origin == ContentOrigin.GENERATED.value + + +def test_a_fresh_page_starts_unindexed( + test_db: Session, knowledge_base: Kind, test_user: User, generation +): + version = generation() + _page(test_db, version, "index", "overview") + enqueued: list[int] = [] + + _publish(test_db, knowledge_base, version, test_user, enqueued) + + document = ( + test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == knowledge_base.id) + .one() + ) + assert document.index_status == DocumentIndexStatus.NOT_INDEXED diff --git a/backend/tests/services/knowledge/code_wiki/test_publish_gate.py b/backend/tests/services/knowledge/code_wiki/test_publish_gate.py new file mode 100644 index 0000000000..48d3120f87 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_publish_gate.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the publish gate. + +The gate is what makes agent-declared deletion safe to allow: it lets a version remove +pages while refusing one that removes most of them. + +Removal is measured over paths, not counts, so the cases below pair a version with the +set of paths currently published rather than with a number. +""" + +from app.services.knowledge.code_wiki.projection_plan import PageSource +from app.services.knowledge.code_wiki.publish_gate import ( + PublishPolicy, + evaluate_publish_gate, +) + + +def _pages(count: int, content: str = "flowchart TD\n A --> B") -> list[PageSource]: + return [ + PageSource(path=f"page-{index}", title=f"Page {index}", content=content) + for index in range(count) + ] + + +def _published(count: int) -> list[str]: + """The paths a previously published version left in the knowledge base.""" + return [f"page-{index}" for index in range(count)] + + +def test_a_version_of_similar_size_passes(): + verdict = evaluate_publish_gate(_pages(10), published_paths=_published(10)) + + assert verdict.passed + + +def test_a_first_publish_has_nothing_to_compare_against(): + verdict = evaluate_publish_gate(_pages(3), published_paths=_published(0)) + + assert verdict.passed + + +def test_a_version_that_produced_nothing_is_rejected(): + """An empty repository still yields an overview page; zero means a failed run.""" + verdict = evaluate_publish_gate([], published_paths=_published(0)) + + assert not verdict.passed + assert "nothing usable" in verdict.reason + + +def test_dropping_most_of_the_wiki_is_rejected(): + verdict = evaluate_publish_gate(_pages(2), published_paths=_published(10)) + + assert not verdict.passed + assert "80%" in verdict.reason + + +def test_a_moderate_removal_is_allowed(): + """Deleting pages is legitimate; this is what the gate deliberately permits.""" + verdict = evaluate_publish_gate(_pages(7), published_paths=_published(10)) + + assert verdict.passed + + +def test_the_removal_limit_is_configurable(): + strict = PublishPolicy(max_removed_share=0.1) + + verdict = evaluate_publish_gate( + _pages(8), published_paths=_published(10), policy=strict + ) + + assert not verdict.passed + + +def test_growth_is_never_treated_as_removal(): + verdict = evaluate_publish_gate(_pages(50), published_paths=_published(2)) + + assert verdict.passed + + +def test_a_broken_diagram_is_reported_but_does_not_block(): + """A diagram that will not render is a local display fault, not a bad version.""" + pages = [ + PageSource(path="index", title="Index", content="```mermaid\nflowchat TD\n```") + ] + + verdict = evaluate_publish_gate(pages, published_paths=["index"]) + + assert verdict.passed + assert len(verdict.warnings) == 1 + assert "index:" in verdict.warnings[0] + + +def test_diagrams_can_be_made_blocking_by_policy(): + pages = [ + PageSource(path="index", title="Index", content="```mermaid\nflowchat TD\n```") + ] + + verdict = evaluate_publish_gate( + pages, published_paths=["index"], policy=PublishPolicy(block_on_mermaid=True) + ) + + assert not verdict.passed + + +def test_a_rejection_still_carries_its_warnings(): + """The verdict is stored to explain the rejection, so it must be complete.""" + pages = [ + PageSource(path="index", title="Index", content="```mermaid\nflowchat TD\n```") + ] + + verdict = evaluate_publish_gate(pages, published_paths=_published(10)) + + assert not verdict.passed + assert verdict.warnings + + +def test_the_verdict_renders_for_storage(): + verdict = evaluate_publish_gate(_pages(1), published_paths=_published(10)) + + stored = verdict.to_ext("2026-07-31T00:00:00") + + assert stored["result"] == "rejected" + assert stored["reason"] + assert stored["checkedAt"] == "2026-07-31T00:00:00" + + +def test_a_passing_verdict_renders_as_passed(): + stored = evaluate_publish_gate(_pages(5), published_paths=_published(5)).to_ext( + "now" + ) + + assert stored["result"] == "passed" + + +# --- removal is about paths, not counts ------------------------------------- + + +def test_replacing_every_path_is_a_mass_deletion_even_at_the_same_size(): + """The case counting cannot see. Ten pages become ten pages, and every one of + them is a new document: the old ids are deleted, and every stored citation and + index entry pointing at them breaks.""" + renamed = [ + PageSource(path=f"guide/page-{index}", title=f"Page {index}", content="body") + for index in range(10) + ] + + verdict = evaluate_publish_gate(renamed, published_paths=_published(10)) + + assert not verdict.passed + assert "100%" in verdict.reason + + +def test_renaming_a_few_paths_is_still_allowed(): + """Moving a page is legitimate; the gate only refuses moving most of them.""" + moved = [ + PageSource(path="guide/page-0", title="Page 0", content="body"), + *_pages(10)[1:], + ] + + verdict = evaluate_publish_gate(moved, published_paths=_published(10)) + + assert verdict.passed + + +def test_a_path_kept_under_a_different_case_is_not_a_removal(): + """The knowledge tables collate case-insensitively, so the two are one page.""" + recased = [ + PageSource(path=f"Page-{index}", title=f"Page {index}", content="body") + for index in range(10) + ] + + verdict = evaluate_publish_gate(recased, published_paths=_published(10)) + + assert verdict.passed + + +# --- sections that hold pages but are not pages ------------------------------ + + +def test_a_section_with_no_page_of_its_own_is_reported_not_refused(): + """The navigation is built from paths, so it renders as a group heading that + cannot be opened — worse to read, nowhere near worth discarding a version.""" + pages = [ + PageSource(path="index", title="Index", content="body"), + PageSource(path="architecture/backend", title="Backend", content="body"), + ] + + verdict = evaluate_publish_gate(pages, published_paths=[]) + + assert verdict.passed + assert any("architecture" in warning for warning in verdict.warnings) + + +def test_a_section_that_is_itself_a_page_draws_no_warning(): + pages = [ + PageSource(path="architecture", title="Architecture", content="body"), + PageSource(path="architecture/backend", title="Backend", content="body"), + ] + + verdict = evaluate_publish_gate(pages, published_paths=[]) + + assert verdict.warnings == () + + +def test_the_diagram_policy_does_not_reject_over_a_missing_section_page(): + """`block_on_mermaid` is named for diagrams; letting it fire on a navigation nit + would make its name a lie and throw away versions nobody meant to refuse.""" + pages = [ + PageSource(path="index", title="Index", content="body"), + PageSource(path="architecture/backend", title="Backend", content="body"), + ] + + verdict = evaluate_publish_gate( + pages, published_paths=[], policy=PublishPolicy(block_on_mermaid=True) + ) + + assert verdict.passed + assert verdict.warnings diff --git a/backend/tests/services/knowledge/code_wiki/test_publisher.py b/backend/tests/services/knowledge/code_wiki/test_publisher.py new file mode 100644 index 0000000000..6f2ec7e99e --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_publisher.py @@ -0,0 +1,555 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for publishing a code wiki version. + +Publishing is the only thing that moves the published pointer, and a rollback is the +same operation aimed at an older version, so these tests cover both through one path. +""" + +from dataclasses import dataclass, field +from datetime import datetime + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import ContentOrigin, KnowledgeDocument +from app.models.wiki import ( + WikiContent, + WikiGeneration, + WikiGenerationStatus, + WikiGenerationType, +) +from app.services.knowledge.code_wiki.projection import ( + PENDING_INDEX_CLEANUP_KEY, + ProjectionSideEffects, +) +from app.services.knowledge.code_wiki.publish_gate import ( + PUBLISH_GATE_EXT_KEY, + PublishPolicy, +) +from app.services.knowledge.code_wiki.publisher import ( + PAGE_ORDER_KEY, + PUBLISHED_AT_KEY, + PUBLISHED_COMMIT_KEY, + PUBLISHED_GENERATION_KEY, + publish_generation, + published_generation_id, + retry_pending_index_cleanup, +) +from app.services.knowledge.code_wiki.version_store import set_page_path + +USER_ID = 11 + + +@dataclass +class FakeEffects: + written: list[str] = field(default_factory=list) + deleted_attachments: list[int] = field(default_factory=list) + deleted_rag: list[int] = field(default_factory=list) + reindexed: list[int] = field(default_factory=list) + failing_rag: set[int] = field(default_factory=set) + next_id: int = 5000 + + def build(self) -> ProjectionSideEffects: + return ProjectionSideEffects( + write_attachment=self._write, + delete_attachment=self.deleted_attachments.append, + delete_rag_document=self._delete_rag, + enqueue_reindex=self.reindexed.append, + ) + + def _write(self, *, filename: str, content: str) -> int: + self.next_id += 1 + self.written.append(filename) + return self.next_id + + def _delete_rag(self, doc_ref: int) -> None: + if doc_ref in self.failing_rag: + raise RuntimeError("vector store down") + self.deleted_rag.append(doc_ref) + + +@pytest.fixture +def effects() -> FakeEffects: + return FakeEffects() + + +@pytest.fixture +def knowledge_base(test_db: Session) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-code-wiki", + namespace="default", + user_id=USER_ID, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def _generation( + test_db: Session, + kind_id: int, + *, + status: WikiGenerationStatus = WikiGenerationStatus.COMPLETED, +) -> WikiGeneration: + generation = WikiGeneration( + project_id=1, + kind_id=kind_id, + user_id=USER_ID, + task_id=0, + team_id=1, + generation_type=WikiGenerationType.FULL, + source_snapshot={}, + status=status, + completed_at=datetime(1970, 1, 1), + ) + test_db.add(generation) + test_db.flush() + return generation + + +def _page(test_db: Session, generation: WikiGeneration, path: str, content: str): + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path.rsplit("/", 1)[-1], + content=content, + parent_id=0, + ) + set_page_path(entry, path) + test_db.add(entry) + test_db.flush() + return entry + + +def _live_paths(test_db: Session, kind_id: int) -> set[str]: + from app.services.knowledge.code_wiki.projection_plan import PAGE_PATH_KEY + + return { + (document.source_config or {}).get(PAGE_PATH_KEY) + for document in test_db.query(KnowledgeDocument) + .filter(KnowledgeDocument.kind_id == kind_id) + .all() + } + + +def test_a_first_publish_creates_the_pages_and_moves_the_pointer( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + generation = _generation(test_db, knowledge_base.id) + _page(test_db, generation, "index", "overview") + _page(test_db, generation, "architecture/backend", "details") + + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert result.published + assert _live_paths(test_db, knowledge_base.id) == {"index", "architecture/backend"} + assert published_generation_id(knowledge_base) == generation.id + + +def test_a_rejected_version_leaves_the_pointer_and_pages_alone( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """The published wiki must survive a run that produced almost nothing.""" + first = _generation(test_db, knowledge_base.id) + for index in range(10): + _page(test_db, first, f"page-{index}", "body") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=first, + user_id=USER_ID, + effects=effects.build(), + ) + + second = _generation(test_db, knowledge_base.id) + _page(test_db, second, "page-0", "body") + + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=second, + user_id=USER_ID, + effects=effects.build(), + ) + + assert not result.published + assert published_generation_id(knowledge_base) == first.id + assert len(_live_paths(test_db, knowledge_base.id)) == 10 + + +def test_a_rejection_is_recorded_on_the_generation( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """Explains why a finished version is not live; the pointer still decides.""" + generation = _generation(test_db, knowledge_base.id) + + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + recorded = (generation.ext or {})[PUBLISH_GATE_EXT_KEY] + assert recorded["result"] == "rejected" + assert recorded["reason"] + + +def test_an_unfinished_generation_is_not_published( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + generation = _generation( + test_db, knowledge_base.id, status=WikiGenerationStatus.RUNNING + ) + _page(test_db, generation, "index", "body") + + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert not result.published + assert published_generation_id(knowledge_base) == 0 + + +def test_publishing_again_with_unchanged_content_does_nothing( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + generation = _generation(test_db, knowledge_base.id) + _page(test_db, generation, "index", "same") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + repeat = FakeEffects() + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=repeat.build(), + ) + + assert result.published + assert repeat.written == [] + assert repeat.reindexed == [] + + +def test_rolling_back_is_publishing_an_older_version( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """No separate machinery: the same call, aimed at the version you want back.""" + first = _generation(test_db, knowledge_base.id) + _page(test_db, first, "index", "original") + _page(test_db, first, "keep-me", "body") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=first, + user_id=USER_ID, + effects=effects.build(), + ) + + second = _generation(test_db, knowledge_base.id) + _page(test_db, second, "index", "rewritten") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=second, + user_id=USER_ID, + effects=effects.build(), + policy=PublishPolicy(max_removed_share=1.0), + ) + assert _live_paths(test_db, knowledge_base.id) == {"index"} + + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=first, + user_id=USER_ID, + effects=effects.build(), + require_completed=False, + ) + + assert result.published + assert _live_paths(test_db, knowledge_base.id) == {"index", "keep-me"} + assert published_generation_id(knowledge_base) == first.id + + +def test_user_content_survives_a_publish( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """It is not regenerable, so the projection must never see it.""" + note = KnowledgeDocument( + kind_id=knowledge_base.id, + attachment_id=1, + name="my note", + file_extension="md", + file_size=4, + user_id=USER_ID, + folder_id=0, + origin=ContentOrigin.USER.value, + ) + test_db.add(note) + test_db.flush() + + generation = _generation(test_db, knowledge_base.id) + _page(test_db, generation, "index", "body") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert test_db.get(KnowledgeDocument, note.id) is not None + + +def test_a_failed_index_deletion_is_parked_for_retry( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """A row gone while its chunks remain leaves retrieval citing a dead page.""" + first = _generation(test_db, knowledge_base.id) + _page(test_db, first, "index", "body") + _page(test_db, first, "doomed", "body") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=first, + user_id=USER_ID, + effects=effects.build(), + ) + doomed_id = next( + document.id + for document in test_db.query(KnowledgeDocument).all() + if document.name == "doomed" + ) + + second = _generation(test_db, knowledge_base.id) + _page(test_db, second, "index", "body") + broken = FakeEffects(failing_rag={doomed_id}) + + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=second, + user_id=USER_ID, + effects=broken.build(), + ) + + # Stated rather than assumed: this version removes exactly half the published + # pages, which sits on the gate's threshold. Without this the test would still + # fail if the gate ever tightened, but as a confusing KeyError below. + assert result.published + parked = (knowledge_base.json or {})["spec"][PENDING_INDEX_CLEANUP_KEY] + assert parked == [str(doomed_id)] + + +def test_parked_cleanup_is_cleared_once_it_succeeds( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + knowledge_base.json = { + "spec": { + "name": "wiki", + "kbType": "code_wiki", + PENDING_INDEX_CLEANUP_KEY: ["42"], + } + } + test_db.flush() + + outstanding = retry_pending_index_cleanup( + test_db, knowledge_base=knowledge_base, effects=effects.build() + ) + + assert outstanding == () + assert knowledge_base.json["spec"][PENDING_INDEX_CLEANUP_KEY] == [] + assert effects.deleted_rag == [42] + + +def test_cleanup_that_still_fails_stays_parked(test_db: Session, knowledge_base: Kind): + knowledge_base.json = {"spec": {"name": "wiki", PENDING_INDEX_CLEANUP_KEY: ["42"]}} + test_db.flush() + broken = FakeEffects(failing_rag={42}) + + outstanding = retry_pending_index_cleanup( + test_db, knowledge_base=knowledge_base, effects=broken.build() + ) + + assert outstanding == ("42",) + + +def test_the_published_pointer_starts_at_nothing(knowledge_base: Kind): + assert published_generation_id(knowledge_base) == 0 + assert PUBLISHED_GENERATION_KEY not in knowledge_base.json["spec"] + + +def test_a_publish_settles_what_the_last_one_could_not( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """There is no sweeper. Publishing is the only thing that drains the debt, which + is what keeps this function the single writer of the parked list — two writers + doing read-modify-write on one spec key eventually drop a ref.""" + knowledge_base.json = { + "spec": { + "name": "wiki", + "kbType": "code_wiki", + PENDING_INDEX_CLEANUP_KEY: ["4242"], + } + } + test_db.flush() + + generation = _generation(test_db, knowledge_base.id) + _page(test_db, generation, "index", "body") + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert result.published + assert 4242 in effects.deleted_rag + assert (knowledge_base.json or {})["spec"][PENDING_INDEX_CLEANUP_KEY] == [] + + +def test_a_rejected_publish_still_settles_the_old_debt( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """The debt has nothing to do with whether this version is publishable.""" + first = _generation(test_db, knowledge_base.id) + for path in ("index", "one", "two", "three"): + _page(test_db, first, path, "body") + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=first, + user_id=USER_ID, + effects=effects.build(), + ) + _update_spec_pending(test_db, knowledge_base, ["4242"]) + + collapsed = _generation(test_db, knowledge_base.id) + _page(test_db, collapsed, "index", "body") + result = publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=collapsed, + user_id=USER_ID, + effects=effects.build(), + ) + + assert not result.published + assert 4242 in effects.deleted_rag + assert (knowledge_base.json or {})["spec"][PENDING_INDEX_CLEANUP_KEY] == [] + + +def _update_spec_pending(test_db: Session, knowledge_base: Kind, refs: list[str]): + payload = dict(knowledge_base.json or {}) + spec = dict(payload.get("spec", {})) + spec[PENDING_INDEX_CLEANUP_KEY] = refs + payload["spec"] = spec + knowledge_base.json = payload + test_db.flush() + + +# --- what a publish records for readers ------------------------------------- + + +def test_publishing_records_the_commit_and_time_for_the_list( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """A list would otherwise join every wiki against its generations for two + fields. Written in the publish transaction, so they cannot drift from the + pointer they sit beside.""" + generation = _generation(test_db, knowledge_base.id) + generation.source_snapshot = {"commit": "abc1234"} + _page(test_db, generation, "index", "body") + test_db.flush() + + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + spec = (knowledge_base.json or {})["spec"] + assert spec[PUBLISHED_COMMIT_KEY] == "abc1234" + assert spec[PUBLISHED_AT_KEY] + + +def test_the_agents_declared_order_is_what_gets_recorded( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """Paths carry hierarchy and say nothing about which section comes first. + Alphabetically "api" precedes the overview, and a wiki read that way reads + wrong.""" + generation = _generation(test_db, knowledge_base.id) + for path in ("api", "index", "architecture"): + _page(test_db, generation, path, "body") + generation.ext = { + "content_write": { + "summary": {"structure_order": ["index", "architecture", "api"]} + } + } + test_db.flush() + + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert (knowledge_base.json or {})["spec"][PAGE_ORDER_KEY] == [ + "index", + "architecture", + "api", + ] + + +def test_pages_the_agent_did_not_rank_follow_the_ones_it_did( + test_db: Session, knowledge_base: Kind, effects: FakeEffects +): + """A page added without updating the declared order must still appear, and + appear somewhere predictable rather than at a position nobody chose.""" + generation = _generation(test_db, knowledge_base.id) + for path in ("index", "stray"): + _page(test_db, generation, path, "body") + generation.ext = {"content_write": {"summary": {"structure_order": ["index"]}}} + test_db.flush() + + publish_generation( + test_db, + knowledge_base=knowledge_base, + generation=generation, + user_id=USER_ID, + effects=effects.build(), + ) + + assert (knowledge_base.json or {})["spec"][PAGE_ORDER_KEY] == ["index", "stray"] diff --git a/backend/tests/services/knowledge/code_wiki/test_repo_state.py b/backend/tests/services/knowledge/code_wiki/test_repo_state.py new file mode 100644 index 0000000000..ce9bc65baa --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_repo_state.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for reading what a repository is at, before a run is decided. + +The value here is entirely in the failure modes. A correct read saves a full rebuild; +a *wrong* read is worse than no read at all, because a partial diff mistaken for a +complete one picks an incremental run for a change that reshaped the repository. + +So every case that cannot be answered confidently has to come back as "unknown" — +which the run-mode rules turn into a full rebuild — and never as "nothing changed". +The one exception is pinned too: a repository already at the documented commit really +has an empty diff, and reporting that as unknown would rebuild a wiki that is current. +""" + +from unittest.mock import patch + +import pytest +from sqlalchemy.orm import Session + +from app.services.knowledge.code_wiki.repo_state import ( + RepositoryState, + read_repository_state, +) +from app.services.knowledge.code_wiki.source import ( + SUPPORTED_SOURCE_TYPES, + SourceRepository, + provider_for, +) + +SOURCE = SourceRepository( + source_type="github", + source_url="https://github.com/wecode-ai/Wegent.git", + project_name="wecode-ai/Wegent", + source_domain="github.com", +) + +HEAD = "bbbbbbb" +PUBLISHED = "aaaaaaa" + + +class FakeProvider: + """A provider that answers exactly what a test tells it to.""" + + def __init__(self, head=None, files=None, head_raises=False, files_raise=False): + self._head = head if head is not None else {"branch": "main", "commit": HEAD} + self._files = files + self._head_raises = head_raises + self._files_raise = files_raise + self.diff_calls = [] + + def get_default_branch_head(self, *, token, git_domain, repo_name): + if self._head_raises: + raise RuntimeError("provider is down") + return self._head + + def get_changed_files(self, *, token, git_domain, repo_name, base, head): + self.diff_calls.append((base, head)) + if self._files_raise: + raise RuntimeError("compare failed") + return self._files + + +def _with(provider, token="t0ken"): + """Patch the provider lookup and credentials this module reaches for.""" + return ( + patch( + "app.services.knowledge.code_wiki.repo_state.provider_for", + return_value=provider, + ), + patch( + "app.services.knowledge.code_wiki.repo_state.get_user_git_info", + return_value=({"token": token} if token else None), + ), + ) + + +def _read(test_db: Session, provider, *, token="t0ken", since=PUBLISHED): + provider_patch, git_patch = _with(provider, token) + with provider_patch, git_patch: + return read_repository_state( + test_db, user_id=1, source=SOURCE, since_commit=since + ) + + +# --- the happy path --------------------------------------------------------- + + +def test_the_default_branch_head_is_read(test_db: Session): + state = _read(test_db, FakeProvider(), since="") + + assert state.head_commit == HEAD + assert state.branch == "main" + + +def test_the_diff_is_read_against_the_published_commit(test_db: Session): + provider = FakeProvider( + files=[ + {"path": "backend/app/main.py", "status": "M"}, + {"path": "backend/app/new.py", "status": "A"}, + ] + ) + + state = _read(test_db, provider) + + assert provider.diff_calls == [(PUBLISHED, HEAD)] + assert [change.path for change in state.changed_paths] == [ + "backend/app/main.py", + "backend/app/new.py", + ] + assert state.changed_paths[1].is_structural_move + + +def test_a_first_run_asks_for_no_diff(test_db: Session): + """There is nothing to compare against, and asking would only cost a call.""" + provider = FakeProvider() + + state = _read(test_db, provider, since="") + + assert provider.diff_calls == [] + assert state.changed_paths is None + + +def test_a_repository_already_at_the_documented_commit_has_an_empty_diff( + test_db: Session, +): + """Empty, not unknown. Unknown would rebuild a wiki that is already current.""" + provider = FakeProvider(head={"branch": "main", "commit": PUBLISHED}) + + state = _read(test_db, provider) + + assert state.head_commit == PUBLISHED + assert state.changed_paths == () + assert provider.diff_calls == [] + + +# --- everything that cannot be answered ------------------------------------- + + +def test_an_unreachable_provider_leaves_the_state_unknown(test_db: Session): + state = _read(test_db, FakeProvider(head_raises=True)) + + assert state == RepositoryState() + + +def test_a_missing_credential_leaves_the_state_unknown(test_db: Session): + state = _read(test_db, FakeProvider(), token="") + + assert state == RepositoryState() + + +def test_an_unsupported_platform_leaves_the_state_unknown(test_db: Session): + state = _read(test_db, None) + + assert state == RepositoryState() + + +def test_a_head_that_cannot_be_read_stops_before_the_diff(test_db: Session): + provider = FakeProvider(head={"branch": "main", "commit": ""}) + + state = _read(test_db, provider) + + assert state.head_commit == "" + assert provider.diff_calls == [] + + +def test_a_failed_diff_is_unknown_rather_than_empty(test_db: Session): + """Reported empty, the run would be skipped and the changes never documented.""" + state = _read(test_db, FakeProvider(files_raise=True)) + + assert state.head_commit == HEAD + assert state.changed_paths is None + + +def test_a_diff_the_provider_would_not_complete_is_unknown(test_db: Session): + """The provider says ``None`` when it truncated or gave up; that must survive.""" + state = _read(test_db, FakeProvider(files=None)) + + assert state.head_commit == HEAD + assert state.changed_paths is None + + +def test_a_genuinely_empty_diff_is_kept_as_empty(test_db: Session): + """The commit moved but nothing changed in it — that is a skip, not a rebuild.""" + state = _read(test_db, FakeProvider(files=[])) + + assert state.changed_paths == () + + +# --- the platforms that must be able to answer ------------------------------ + + +@pytest.mark.parametrize("source_type", SUPPORTED_SOURCE_TYPES) +def test_every_supported_platform_can_report_repository_state(source_type: str): + """A platform a code wiki accepts but cannot read state for would silently + rebuild from scratch on every run.""" + provider = provider_for(source_type) + + assert provider is not None + assert hasattr(provider, "get_default_branch_head") + assert hasattr(provider, "get_changed_files") diff --git a/backend/tests/services/knowledge/code_wiki/test_resolution.py b/backend/tests/services/knowledge/code_wiki/test_resolution.py new file mode 100644 index 0000000000..1efcfa5948 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_resolution.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for describing a repository before a wiki is bound to it. + +The case worth pinning is the one the repository selector cannot reach: a public +repository the caller is not a member of. It never appears in the list the selector +draws from, and until this existed it could not be documented at all — a wiki more +closed than the repository it describes. +""" + +from unittest.mock import patch + +import pytest + +from app.services.knowledge.code_wiki import resolution +from app.services.knowledge.code_wiki.resolution import resolve_repository +from app.services.knowledge.code_wiki.source import ( + SourceAccessDenied, + SourceRepository, +) + +PUBLIC = { + "visibility": "public", + "default_branch": "main", + "name": "wecode-ai/Wegent", + "description": "An agent operating system", +} + + +@pytest.fixture(autouse=True) +def _no_cache(monkeypatch): + """The cache is an optimisation; these tests are about what is resolved.""" + monkeypatch.setattr(resolution, "_cached", lambda source, has_token: None) + monkeypatch.setattr( + resolution, "_remember", lambda source, has_token, resolved: None + ) + + +@pytest.fixture +def source() -> SourceRepository: + return SourceRepository.from_url( + "github", "https://github.com/wecode-ai/Wegent.git" + ) + + +def _with(token: str | None, described): + """Patch the two collaborators: whose token, and what the provider answers.""" + provider = type("P", (), {"describe_repository": lambda self, **kw: described})() + return ( + patch.object( + resolution, + "get_user_git_info", + return_value={"token": token} if token else None, + ), + patch.object(resolution, "provider_for", return_value=provider), + ) + + +def test_a_public_repository_resolves_without_any_credential(source): + """The selector only lists repositories the caller belongs to, so this is the + only way a public repository can be documented at all.""" + git_info, provider = _with(None, PUBLIC) + + with git_info, provider: + resolved = resolve_repository(db=None, user_id=1, source=source) + + assert resolved.exists is True + assert resolved.visibility == "public" + assert resolved.access == "public" + assert resolved.default_branch == "main" + + +def test_an_unreadable_repository_is_reported_without_saying_why(source): + """Private and absent must look alike: telling them apart would disclose which + private repositories exist.""" + git_info, provider = _with(None, None) + + with git_info, provider: + resolved = resolve_repository(db=None, user_id=1, source=source) + + assert resolved.exists is False + assert resolved.visibility == "" + assert resolved.access == "none" + + +def test_a_credential_is_used_when_the_caller_has_one(source): + git_info, provider = _with("ghp_token", {**PUBLIC, "visibility": "private"}) + + with git_info, provider: + resolved = resolve_repository(db=None, user_id=1, source=source) + + assert resolved.access == "member" + assert resolved.visibility == "private" + + +def test_the_default_branch_comes_back_so_branches_need_not_be_listed(source): + """Listing branches has no anonymous path; taking the default from here is what + lets the create form work for a public repository.""" + git_info, provider = _with(None, {**PUBLIC, "default_branch": "develop"}) + + with git_info, provider: + assert resolve_repository(db=None, user_id=1, source=source).default_branch == ( + "develop" + ) + + +def test_an_unsupported_type_is_refused_rather_than_probed(source): + other = SourceRepository( + source_type="svn", + source_url="https://example.com/x/y.git", + project_name="x/y", + source_domain="example.com", + ) + + with pytest.raises(SourceAccessDenied): + resolve_repository(db=None, user_id=1, source=other) + + +def test_an_unreadable_result_is_not_cached(source, monkeypatch): + """A repository about to be granted to the caller must not stay unreadable for + the length of the cache because they asked one moment early.""" + remembered = [] + monkeypatch.setattr( + resolution, + "_remember", + lambda source, has_token, resolved: remembered.append(resolved), + ) + git_info, provider = _with(None, None) + + with git_info, provider: + resolve_repository(db=None, user_id=1, source=source) + + assert remembered == [] diff --git a/backend/tests/services/knowledge/code_wiki/test_run_mode.py b/backend/tests/services/knowledge/code_wiki/test_run_mode.py new file mode 100644 index 0000000000..051b2f4792 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_run_mode.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for code wiki run mode selection. + +The mode decides whether a run's reported page set may be used to delete pages, so +these tests guard against the failure that would delete a whole wiki. +""" + +from app.services.knowledge.code_wiki.run_mode import ( + ChangedPath, + RunMode, + RunModeDecision, + RunModePolicy, + decide_run_mode, +) + +HEAD = "aaaaaaa" +PREVIOUS = "bbbbbbb" + + +def _edits(count: int) -> list[ChangedPath]: + return [ChangedPath(f"src/module_{i}.py", "M") for i in range(count)] + + +def test_first_run_rebuilds_everything(): + decision = decide_run_mode(head_commit=HEAD, last_commit=None) + + assert decision.mode is RunMode.FULL + assert decision.seeds_from_published is False + + +def test_unchanged_repository_is_skipped(): + decision = decide_run_mode(head_commit=HEAD, last_commit=HEAD) + + assert decision.mode is RunMode.SKIP + assert decision.seeds_from_published is False + + +def test_small_change_stays_incremental(): + decision = decide_run_mode( + head_commit=HEAD, last_commit=PREVIOUS, changed_paths=_edits(3) + ) + + assert decision.mode is RunMode.INCREMENTAL + + +def test_an_incremental_run_must_be_seeded(): + """An incremental run only revises the pages its diff affects. + + Without a seed the version would hold just those pages, and the projection — + which compares complete snapshots — would read every untouched page as an orphan + and delete it. That is the whole wiki minus a handful. + """ + decision = decide_run_mode( + head_commit=HEAD, last_commit=PREVIOUS, changed_paths=_edits(1) + ) + + assert decision.mode is RunMode.INCREMENTAL + assert decision.seeds_from_published is True + + +def test_a_full_rebuild_starts_from_an_empty_version(): + assert ( + decide_run_mode(head_commit=HEAD, last_commit=None).seeds_from_published + is False + ) + + +def test_unknown_diff_rebuilds_rather_than_guessing(): + decision = decide_run_mode( + head_commit=HEAD, last_commit=PREVIOUS, changed_paths=None + ) + + assert decision.mode is RunMode.FULL + assert "unknown" in decision.reason + + +def test_commit_moved_but_nothing_documented_changed_is_skipped(): + decision = decide_run_mode(head_commit=HEAD, last_commit=PREVIOUS, changed_paths=[]) + + assert decision.mode is RunMode.SKIP + + +def test_dependency_manifest_change_forces_a_rebuild(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=[ChangedPath("backend/pyproject.toml", "M")], + ) + + assert decision.mode is RunMode.FULL + assert "manifest" in decision.reason + + +def test_too_many_changed_files_forces_a_rebuild(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(51), + policy=RunModePolicy(max_changed_files=50), + ) + + assert decision.mode is RunMode.FULL + assert "over the limit" in decision.reason + + +def test_large_share_of_the_repository_forces_a_rebuild(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(30), + total_source_files=100, + policy=RunModePolicy(max_changed_files=50, max_changed_ratio=0.25), + ) + + assert decision.mode is RunMode.FULL + assert "of files changed" in decision.reason + + +def test_accumulated_incremental_runs_force_a_periodic_rebuild(): + """Restructuring orphans are invisible to incremental runs, so full runs recur.""" + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(2), + incrementals_since_full=10, + policy=RunModePolicy(max_incrementals_since_full=10), + ) + + assert decision.mode is RunMode.FULL + assert "incremental runs since" in decision.reason + + +def test_age_since_last_full_run_forces_a_periodic_rebuild(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(2), + days_since_full=31, + policy=RunModePolicy(max_days_since_full=30), + ) + + assert decision.mode is RunMode.FULL + assert "days since" in decision.reason + + +def test_skip_takes_precedence_over_a_due_periodic_rebuild(): + """There is nothing to rebuild when the repository has not moved.""" + decision = decide_run_mode( + head_commit=HEAD, + last_commit=HEAD, + incrementals_since_full=99, + days_since_full=999, + ) + + assert decision.mode is RunMode.SKIP + + +def test_many_added_or_removed_files_force_a_rebuild_sooner_than_edits(): + """Files appearing and disappearing reshape the wiki more than edits do.""" + moves = [ChangedPath(f"src/new_{i}.py", "A") for i in range(16)] + + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=moves, + policy=RunModePolicy(max_changed_files=50, max_structural_moves=15), + ) + + assert decision.mode is RunMode.FULL + assert "added, removed or renamed" in decision.reason + + +def test_the_same_number_of_plain_edits_stays_incremental(): + decision = decide_run_mode( + head_commit=HEAD, + last_commit=PREVIOUS, + changed_paths=_edits(16), + policy=RunModePolicy(max_changed_files=50, max_structural_moves=15), + ) + + assert decision.mode is RunMode.INCREMENTAL + + +def test_a_mode_stored_as_a_plain_string_still_seeds(): + """Modes survive task payloads as strings; identity comparison would fail. + + Getting this wrong leaves an incremental version unseeded, which the projection + then treats as a complete snapshot containing only the revised pages. + """ + assert RunModeDecision("incremental", "restored").seeds_from_published is True + assert RunModeDecision("full", "restored").seeds_from_published is False + + +def test_changed_path_recognises_structural_moves(): + assert ChangedPath("a.py", "A").is_structural_move is True + assert ChangedPath("a.py", "D").is_structural_move is True + assert ChangedPath("a.py", "R100").is_structural_move is True + assert ChangedPath("a.py", "M").is_structural_move is False diff --git a/backend/tests/services/knowledge/code_wiki/test_runner.py b/backend/tests/services/knowledge/code_wiki/test_runner.py new file mode 100644 index 0000000000..81f521d6ba --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_runner.py @@ -0,0 +1,524 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for driving a run from a knowledge base to a task and back. + +The pieces either side of the runner are covered on their own, so what is asserted +here is the wiring: that the agent gets instructions matching the mode the version +store chose, that a run which cannot get a task does not sit RUNNING and block the +wiki, and that the commit the agent reports is the one the next run compares against. +""" + +from dataclasses import dataclass, field +from datetime import datetime + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.user import User +from app.models.wiki import WikiContent, WikiGeneration, WikiGenerationStatus +from app.schemas.task import TaskCreate +from app.services.knowledge.code_wiki.generation import published_commit +from app.services.knowledge.code_wiki.publisher import published_generation_id +from app.services.knowledge.code_wiki.run_mode import ChangedPath +from app.services.knowledge.code_wiki.runner import ( + CodeWikiRunError, + finish_run, + is_code_wiki_generation, + source_of, + start_run, +) +from app.services.knowledge.code_wiki.version_store import set_page_path + +HEAD = "aaaaaaa" +NEXT_HEAD = "bbbbbbb" + +SOURCE = { + "sourceType": "github", + "sourceUrl": "https://github.com/wecode-ai/Wegent.git", + "sourceDomain": "github.com", + "projectName": "wecode-ai/Wegent", +} + + +@dataclass +class FakeTasks: + """Stands in for the task service, capturing what the agent would be sent.""" + + created: list[TaskCreate] = field(default_factory=list) + next_id: int = 500 + fails: bool = False + + def create_task_id(self, db, user_id: int) -> int: + self.next_id += 1 + return self.next_id + + def create_task_or_append(self, *, db, obj_in: TaskCreate, user, task_id): + if self.fails: + raise RuntimeError("no executor available") + self.created.append(obj_in) + return {"id": task_id} + + @property + def prompt(self) -> str: + return self.created[-1].prompt + + +@pytest.fixture +def tasks(monkeypatch, test_db: Session, test_user: User) -> FakeTasks: + """Replace team lookup and task creation, which reach outside this unit.""" + from app.services.adapters import task_kinds, team_kinds + + team = Kind( + kind="Team", + name="code-wiki-team", + namespace="default", + user_id=test_user.id, + json={"spec": {"description": "code wiki"}}, + is_active=True, + ) + test_db.add(team) + test_db.flush() + + fake = FakeTasks() + monkeypatch.setattr( + team_kinds.team_kinds_service, + "get_team_by_name_and_namespace", + lambda **kwargs: team, + ) + monkeypatch.setattr( + task_kinds.task_kinds_service, "create_task_id", fake.create_task_id + ) + monkeypatch.setattr( + task_kinds.task_kinds_service, + "create_task_or_append", + fake.create_task_or_append, + ) + return fake + + +@pytest.fixture +def knowledge_base(test_db: Session, test_user: User) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-runner", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "wiki", "kbType": "code_wiki", "source": SOURCE}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +@dataclass +class FakeEffects: + written: list[str] = field(default_factory=list) + next_id: int = 8000 + + def _write(self, *, filename: str, content: str) -> int: + self.next_id += 1 + self.written.append(filename) + return self.next_id + + +@pytest.fixture +def no_side_effects(monkeypatch) -> FakeEffects: + """Publish without touching attachment storage, the index or the queue.""" + from app.services.knowledge.code_wiki import runner + from app.services.knowledge.code_wiki.projection import ProjectionSideEffects + + fake = FakeEffects() + monkeypatch.setattr( + runner, + "build_projection_side_effects", + lambda db, *, knowledge_base, user: ProjectionSideEffects( + write_attachment=fake._write, + delete_attachment=lambda _: None, + delete_rag_document=lambda _: None, + enqueue_reindex=lambda _: None, + ), + ) + return fake + + +def _write_page(test_db: Session, generation: WikiGeneration, path: str): + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path, + content="body", + parent_id=0, + ) + set_page_path(entry, path) + test_db.add(entry) + test_db.flush() + + +# --- what the wiki is bound to --------------------------------------------- + + +def test_a_knowledge_base_that_is_not_a_code_wiki_cannot_be_generated( + test_db: Session, test_user: User +): + notebook = Kind( + kind="KnowledgeBase", + name="kb-notebook", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "notes", "kbType": "notebook"}}, + is_active=True, + ) + test_db.add(notebook) + test_db.flush() + + with pytest.raises(CodeWikiRunError, match="not a code wiki"): + source_of(notebook) + + +def test_a_code_wiki_with_no_repository_is_refused_before_a_task_exists( + test_db: Session, test_user: User +): + """Otherwise it surfaces much later as a task cloning an empty URL.""" + unbound = Kind( + kind="KnowledgeBase", + name="kb-unbound", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(unbound) + test_db.flush() + + with pytest.raises(CodeWikiRunError, match="no source repository"): + source_of(unbound) + + +# --- starting a run --------------------------------------------------------- + + +def test_a_first_run_sends_the_agent_a_task_for_the_bound_repository( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + assert started.started + created = tasks.created[0] + assert created.git_url == SOURCE["sourceUrl"] + assert created.git_repo == SOURCE["projectName"] + assert created.git_domain == SOURCE["sourceDomain"] + assert created.source == "code_wiki" + + +def test_the_task_clones_the_default_branch( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + """A pinned branch would document whatever it was at the day the wiki was made.""" + start_run(test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD) + + assert tasks.created[0].branch_name == "" + + +def test_the_run_is_reachable_from_the_task_it_created( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + assert started.generation.task_id == started.task_id + assert started.task_id > 0 + + +def test_a_first_run_gets_the_full_rebuild_instructions( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + start_run(test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD) + + assert "begins empty" in tasks.prompt + assert "copy of the published" not in tasks.prompt + + +def test_the_prompt_carries_the_generation_the_agent_must_write_into( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + assert str(started.generation.id) in tasks.prompt + + +def test_an_unchanged_repository_starts_no_task_at_all( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + assert not started.started + assert started.mode == "skip" + assert len(tasks.created) == 1 + + +def test_an_incremental_run_tells_the_agent_which_pages_already_exist( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + """Without them the agent cannot tell an update from a new page.""" + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + + start_run( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + ) + + assert "copy of the published" in tasks.prompt + assert "architecture/backend" in tasks.prompt + + +def test_an_incremental_run_tells_the_agent_what_changed( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + + start_run( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + ) + + assert "src/one.py" in tasks.prompt + assert HEAD in tasks.prompt and NEXT_HEAD in tasks.prompt + + +def test_a_run_whose_task_cannot_be_created_does_not_block_the_wiki( + test_db: Session, knowledge_base: Kind, test_user: User, tasks: FakeTasks +): + """Left RUNNING it would refuse to regenerate until the six-hour sweep.""" + tasks.fails = True + + with pytest.raises(CodeWikiRunError, match="Could not start the wiki task"): + start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + generation = test_db.query(WikiGeneration).one() + assert generation.status == WikiGenerationStatus.FAILED + assert "task creation failed" in generation.ext["errorMessage"] + + tasks.fails = False + retry = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + assert retry.started + + +# --- finishing a run -------------------------------------------------------- + + +def _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) -> WikiGeneration: + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + _write_page(test_db, started.generation, "index") + _write_page(test_db, started.generation, "architecture/backend") + finish_run(test_db, generation=started.generation, succeeded=True, head_commit=HEAD) + return started.generation + + +def test_a_successful_run_publishes_into_the_knowledge_base( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + generation = _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + + assert published_generation_id(knowledge_base) == generation.id + + +def test_the_commit_the_agent_reports_is_what_the_next_run_compares_against( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + """The trigger only knew what it was told; the agent read the working tree.""" + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit="" + ) + _write_page(test_db, started.generation, "index") + + finish_run( + test_db, generation=started.generation, succeeded=True, head_commit=NEXT_HEAD + ) + + assert published_commit(test_db, knowledge_base) == NEXT_HEAD + + +def test_a_failed_run_publishes_nothing( + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + first = _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + + started = start_run( + test_db, + knowledge_base=knowledge_base, + user=test_user, + head_commit=NEXT_HEAD, + changed_paths=[ChangedPath("src/one.py", "M")], + ) + result = finish_run( + test_db, + generation=started.generation, + succeeded=False, + error_message="model timed out", + ) + + assert result is None + assert published_generation_id(knowledge_base) == first.id + + +def test_a_run_belonging_to_no_knowledge_base_is_not_a_code_wiki_run( + test_db: Session, test_user: User +): + """The legacy wiki writes through the same API and must keep its own behaviour.""" + legacy = WikiGeneration( + project_id=1, + kind_id=0, + user_id=test_user.id, + task_id=0, + team_id=1, + generation_type="full", + source_snapshot={}, + status=WikiGenerationStatus.RUNNING, + completed_at=datetime(1970, 1, 1), + ) + test_db.add(legacy) + test_db.flush() + + assert not is_code_wiki_generation(test_db, legacy) + + +# --- resolving the repository's state when the caller did not supply it ------ + + +def _repository_at(monkeypatch, head: str, changed=None): + """Answer as the provider would, without reaching one.""" + from app.services.knowledge.code_wiki import runner + from app.services.knowledge.code_wiki.repo_state import RepositoryState + + monkeypatch.setattr( + runner, + "read_repository_state", + lambda db, *, user_id, source, since_commit: RepositoryState( + head_commit=head, branch="main", changed_paths=changed + ), + ) + + +def test_an_unchanged_repository_is_recognised_without_being_told( + monkeypatch, + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + """The whole point of reading HEAD: a schedule over a quiet repository must + cost one comparison, not a full pass through the model.""" + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + _repository_at(monkeypatch, HEAD, changed=()) + + started = start_run(test_db, knowledge_base=knowledge_base, user=test_user) + + assert not started.started + assert started.mode == "skip" + assert len(tasks.created) == 1 + + +def test_a_changed_repository_is_updated_incrementally_without_being_told( + monkeypatch, + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + _repository_at(monkeypatch, NEXT_HEAD, changed=(ChangedPath("src/one.py", "M"),)) + + started = start_run(test_db, knowledge_base=knowledge_base, user=test_user) + + assert started.mode == "incremental" + assert "src/one.py" in tasks.prompt + + +def test_a_repository_that_cannot_be_read_falls_back_to_a_rebuild( + monkeypatch, + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, + no_side_effects: FakeEffects, +): + """Expensive, and the only safe answer: an unknown diff might be anything.""" + _publish_a_first_wiki(test_db, knowledge_base, test_user, tasks) + _repository_at(monkeypatch, "", changed=None) + + started = start_run(test_db, knowledge_base=knowledge_base, user=test_user) + + assert started.mode == "full" + + +def test_a_supplied_commit_is_not_second_guessed( + monkeypatch, + test_db: Session, + knowledge_base: Kind, + test_user: User, + tasks: FakeTasks, +): + """A caller that knows the commit — a webhook, a test — must be believed.""" + from app.services.knowledge.code_wiki import runner + + def refuse(*args, **kwargs): + raise AssertionError("the provider must not be consulted") + + monkeypatch.setattr(runner, "read_repository_state", refuse) + + started = start_run( + test_db, knowledge_base=knowledge_base, user=test_user, head_commit=HEAD + ) + + assert started.started diff --git a/backend/tests/services/knowledge/code_wiki/test_source.py b/backend/tests/services/knowledge/code_wiki/test_source.py new file mode 100644 index 0000000000..ad07b7b192 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_source.py @@ -0,0 +1,364 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the repository access gate guarding code wiki creation.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from app.services.knowledge.code_wiki.source import ( + SourceAccessDenied, + SourceRepository, + assert_user_can_read_source, +) + +GITHUB_SOURCE = SourceRepository.from_url( + "github", "https://github.com/wecode-ai/Wegent.git" +) +GITLAB_SOURCE = SourceRepository.from_url( + "gitlab", "https://gitlab.example.com/team/app.git" +) + + +def _granted(): + return {"has_access": True, "access_level_name": "Reporter"} + + +def test_access_granted_returns_provider_details(): + provider = MagicMock() + provider.check_user_project_access.return_value = _granted() + + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"type": "github", "token": "t0ken"}, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + result = assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + assert result["has_access"] is True + provider.check_user_project_access.assert_called_once_with( + token="t0ken", git_domain="github.com", repo_name="wecode-ai/Wegent" + ) + + +def test_gitlab_is_identified_by_project_id(): + provider = MagicMock() + provider.check_user_project_access.return_value = _granted() + + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"type": "gitlab", "token": "t0ken"}, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + assert_user_can_read_source(MagicMock(), 1, GITLAB_SOURCE) + + provider.check_user_project_access.assert_called_once_with( + token="t0ken", git_domain="gitlab.example.com", project_id="team/app" + ) + + +def test_a_public_repository_is_readable_without_any_credential(): + """The repository selector only lists repositories the caller belongs to, so + refusing here would leave a publicly readable repository undocumentable — a wiki + more closed than its own source.""" + provider = MagicMock() + provider.describe_repository.return_value = { + "visibility": "public", + "default_branch": "main", + "name": "wecode-ai/Wegent", + "description": "", + } + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value=None, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + result = assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + assert result["has_access"] is True + # Read and no more: nobody has write access to a repository they reached without + # a credential, so this can never satisfy the gate on regenerating. + assert result["access_level"] == 10 + + +def test_a_private_repository_without_a_credential_is_denied(): + provider = MagicMock() + provider.describe_repository.return_value = None + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value=None, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="not readable without one"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_no_repository_access_is_denied(): + provider = MagicMock() + provider.check_user_project_access.return_value = { + "has_access": False, + "error": "Not a member", + } + + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"type": "github", "token": "t0ken"}, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="do not have read access"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_provider_error_denies_rather_than_allows(): + """An unreachable provider must not become an open door.""" + provider = MagicMock() + provider.check_user_project_access.side_effect = RuntimeError("gateway timeout") + + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"type": "github", "token": "t0ken"}, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="Could not verify access"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_unsupported_repository_type_is_refused_at_construction(): + with pytest.raises(SourceAccessDenied, match="Unsupported repository type"): + SourceRepository.from_url("svn", "svn://example.com/app") + + +def test_the_checked_repository_is_the_one_that_will_be_cloned(): + """Host and project come from the URL, so they cannot disagree with it. + + Accepting them separately would let a caller pass a repository they can read while + storing a URL pointing at a private one — the gate would approve one repository and + the wiki would be built from another. + """ + source = SourceRepository.from_url( + "gitlab", "https://gitlab.internal.corp/secret/payroll.git" + ) + + assert source.source_domain == "gitlab.internal.corp" + assert source.project_name == "secret/payroll" + assert source.source_url == "https://gitlab.internal.corp/secret/payroll.git" + + +def test_nested_group_paths_are_preserved(): + source = SourceRepository.from_url( + "gitlab", "https://gitlab.example.com/weibo_rd/common/wecode/wegent.git" + ) + + assert source.project_name == "weibo_rd/common/wecode/wegent" + + +def test_credentials_embedded_in_a_url_are_not_stored(): + """The stored URL is visible to everyone who can read the knowledge base.""" + source = SourceRepository.from_url( + "github", "https://ghp_secrettoken@github.com/owner/repo.git" + ) + + assert "ghp_secrettoken" not in source.source_url + assert source.source_url == "https://github.com/owner/repo.git" + + +def test_an_unusable_url_is_refused(): + with pytest.raises(SourceAccessDenied, match="Could not read a repository"): + SourceRepository.from_url("github", "not-a-url") + + +def test_credentials_for_another_platform_are_refused(): + """Asking one provider about another's repository gives a meaningless answer.""" + with patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"type": "gitlab", "token": "t0ken"}, + ): + with pytest.raises(SourceAccessDenied, match="configured as 'gitlab'"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_source_survives_a_round_trip_through_the_spec(): + assert SourceRepository.from_spec(GITLAB_SOURCE.to_spec()) == GITLAB_SOURCE + + +def test_a_knowledge_base_without_a_source_reads_as_none(): + assert SourceRepository.from_spec(None) is None + assert SourceRepository.from_spec({}) is None + + +# --- what a repository URL may name ----------------------------------------- + + +@pytest.mark.parametrize( + "url", + [ + "https://169.254.169.254/owner/repo.git", + "https://[fe80::1]/owner/repo.git", + "https://127.0.0.1/owner/repo.git", + "https://localhost/owner/repo.git", + # Written the long way round. The first two rely on the standard library + # folding IPv4-mapped forms into is_loopback/is_link_local, which is worth + # pinning because it is not this module's behaviour to keep. The last two + # are neither loopback nor link-local, and still reach the local host. + "https://[::ffff:127.0.0.1]/owner/repo.git", + "https://[::ffff:169.254.169.254]/owner/repo.git", + "https://0.0.0.0/owner/repo.git", + "https://[::]/owner/repo.git", + ], +) +def test_a_host_that_is_never_a_git_server_is_refused(url: str): + """Binding a repository makes the server fetch from the host the URL names, + carrying the caller's token. The metadata endpoint is link-local and answers + unauthenticated requests with instance credentials.""" + with pytest.raises(SourceAccessDenied, match="not a reachable repository host"): + SourceRepository.from_url("gitlab", url) + + +@pytest.mark.parametrize( + "url", + [ + "https://gitlab.internal.example.com/owner/repo.git", + "https://10.0.0.7/owner/repo.git", + "https://192.168.1.10/owner/repo.git", + ], +) +def test_a_self_hosted_host_on_an_internal_network_is_still_allowed(url: str): + """The normal deployment here. Blocking private ranges would break the product + to close a much smaller hole than the one above.""" + source = SourceRepository.from_url("gitlab", url) + + assert source.project_name == "owner/repo" + + +@pytest.mark.parametrize( + "git_info", + [ + None, + {}, + {"type": "github"}, + {"type": "github", "token": ""}, + ], + ids=["no-record", "empty-record", "no-token-key", "blank-token"], +) +def test_a_credential_record_without_a_usable_token_denies_access(git_info): + """Every shape of "configured but unusable" has to deny. A record that exists + but carries no token would otherwise reach the provider with ``None``.""" + provider = MagicMock() + provider.describe_repository.return_value = None + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value=git_info, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="not readable without one"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_a_public_repository_survives_a_token_that_does_not_reach_it(): + """Both providers answer membership questions, and "not a member" is also the + answer for a public repository nobody has joined. Without this fallback a stale + or unrelated token makes a world-readable repository undocumentable.""" + provider = MagicMock() + provider.check_user_project_access.return_value = { + "has_access": False, + "error": "Not a member", + } + provider.describe_repository.return_value = { + "visibility": "public", + "default_branch": "main", + "name": "wecode-ai/Wegent", + "description": "", + } + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"token": "stale", "type": "github"}, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + result = assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + assert result["has_access"] is True + assert result["access_level"] == 10 + + +def test_a_private_repository_a_token_cannot_reach_is_still_denied(): + """The fallback must not become an open door: when the public probe also says + no, the original and more specific refusal is what surfaces.""" + provider = MagicMock() + provider.check_user_project_access.return_value = { + "has_access": False, + "error": "Not a member", + } + provider.describe_repository.return_value = None + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"token": "stale", "type": "github"}, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="do not have read access"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) + + +def test_an_unreachable_provider_is_still_denied(): + """A provider that is down fails the public probe too, so the refusal stands.""" + provider = MagicMock() + provider.check_user_project_access.side_effect = RuntimeError("connection refused") + provider.describe_repository.return_value = None + with ( + patch( + "app.services.knowledge.code_wiki.source.get_user_git_info", + return_value={"token": "t0ken", "type": "github"}, + ), + patch( + "app.services.knowledge.code_wiki.source.provider_for", + return_value=provider, + ), + ): + with pytest.raises(SourceAccessDenied, match="Could not verify access"): + assert_user_can_read_source(MagicMock(), 1, GITHUB_SOURCE) diff --git a/backend/tests/services/knowledge/code_wiki/test_submit_to_publish.py b/backend/tests/services/knowledge/code_wiki/test_submit_to_publish.py new file mode 100644 index 0000000000..31cd325859 --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_submit_to_publish.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the agent's final submission reaching the knowledge base. + +The agent has one channel back to the server, and the last thing it sends through it +is a summary saying the run finished. For a code wiki that has to do more than record +a status: the version is offered to the publish gate and projected into the knowledge +base. These tests cover that handoff, including the case it must not affect — the +legacy wiki, which writes through the same endpoint and only records a status. +""" + +from dataclasses import dataclass, field +from datetime import datetime + +import pytest +from sqlalchemy.orm import Session + +from app.models.kind import Kind +from app.models.knowledge import KnowledgeDocument +from app.models.user import User +from app.models.wiki import ( + WikiContent, + WikiGeneration, + WikiGenerationStatus, + WikiGenerationType, +) +from app.schemas.wiki import ( + WikiContentSection, + WikiContentSummary, + WikiContentWriteRequest, +) +from app.services.knowledge.code_wiki.generation import published_commit +from app.services.knowledge.code_wiki.publish_gate import PUBLISH_GATE_EXT_KEY +from app.services.knowledge.code_wiki.publisher import published_generation_id +from app.services.knowledge.code_wiki.version_store import set_page_path +from app.services.wiki_service import WikiService + +HEAD = "aaaaaaa" + + +@dataclass +class FakeEffects: + written: list[str] = field(default_factory=list) + next_id: int = 9000 + + def _write(self, *, filename: str, content: str) -> int: + self.next_id += 1 + self.written.append(filename) + return self.next_id + + +@pytest.fixture +def no_side_effects(monkeypatch) -> FakeEffects: + from app.services.knowledge.code_wiki import runner + from app.services.knowledge.code_wiki.projection import ProjectionSideEffects + + fake = FakeEffects() + monkeypatch.setattr( + runner, + "build_projection_side_effects", + lambda db, *, knowledge_base, user: ProjectionSideEffects( + write_attachment=fake._write, + delete_attachment=lambda _: None, + delete_rag_document=lambda _: None, + enqueue_reindex=lambda _: None, + ), + ) + return fake + + +@pytest.fixture +def knowledge_base(test_db: Session, test_user: User) -> Kind: + kind = Kind( + kind="KnowledgeBase", + name="kb-submit", + namespace="default", + user_id=test_user.id, + json={"spec": {"name": "wiki", "kbType": "code_wiki"}}, + is_active=True, + ) + test_db.add(kind) + test_db.flush() + return kind + + +def _generation(test_db: Session, user: User, kind_id: int) -> WikiGeneration: + record = WikiGeneration( + project_id=0, + kind_id=kind_id, + user_id=user.id, + task_id=1, + team_id=1, + generation_type=WikiGenerationType.FULL, + source_snapshot={}, + status=WikiGenerationStatus.RUNNING, + completed_at=datetime(1970, 1, 1), + ) + test_db.add(record) + test_db.flush() + return record + + +def _seed_page(test_db: Session, generation: WikiGeneration, path: str): + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path, + content="body", + parent_id=0, + ) + set_page_path(entry, path) + test_db.add(entry) + test_db.flush() + + +def _submit(db: Session, generation: WikiGeneration, **summary_fields): + WikiService().save_generation_contents( + db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[], + summary=WikiContentSummary(**summary_fields), + ), + ) + + +def _documents(db: Session, kind_id: int) -> list[KnowledgeDocument]: + return ( + db.query(KnowledgeDocument).filter(KnowledgeDocument.kind_id == kind_id).all() + ) + + +def test_completing_a_code_wiki_run_publishes_it_into_the_knowledge_base( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + generation = _generation(test_db, test_user, knowledge_base.id) + _seed_page(test_db, generation, "index") + + _submit(test_db, generation, status="COMPLETED", head_commit=HEAD) + + assert published_generation_id(knowledge_base) == generation.id + assert [doc.name for doc in _documents(test_db, knowledge_base.id)] == ["index"] + + +def test_a_published_run_ends_completed_rather_than_running( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + generation = _generation(test_db, test_user, knowledge_base.id) + _seed_page(test_db, generation, "index") + + _submit(test_db, generation, status="COMPLETED", head_commit=HEAD) + + test_db.refresh(generation) + assert generation.status == WikiGenerationStatus.COMPLETED + + +def test_a_version_the_gate_refuses_does_not_replace_the_published_one( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + """This is why the agent's word is not enough on its own: it says the run + succeeded, and the gate still decides whether the result may go live.""" + first = _generation(test_db, test_user, knowledge_base.id) + for path in ("index", "one", "two", "three"): + _seed_page(test_db, first, path) + _submit(test_db, first, status="COMPLETED", head_commit=HEAD) + + collapsed = _generation(test_db, test_user, knowledge_base.id) + _seed_page(test_db, collapsed, "index") + + _submit(test_db, collapsed, status="COMPLETED", head_commit="bbbbbbb") + + test_db.refresh(collapsed) + assert published_generation_id(knowledge_base) == first.id + assert collapsed.ext[PUBLISH_GATE_EXT_KEY]["result"] == "rejected" + assert {doc.name for doc in _documents(test_db, knowledge_base.id)} == { + "index", + "one", + "two", + "three", + } + + +def test_the_reported_commit_reaches_the_published_version( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + generation = _generation(test_db, test_user, knowledge_base.id) + _seed_page(test_db, generation, "index") + + _submit(test_db, generation, status="COMPLETED", head_commit=HEAD) + + assert published_commit(test_db, knowledge_base) == HEAD + + +def test_a_failed_submission_publishes_nothing( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + generation = _generation(test_db, test_user, knowledge_base.id) + _seed_page(test_db, generation, "index") + + _submit(test_db, generation, status="FAILED", error_message="ran out of budget") + + test_db.refresh(generation) + assert generation.status == WikiGenerationStatus.FAILED + assert published_generation_id(knowledge_base) == 0 + assert _documents(test_db, knowledge_base.id) == [] + + +def test_pages_sent_with_the_final_summary_are_published_too( + test_db: Session, + knowledge_base: Kind, + test_user: User, + no_side_effects: FakeEffects, +): + """The agent may finish in one call, so the write must be committed first.""" + generation = _generation(test_db, test_user, knowledge_base.id) + + WikiService().save_generation_contents( + test_db, + WikiContentWriteRequest( + generation_id=generation.id, + sections=[ + WikiContentSection( + type="chapter", + title="Backend Architecture", + content="body", + path="architecture/backend", + ) + ], + summary=WikiContentSummary(status="COMPLETED", head_commit=HEAD), + ), + ) + + # Named for the title, which is what a reader sees. The path stays the identity + # in source_config, so rewording the heading renames the document without moving + # it or changing the id the RAG index is keyed on. + assert [doc.name for doc in _documents(test_db, knowledge_base.id)] == [ + "Backend Architecture" + ] + + +def test_a_legacy_generation_only_records_its_status( + test_db: Session, test_user: User, no_side_effects: FakeEffects +): + """Nothing to publish into, so the old behaviour has to survive untouched.""" + generation = _generation(test_db, test_user, kind_id=0) + _seed_page(test_db, generation, "index") + + _submit(test_db, generation, status="COMPLETED") + + test_db.refresh(generation) + assert generation.status == WikiGenerationStatus.COMPLETED + assert no_side_effects.written == [] diff --git a/backend/tests/services/knowledge/code_wiki/test_version_store.py b/backend/tests/services/knowledge/code_wiki/test_version_store.py new file mode 100644 index 0000000000..a50cc2d7fe --- /dev/null +++ b/backend/tests/services/knowledge/code_wiki/test_version_store.py @@ -0,0 +1,448 @@ +# SPDX-FileCopyrightText: 2026 Weibo, Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the code wiki version store. + +The store exists so that a failed or half-finished run stays invisible, so most of +these tests are about what survives a run going wrong. +""" + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy.orm import Session + +from app.models.wiki import ( + WikiContent, + WikiGeneration, + WikiGenerationStatus, + WikiGenerationType, +) +from app.services.knowledge.code_wiki.page_path import InvalidPagePath +from app.services.knowledge.code_wiki.version_store import ( + STALE_RUN_AFTER_HOURS, + apply_retention, + page_path_of, + reclaim_stale_generations, + remove_page, + seed_from_published, + set_page_path, +) + +KIND_ID = 77 +NOW = datetime(2026, 7, 31, 12, 0, 0) + + +def _generation( + db: Session, + *, + status: WikiGenerationStatus = WikiGenerationStatus.COMPLETED, + created_at: datetime = NOW, + updated_at: datetime = NOW, + kind_id: int = KIND_ID, +) -> WikiGeneration: + generation = WikiGeneration( + project_id=1, + kind_id=kind_id, + user_id=1, + task_id=0, + team_id=1, + generation_type=WikiGenerationType.FULL, + source_snapshot={}, + status=status, + created_at=created_at, + updated_at=updated_at, + completed_at=created_at, + ) + db.add(generation) + db.flush() + return generation + + +def _page( + db: Session, generation: WikiGeneration, path: str, content: str = "body" +) -> WikiContent: + entry = WikiContent( + generation_id=generation.id, + type="chapter", + title=path.rsplit("/", 1)[-1], + content=content, + parent_id=0, + ) + set_page_path(entry, path) + db.add(entry) + db.flush() + return entry + + +def _paths(db: Session, generation_id: int) -> set[str]: + return { + page_path_of(entry) + for entry in db.query(WikiContent) + .filter(WikiContent.generation_id == generation_id) + .all() + } + + +# --- seeding --------------------------------------------------------------- + + +def test_seeding_copies_the_published_version_verbatim(test_db: Session): + published = _generation(test_db) + _page(test_db, published, "index", "overview") + _page(test_db, published, "architecture/backend", "backend notes") + target = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + outcome = seed_from_published( + test_db, + target_generation_id=target.id, + published_generation_id=published.id, + ) + + assert outcome.seeded + assert outcome.copied_pages == 2 + assert _paths(test_db, target.id) == {"index", "architecture/backend"} + + +def test_seeded_content_is_byte_identical(test_db: Session): + """The projection skips pages whose hash is unchanged, which relies on this.""" + published = _generation(test_db) + _page(test_db, published, "index", "exact bytes") + target = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + seed_from_published( + test_db, + target_generation_id=target.id, + published_generation_id=published.id, + ) + + copied = ( + test_db.query(WikiContent).filter(WikiContent.generation_id == target.id).one() + ) + assert copied.content == "exact bytes" + + +def test_seeding_does_not_carry_over_parent_ids(test_db: Session): + """They name rows in the source version; copying them would dangle.""" + published = _generation(test_db) + parent = _page(test_db, published, "architecture") + child = _page(test_db, published, "architecture/backend") + child.parent_id = parent.id + test_db.flush() + target = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + seed_from_published( + test_db, + target_generation_id=target.id, + published_generation_id=published.id, + ) + + copied = ( + test_db.query(WikiContent).filter(WikiContent.generation_id == target.id).all() + ) + assert {entry.parent_id for entry in copied} == {0} + + +def test_seeding_twice_does_not_double_the_version(test_db: Session): + """A retried scheduling attempt must not duplicate every page.""" + published = _generation(test_db) + _page(test_db, published, "index") + target = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + seed_from_published( + test_db, + target_generation_id=target.id, + published_generation_id=published.id, + ) + second = seed_from_published( + test_db, + target_generation_id=target.id, + published_generation_id=published.id, + ) + + assert not second.seeded + assert len(_paths(test_db, target.id)) == 1 + + +def test_a_first_run_has_nothing_to_seed_from(test_db: Session): + target = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + outcome = seed_from_published( + test_db, target_generation_id=target.id, published_generation_id=0 + ) + + assert not outcome.seeded + assert _paths(test_db, target.id) == set() + + +# --- agent-declared deletion ---------------------------------------------- + + +def test_an_agent_can_drop_a_page_from_the_in_flight_version(test_db: Session): + generation = _generation(test_db, status=WikiGenerationStatus.RUNNING) + _page(test_db, generation, "index") + _page(test_db, generation, "modules/legacy") + + assert remove_page(test_db, generation_id=generation.id, path="modules/legacy") + + assert _paths(test_db, generation.id) == {"index"} + + +def test_removing_a_page_leaves_the_published_version_untouched(test_db: Session): + """Deletion is recoverable precisely because it lands in an unpublished version.""" + published = _generation(test_db) + _page(test_db, published, "modules/legacy") + draft = _generation(test_db, status=WikiGenerationStatus.RUNNING) + _page(test_db, draft, "modules/legacy") + + remove_page(test_db, generation_id=draft.id, path="modules/legacy") + + assert _paths(test_db, published.id) == {"modules/legacy"} + + +def test_removing_matches_the_page_regardless_of_case(test_db: Session): + generation = _generation(test_db, status=WikiGenerationStatus.RUNNING) + _page(test_db, generation, "Modules/Legacy") + + assert remove_page(test_db, generation_id=generation.id, path="modules/legacy") + + +def test_removing_an_unknown_page_reports_that_nothing_happened(test_db: Session): + generation = _generation(test_db, status=WikiGenerationStatus.RUNNING) + _page(test_db, generation, "index") + + assert not remove_page(test_db, generation_id=generation.id, path="nope") + + +def test_removing_rejects_a_malformed_path(test_db: Session): + generation = _generation(test_db, status=WikiGenerationStatus.RUNNING) + + with pytest.raises(InvalidPagePath): + remove_page(test_db, generation_id=generation.id, path="../escape") + + +# --- stale run reclamation ------------------------------------------------- + + +def test_an_abandoned_run_is_failed_so_the_wiki_is_not_blocked(test_db: Session): + """A crashed worker would otherwise hold the wiki forever.""" + stuck = _generation( + test_db, + status=WikiGenerationStatus.RUNNING, + updated_at=NOW - timedelta(hours=STALE_RUN_AFTER_HOURS + 1), + ) + + reclaimed = reclaim_stale_generations(test_db, kind_id=KIND_ID, now=NOW) + + assert reclaimed == (stuck.id,) + assert stuck.status == WikiGenerationStatus.FAILED + + +def test_a_slow_but_live_run_is_left_alone(test_db: Session): + _generation( + test_db, + status=WikiGenerationStatus.RUNNING, + updated_at=NOW - timedelta(hours=STALE_RUN_AFTER_HOURS - 1), + ) + + assert reclaim_stale_generations(test_db, kind_id=KIND_ID, now=NOW) == () + + +def test_reclamation_ignores_finished_runs(test_db: Session): + _generation( + test_db, + status=WikiGenerationStatus.COMPLETED, + updated_at=NOW - timedelta(days=30), + ) + + assert reclaim_stale_generations(test_db, kind_id=KIND_ID, now=NOW) == () + + +def test_reclamation_is_scoped_to_one_knowledge_base(test_db: Session): + _generation( + test_db, + kind_id=KIND_ID + 1, + status=WikiGenerationStatus.RUNNING, + updated_at=NOW - timedelta(days=5), + ) + + assert reclaim_stale_generations(test_db, kind_id=KIND_ID, now=NOW) == () + + +# --- retention ------------------------------------------------------------- + + +def test_retention_keeps_the_newest_successful_versions(test_db: Session): + generations = [ + _generation(test_db, created_at=NOW - timedelta(days=index)) + for index in range(5) + ] + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=generations[0].id, + keep_successful=3, + now=NOW, + ) + + assert set(removed) == {generations[3].id, generations[4].id} + + +def test_retention_deletes_the_pages_of_a_collected_version(test_db: Session): + old = _generation(test_db, created_at=NOW - timedelta(days=10)) + _page(test_db, old, "index") + current = _generation(test_db, created_at=NOW) + + apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=current.id, + keep_successful=1, + now=NOW, + ) + + assert _paths(test_db, old.id) == set() + + +def test_the_published_version_survives_newer_versions_that_never_published( + test_db: Session, +): + """A rejected publish gate leaves a completed version that was never published. + + Enough of those push the published one out of the newest ``keep_successful``, + which would collect the only version there is to roll back to. Note that runs + which *fail* cannot cause this — they never enter the successful list at all. + """ + published = _generation(test_db, created_at=NOW - timedelta(days=9)) + for index in range(5): + _generation(test_db, created_at=NOW - timedelta(days=index)) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=published.id, + keep_successful=3, + now=NOW, + ) + + assert published.id not in removed + assert test_db.get(WikiGeneration, published.id) is not None + + +def test_the_published_version_survives_a_repository_that_never_changes( + test_db: Session, +): + """With no new commits there are no new versions, so every version ages out.""" + published = _generation(test_db, created_at=NOW - timedelta(days=400)) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=published.id, + max_age_days=90, + now=NOW, + ) + + assert removed == () + assert test_db.get(WikiGeneration, published.id) is not None + + +def test_retention_drops_versions_past_the_age_limit(test_db: Session): + stale = _generation(test_db, created_at=NOW - timedelta(days=200)) + current = _generation(test_db, created_at=NOW) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=current.id, + max_age_days=90, + now=NOW, + ) + + assert removed == (stale.id,) + + +def test_failed_versions_are_kept_briefly_for_investigation(test_db: Session): + recent = _generation( + test_db, + status=WikiGenerationStatus.FAILED, + created_at=NOW - timedelta(days=2), + ) + ancient = _generation( + test_db, + status=WikiGenerationStatus.FAILED, + created_at=NOW - timedelta(days=30), + ) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=0, + failed_retention_days=7, + now=NOW, + ) + + assert removed == (ancient.id,) + assert test_db.get(WikiGeneration, recent.id) is not None + + +def test_retention_never_collects_a_run_that_is_still_in_flight(test_db: Session): + """Its pages are being written to; deleting them would corrupt a live run. + + Reclamation normally fails an abandoned run long before it reaches this age, but + retention must not rely on that having happened. + """ + for status in (WikiGenerationStatus.PENDING, WikiGenerationStatus.RUNNING): + in_flight = _generation( + test_db, status=status, created_at=NOW - timedelta(days=30) + ) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=0, + failed_retention_days=7, + now=NOW, + ) + + assert in_flight.id not in removed + assert test_db.get(WikiGeneration, in_flight.id) is not None + + +def test_an_aware_reference_time_is_converted_rather_than_truncated(test_db: Session): + """Dropping the offset instead of converting shifts the cutoff by that offset. + + The version sits just inside the retention window once the +14:00 reference is + properly converted to UTC, and just outside it if the offset is merely discarded. + Anything further from the boundary passes either way and would prove nothing. + """ + _generation( + test_db, + status=WikiGenerationStatus.FAILED, + created_at=NOW - timedelta(days=7, hours=6), + ) + aware_now = NOW.replace(tzinfo=timezone(timedelta(hours=14))) + + removed = apply_retention( + test_db, + kind_id=KIND_ID, + published_generation_id=0, + failed_retention_days=7, + now=aware_now, + ) + + assert removed == () + + +def test_retention_is_scoped_to_one_knowledge_base(test_db: Session): + other = _generation( + test_db, kind_id=KIND_ID + 1, created_at=NOW - timedelta(days=999) + ) + + removed = apply_retention( + test_db, kind_id=KIND_ID, published_generation_id=0, now=NOW + ) + + assert other.id not in removed diff --git a/docs/en/wegent/reference/yaml-specification.md b/docs/en/wegent/reference/yaml-specification.md index 6567f85038..9fcff4e8d9 100644 --- a/docs/en/wegent/reference/yaml-specification.md +++ b/docs/en/wegent/reference/yaml-specification.md @@ -452,6 +452,40 @@ KnowledgeBase is used to manage document knowledge bases, retrieval configuratio --- +### Code wiki fields + +A knowledge base with `spec.kbType = "code_wiki"` is generated by an agent from a +source repository. Every field below is written by the system and should not be +edited by hand. + +| Field | Type | Description | +| ---------------------------- | ------- | -------------------------------------------------------------------- | +| `spec.kbType` | string | `notebook`, `classic` or `code_wiki`. **Cannot move into or out of `code_wiki`** | +| `spec.source.sourceType` | string | `github`, `gitlab` or `gitea` | +| `spec.source.sourceUrl` | string | Repository URL, derived at creation with any credentials stripped | +| `spec.source.projectName` | string | `owner/repo` | +| `spec.source.sourceDomain` | string | Repository host | +| `spec.publishedGenerationId` | integer | **Sole authority for which version is live**; 0 when nothing is | +| `spec.lastPublishedAt` | string | When that version was published, written with the pointer | +| `spec.lastPublishedCommit` | string | Commit the live version documents | +| `spec.pageOrder` | array | Page paths in display order; hierarchy comes from the paths, order only from here | +| `spec.pendingIndexCleanup` | array | Vector store references owed for removed pages, retried at the next publish | + +**Invariants:** + +- **`publishedGenerationId` is the only authority.** Never infer the live version + from the newest completed generation: one can finish and still be refused by the + publish gate. +- **Ownership is fixed to the wiki account** (`WIKI_DEFAULT_USER_ID`) in the + `default` namespace. Who may read is decided by **repository access**, not by + knowledge base ACLs. +- **One repository has one wiki**, enforced by the UNIQUE on + `wiki_projects.source_url`. +- Generated pages carry `origin = generated`; the projection only adds and removes + those, so user content is never touched. + +--- + ## 🤝 Collaboration Collaboration defines the interaction patterns and workflows between Bots in a Team. diff --git a/docs/zh/wegent/reference/yaml-specification.md b/docs/zh/wegent/reference/yaml-specification.md index 45f0ea995a..ad32d411be 100644 --- a/docs/zh/wegent/reference/yaml-specification.md +++ b/docs/zh/wegent/reference/yaml-specification.md @@ -452,6 +452,32 @@ KnowledgeBase 用于管理文档知识库、检索配置和摘要能力。 --- +### 代码 Wiki 专属字段 + +`spec.kbType = "code_wiki"` 的知识库由智能体从一个源仓库生成，下列字段**全部由系统写入**，不应手工编辑。 + +| 字段 | 类型 | 说明 | +| --------------------------- | ------- | -------------------------------------------------------------------- | +| `spec.kbType` | string | `notebook`、`classic` 或 `code_wiki`。**创建后不能进出 `code_wiki`** | +| `spec.source.sourceType` | string | `github`、`gitlab` 或 `gitea` | +| `spec.source.sourceUrl` | string | 仓库地址，创建时从 URL 推导并剥除凭据 | +| `spec.source.projectName` | string | `owner/repo` | +| `spec.source.sourceDomain` | string | 仓库主机 | +| `spec.publishedGenerationId`| integer | **当前生效版本的唯一权威**，0 表示尚未发布 | +| `spec.lastPublishedAt` | string | 上次发布时间，与指针同事务写入 | +| `spec.lastPublishedCommit` | string | 生效版本所记录的 commit | +| `spec.pageOrder` | array | 页面路径的展示顺序；层级从路径推导，顺序只在这里 | +| `spec.pendingIndexCleanup` | array | 已删页面中尚未清理的向量库引用，下次发布时重试 | + +**几个不变量：** + +- **`publishedGenerationId` 是唯一权威** —— 不要从「最新完成的生成」推断生效版本：一次生成可以完成后被发布闸门拒绝 +- **归属固定在 wiki 账号**（`WIKI_DEFAULT_USER_ID`）、namespace 固定 `default` —— 谁能读由**仓库权限**决定，不由知识库 ACL 决定 +- **一个仓库一份 wiki**，由 `wiki_projects.source_url` 的唯一约束保证 +- 生成页面的 `origin` 为 `generated`；投影只增删这一类，用户内容永不受影响 + +--- + ## 🤝 Collaboration Collaboration 定义了团队中 Bot 之间的交互模式和工作流程。 diff --git a/frontend/src/__tests__/features/knowledge/code-wiki/create-kind-selection.test.tsx b/frontend/src/__tests__/features/knowledge/code-wiki/create-kind-selection.test.tsx new file mode 100644 index 0000000000..3ff14c9936 --- /dev/null +++ b/frontend/src/__tests__/features/knowledge/code-wiki/create-kind-selection.test.tsx @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * The create dialog now covers both kinds, so what is pinned here is that choosing + * one does not leave the other's rules in force — most of all the name, which is + * required for a document knowledge base and deliberately optional for a code wiki. + */ + +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { CreateKnowledgeBaseDialog } from '@/features/knowledge/document/components/CreateKnowledgeBaseDialog' + +jest.mock('@/hooks/useTranslation', () => ({ + useTranslation: () => ({ t: (key: string) => key }), +})) + +jest.mock('@/apis/code-wiki', () => ({ + codeWikiApi: { resolve: jest.fn().mockResolvedValue(null) }, +})) + +jest.mock('@/features/tasks/components/selector', () => ({ + RepositorySelector: () =>
, +})) + +describe('CreateKnowledgeBaseDialog kind selection', () => { + it('starts on documents, where a name is required', async () => { + const onSubmit = jest.fn() + render() + + fireEvent.click(screen.getByTestId('submit-create-kb')) + + await waitFor(() => expect(onSubmit).not.toHaveBeenCalled()) + }) + + it('swaps the opening-view field for repository fields when code is chosen', () => { + render() + + fireEvent.click(screen.getByTestId('create-kb-kind-code')) + + // A code wiki has no notebook/classic choice: it has a reader of its own. + expect(screen.queryByTestId('switch-kb-type')).not.toBeInTheDocument() + expect(screen.getByTestId('repository-selector')).toBeInTheDocument() + expect(screen.getByTestId('code-wiki-language')).toBeInTheDocument() + }) + + it('offers both ways to name a repository', () => { + render() + + fireEvent.click(screen.getByTestId('create-kb-kind-code')) + + // The selector alone is not enough: its list is membership-scoped, so a public + // repository the caller can read in a browser never appears in it. + expect(screen.getByTestId('code-wiki-source-mode-select')).toBeInTheDocument() + fireEvent.click(screen.getByTestId('code-wiki-source-mode-url')) + expect(screen.getByTestId('code-wiki-source-url')).toBeInTheDocument() + }) +}) diff --git a/frontend/src/apis/code-wiki.ts b/frontend/src/apis/code-wiki.ts new file mode 100644 index 0000000000..d63a80be9e --- /dev/null +++ b/frontend/src/apis/code-wiki.ts @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +import type { + CodeWikiCreateRequest, + CodeWikiListResponse, + CodeWikiPageTree, + CodeWikiResolution, + CodeWikiRunResponse, + CodeWikiSourceType, + CodeWikiSummary, +} from '@/types/code-wiki' +import client from './client' + +export const codeWikiApi = { + /** + * Code wikis whose repositories the caller can read. + * + * Separate from the knowledge base list on purpose: a code wiki belongs to the wiki + * account, so it appears in none of that list's scopes. + */ + list: async (params?: { page?: number; limit?: number }): Promise => { + const query = new URLSearchParams() + if (params?.page) query.append('page', String(params.page)) + if (params?.limit) query.append('limit', String(params.limit)) + const suffix = query.toString() + return client.get( + `/knowledge-bases/code-wikis${suffix ? `?${suffix}` : ''}` + ) + }, + + /** + * What is known about a repository, before deciding to bind a wiki to it. + * + * Answers 200 with `exists: false` for one the caller cannot read: this assists + * the form rather than asserting something is missing, and a 404 would also make + * private and absent distinguishable, which they must not be. + */ + resolve: async ( + source_type: CodeWikiSourceType, + source_url: string + ): Promise => + client.post('/knowledge-bases/code-wikis/resolve', { + source_type, + source_url, + }), + + /** + * Bind a repository and create its wiki, or return the one it already has. + * + * One repository has one wiki, so this answers 200 with the existing one rather + * than refusing — the caller wanted that repository's wiki, not the act of + * creating it. + */ + create: async (data: CodeWikiCreateRequest): Promise => + client.post('/knowledge-bases/code-wikis', data), + + /** + * The navigation: every published page, already nested and ordered. + * + * Assembled server-side because the hierarchy is in the page paths and the order + * is on the knowledge base — merging them here would be a second place for the + * tree to be wrong. + */ + pages: async (knowledgeBaseId: number): Promise => + client.get(`/knowledge-bases/${knowledgeBaseId}/code-wiki/pages`), + + /** + * Regenerate now, without waiting for a schedule. + * + * Answers 202 even when nothing was needed; read `started` to tell which happened. + */ + regenerate: async (knowledgeBaseId: number): Promise => + client.post( + `/knowledge-bases/${knowledgeBaseId}/code-wiki/generations`, + {} + ), +} diff --git a/frontend/src/app/(tasks)/knowledge/[namespace]/[kbName]/[[...docPath]]/page.tsx b/frontend/src/app/(tasks)/knowledge/[namespace]/[kbName]/[[...docPath]]/page.tsx index 721be12753..b3ca981c80 100644 --- a/frontend/src/app/(tasks)/knowledge/[namespace]/[kbName]/[[...docPath]]/page.tsx +++ b/frontend/src/app/(tasks)/knowledge/[namespace]/[kbName]/[[...docPath]]/page.tsx @@ -21,7 +21,7 @@ * /knowledge/{namespace}/{kbName}/path/doc.md */ -import { Suspense, useState, useCallback, useEffect } from 'react' +import { Suspense, useState, useEffect } from 'react' import dynamic from 'next/dynamic' import { useParams, useRouter } from 'next/navigation' import { BookOpen, FileText } from 'lucide-react' @@ -42,7 +42,6 @@ import { useTaskSession } from '@/features/tasks/session/TaskSession' import { paths } from '@/config/paths' import { Spinner } from '@/components/ui/spinner' import { useWikiProjects } from '@/features/knowledge/useWikiProjects' -import { KnowledgeTabs } from '@/features/knowledge/KnowledgeTabs' import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' import { useTranslation } from '@/hooks/useTranslation' import type { KnowledgeView } from '@/types/knowledge' @@ -65,9 +64,6 @@ const KnowledgeDocumentPage = dynamic( { ssr: false } ) -// Storage key for knowledge sidebar collapsed state -const KNOWLEDGE_SIDEBAR_COLLAPSED_KEY = 'knowledge-sidebar-collapsed' - function KnowledgeVirtualPageContent() { const params = useParams() const router = useRouter() @@ -107,48 +103,11 @@ function KnowledgeVirtualPageContent() { // Mobile sidebar state const [isMobileSidebarOpen, setIsMobileSidebarOpen] = useState(false) - - // Knowledge sidebar collapsed state (for document tab) - const [isKnowledgeSidebarCollapsed, setIsKnowledgeSidebarCollapsed] = useState(() => { - if (typeof window !== 'undefined') { - return localStorage.getItem(KNOWLEDGE_SIDEBAR_COLLAPSED_KEY) === 'true' - } - return false - }) - const [knowledgeViewState, setKnowledgeViewState] = useState({ visible: false, currentView: 'notebook', }) - // Listen for knowledge sidebar collapse changes from KnowledgeDocumentPageDesktop - useEffect(() => { - const handleCollapseChange = (event: CustomEvent<{ collapsed: boolean }>) => { - setIsKnowledgeSidebarCollapsed(event.detail.collapsed) - } - - window.addEventListener( - 'knowledge-sidebar-collapse-change', - handleCollapseChange as EventListener - ) - - return () => { - window.removeEventListener( - 'knowledge-sidebar-collapse-change', - handleCollapseChange as EventListener - ) - } - }, []) - - // Handle expanding the knowledge sidebar from TopNavigation - const handleExpandKnowledgeSidebar = useCallback(() => { - setIsKnowledgeSidebarCollapsed(false) - localStorage.setItem(KNOWLEDGE_SIDEBAR_COLLAPSED_KEY, 'false') - window.dispatchEvent( - new CustomEvent('knowledge-sidebar-collapse-change', { detail: { collapsed: false } }) - ) - }, []) - useEffect(() => { saveLastTab('wiki') }, []) @@ -228,18 +187,6 @@ function KnowledgeVirtualPageContent() { { - if (tab === 'code') { - router.push('/knowledge?type=code') - } - }} - isKnowledgeSidebarCollapsed={isKnowledgeSidebarCollapsed} - onExpandClick={handleExpandKnowledgeSidebar} - /> - } onMobileSidebarToggle={() => setIsMobileSidebarOpen(true)} isSidebarCollapsed={isTaskSidebarCollapsed} > diff --git a/frontend/src/app/(tasks)/knowledge/code-wiki/[knowledgeBaseId]/page.tsx b/frontend/src/app/(tasks)/knowledge/code-wiki/[knowledgeBaseId]/page.tsx new file mode 100644 index 0000000000..556ce229d2 --- /dev/null +++ b/frontend/src/app/(tasks)/knowledge/code-wiki/[knowledgeBaseId]/page.tsx @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useEffect, useState } from 'react' +import { useParams, useRouter } from 'next/navigation' +import { ArrowLeft } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Spinner } from '@/components/ui/spinner' +import TopNavigation from '@/features/layout/TopNavigation' +import UserMenu from '@/features/layout/UserMenu' +import { GithubStarButton } from '@/features/layout/GithubStarButton' +import { CodeWikiReader } from '@/features/knowledge/code-wiki/CodeWikiReader' +import { codeWikiApi } from '@/apis/code-wiki' +import type { CodeWikiSummary } from '@/types/code-wiki' +import '@/app/tasks/tasks.css' +import '@/features/common/scrollbar.css' + +export default function CodeWikiPage() { + const params = useParams() + const router = useRouter() + const knowledgeBaseId = Number(params.knowledgeBaseId) + + const [wiki, setWiki] = useState(null) + const [loading, setLoading] = useState(true) + + useEffect(() => { + // The list is the only place a wiki's summary comes from, and it is the same + // request that decides whether this reader may show anything at all. + let cancelled = false + codeWikiApi + .list() + .then(response => { + if (cancelled) return + setWiki(response.items.find(item => item.id === knowledgeBaseId) ?? null) + }) + .finally(() => { + if (!cancelled) setLoading(false) + }) + return () => { + cancelled = true + } + }, [knowledgeBaseId]) + + return ( +
+ + + + + +
+
+ +
+ + {loading ? ( +
+ +
+ ) : wiki ? ( + + ) : ( +

404

+ )} +
+
+ ) +} diff --git a/frontend/src/app/(tasks)/knowledge/page.tsx b/frontend/src/app/(tasks)/knowledge/page.tsx index 1326493f2c..64b1c3f878 100644 --- a/frontend/src/app/(tasks)/knowledge/page.tsx +++ b/frontend/src/app/(tasks)/knowledge/page.tsx @@ -4,9 +4,9 @@ 'use client' -import { Suspense, useState, useEffect, useCallback } from 'react' +import { Suspense, useState, useEffect } from 'react' import dynamic from 'next/dynamic' -import { useRouter, useSearchParams } from 'next/navigation' +import { useRouter } from 'next/navigation' import { BookOpen, FileText } from 'lucide-react' import TopNavigation from '@/features/layout/TopNavigation' import { @@ -21,32 +21,15 @@ import { GithubStarButton } from '@/features/layout/GithubStarButton' import { ThemeToggle } from '@/features/theme/ThemeToggle' import { useTranslation } from '@/hooks/useTranslation' import { saveLastTab } from '@/utils/userPreferences' -import { useUser } from '@/features/common/UserContext' import { useIsMobile } from '@/features/layout/hooks/useMediaQuery' import { useTaskSession } from '@/features/tasks/session/TaskSession' import { paths } from '@/config/paths' import { Spinner } from '@/components/ui/spinner' -import { useWikiProjects } from '@/features/knowledge/useWikiProjects' -import { SearchBox } from '@/features/knowledge/SearchBox' -import { KnowledgeTabs } from '@/features/knowledge/KnowledgeTabs' import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' import type { KnowledgeView } from '@/types/knowledge' -import type { KnowledgeTabType } from '@/features/knowledge/KnowledgeTabs' import type { KnowledgeViewState } from '@/features/knowledge/document/components/KnowledgeDocumentPage' import { useKnowledgeTaskSidebar } from '@/features/knowledge/document/hooks/useKnowledgeTaskSidebar' -const WikiProjectList = dynamic(() => import('@/features/knowledge/WikiProjectList'), { - ssr: false, -}) - -const AddRepoModal = dynamic(() => import('@/features/knowledge/AddRepoModal'), { - ssr: false, -}) - -const CancelConfirmDialog = dynamic(() => import('@/features/knowledge/CancelConfirmDialog'), { - ssr: false, -}) - const KnowledgeDocumentPage = dynamic( () => import('@/features/knowledge/document/components/KnowledgeDocumentPage').then(mod => ({ @@ -55,15 +38,10 @@ const KnowledgeDocumentPage = dynamic( { ssr: false } ) -// Storage key for knowledge sidebar collapsed state -const KNOWLEDGE_SIDEBAR_COLLAPSED_KEY = 'knowledge-sidebar-collapsed' - // Main knowledge page content with URL parameter support function KnowledgePageContent() { const { t } = useTranslation() const router = useRouter() - const searchParams = useSearchParams() - const { user } = useUser() const { selectTask } = useTaskSession() const isMobile = useIsMobile() const [knowledgeViewState, setKnowledgeViewState] = useState({ @@ -71,178 +49,54 @@ function KnowledgePageContent() { currentView: 'notebook', }) - // Get initial knowledge type tab from URL parameter - const getInitialKnowledgeTab = useCallback((): KnowledgeTabType => { - const type = searchParams.get('type') - if (type === 'code') return 'code' - return 'document' // default - }, [searchParams]) - - // Use shared Hook to manage all state and logic - const { - projects, - loading, - loadingMore, - error, - cancellingIds, - hasMore, - isModalOpen, - formErrors, - isSubmitting, - confirmDialogOpen, - selectedRepo, - // Wiki config state (system-level configuration) - wikiConfig, - loadProjects, - loadMoreProjects, - handleAddRepo, - handleCloseModal, - handleRepoChange, - handleSubmit, - handleCancelClick, - confirmCancelGeneration, - setConfirmDialogOpen, - setPendingCancelProjectId, - } = useWikiProjects() - - // Active knowledge tab - initialized from URL - const [activeTab, setActiveTab] = useState(getInitialKnowledgeTab) - - // Search term for project list - const [mainSearchTerm, setMainSearchTerm] = useState('') - // Mobile sidebar state const [isMobileSidebarOpen, setIsMobileSidebarOpen] = useState(false) - // Knowledge sidebar collapsed state (for document tab) - // This is synced with KnowledgeDocumentPageDesktop via localStorage and custom events - const [isKnowledgeSidebarCollapsed, setIsKnowledgeSidebarCollapsed] = useState(() => { - if (typeof window !== 'undefined') { - return localStorage.getItem(KNOWLEDGE_SIDEBAR_COLLAPSED_KEY) === 'true' - } - return false - }) - - // Listen for knowledge sidebar collapse changes from KnowledgeDocumentPageDesktop - useEffect(() => { - const handleCollapseChange = (event: CustomEvent<{ collapsed: boolean }>) => { - setIsKnowledgeSidebarCollapsed(event.detail.collapsed) - } - - window.addEventListener( - 'knowledge-sidebar-collapse-change', - handleCollapseChange as EventListener - ) - - return () => { - window.removeEventListener( - 'knowledge-sidebar-collapse-change', - handleCollapseChange as EventListener - ) - } - }, []) - - // Handle expanding the knowledge sidebar from TopNavigation - const handleExpandKnowledgeSidebar = useCallback(() => { - setIsKnowledgeSidebarCollapsed(false) - localStorage.setItem(KNOWLEDGE_SIDEBAR_COLLAPSED_KEY, 'false') - // Dispatch event to notify KnowledgeDocumentPageDesktop - window.dispatchEvent( - new CustomEvent('knowledge-sidebar-collapse-change', { detail: { collapsed: false } }) - ) - }, []) - - // Handle knowledge type tab change with URL update - const handleTabChange = useCallback( - (tab: KnowledgeTabType) => { - setActiveTab(tab) - // Update URL - preserve other params like tab and group for document type - if (tab === 'code') { - router.replace('?type=code') - } else { - // For document tab, just set type=document (sub-tab state is managed by KnowledgeDocumentPage) - router.replace('?type=document') - } - }, - [router] - ) - - const navigateToKnowledgeDetail = (projectId: number) => { - router.push(`/knowledge/project/${projectId}?from=code`) - } - - const navigateToTask = (taskId: number) => { - router.push(`/chat?taskId=${taskId}`) - } - - const knowledgeViewSwitcher = - activeTab === 'document' && knowledgeViewState.visible ? ( - knowledgeViewState.onViewChange?.(value as KnowledgeView)} - className="flex-shrink-0" - > - - - - - {t('knowledge:document.knowledgeBase.typeClassic')} - - - - - - {t('knowledge:document.knowledgeBase.typeNotebook')} - - - - - ) : null + const knowledgeViewSwitcher = knowledgeViewState.visible ? ( + knowledgeViewState.onViewChange?.(value as KnowledgeView)} + className="flex-shrink-0" + > + + + + + {t('knowledge:document.knowledgeBase.typeClassic')} + + + + + + {t('knowledge:document.knowledgeBase.typeNotebook')} + + + + + ) : null const isWorkspaceView = - activeTab === 'document' && - knowledgeViewState.visible && - knowledgeViewState.currentView === 'notebook' + knowledgeViewState.visible && knowledgeViewState.currentView === 'notebook' const { isCollapsed: isTaskSidebarCollapsed, toggle: handleToggleCollapsed } = useKnowledgeTaskSidebar({ isMobile, isWorkspaceView, }) - // Filter projects to show only those with user's generations - // This ensures the knowledge page only shows projects created by the current user - const userProjects = projects.filter(project => { - // Check if user has any generations for this project - return ( - project.generations && - project.generations.length > 0 && - (project.generations[0].status === 'RUNNING' || - project.generations[0].status === 'COMPLETED' || - project.generations[0].status === 'PENDING' || - project.generations[0].status === 'FAILED' || - project.generations[0].status === 'CANCELLED') - ) - }) - useEffect(() => { saveLastTab('wiki') }, []) - useEffect(() => { - if (!user) return - loadProjects() - }, [user, loadProjects]) - // Handle new task from collapsed sidebar button const handleNewTask = () => { // IMPORTANT: Clear selected task FIRST to ensure UI state is reset immediately @@ -283,14 +137,6 @@ function KnowledgePageContent() { - } onMobileSidebarToggle={() => setIsMobileSidebarOpen(true)} isSidebarCollapsed={isTaskSidebarCollapsed} > @@ -299,67 +145,13 @@ function KnowledgePageContent() { {/* Content area based on active tab */} - {activeTab === 'code' && ( -
- {/* Center search box - using shared component */} - - {/* Project list */} - -
- )} - {/* Document knowledge - no padding, full height */} - {activeTab === 'document' && ( + {
- )} + }
- - {/* Add repository modal */} - {isModalOpen && ( - - )} - {/* Cancel confirm dialog */} - {confirmDialogOpen && ( - { - setConfirmDialogOpen(false) - setPendingCancelProjectId(null) - }} - onConfirm={confirmCancelGeneration} - /> - )} ) } diff --git a/frontend/src/features/knowledge/code-wiki/CodeWikiReader.tsx b/frontend/src/features/knowledge/code-wiki/CodeWikiReader.tsx new file mode 100644 index 0000000000..52fcf8ade9 --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/CodeWikiReader.tsx @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useCallback, useEffect, useMemo, useState } from 'react' +import { ArrowLeft, RefreshCw } from 'lucide-react' +import { toast } from 'sonner' +import { Button } from '@/components/ui/button' +import { Spinner } from '@/components/ui/spinner' +import { ChatArea } from '@/features/tasks/components/chat' +import { useTeamContext } from '@/contexts/TeamContext' +import { useTranslation } from '@/hooks/useTranslation' +import { codeWikiApi } from '@/apis/code-wiki' +import type { CodeWikiPageNode, CodeWikiSummary } from '@/types/code-wiki' +import { PageOutline } from './PageOutline' +import { WikiNavigation } from './WikiNavigation' +import { WikiPageContent } from './WikiPageContent' + +interface CodeWikiReaderProps { + wiki: CodeWikiSummary +} + +/** Depth-first, so "the first page" means the first one the reader would see. */ +const firstReadable = (nodes: CodeWikiPageNode[]): CodeWikiPageNode | null => { + for (const node of nodes) { + if (node.has_content) return node + const child = firstReadable(node.children) + if (child) return child + } + return null +} + +const findByPath = (nodes: CodeWikiPageNode[], path: string): CodeWikiPageNode | null => { + for (const node of nodes) { + if (node.path === path) return node + const found = findByPath(node.children, path) + if (found) return found + } + return null +} + +/** + * Three regions: the wiki's structure, the page, and the page's own outline. + * + * The middle switches between reading and a conversation rather than splitting. + * They never need to be visible at once — asking something replaces the page with + * the exchange, and going back returns to the page still scrolled where it was, + * which is what opening the conversation on its own route would lose. + */ +export function CodeWikiReader({ wiki }: CodeWikiReaderProps) { + const { t } = useTranslation() + const { teams, isTeamsLoading, refreshTeams } = useTeamContext() + + const [pages, setPages] = useState([]) + const [loading, setLoading] = useState(true) + const [activePath, setActivePath] = useState('') + const [markdown, setMarkdown] = useState('') + const [mode, setMode] = useState<'read' | 'chat'>('read') + const [regenerating, setRegenerating] = useState(false) + const [scrollHost, setScrollHost] = useState(null) + + useEffect(() => { + let cancelled = false + setLoading(true) + codeWikiApi + .pages(wiki.id) + .then(response => { + if (cancelled) return + setPages(response.pages) + const first = firstReadable(response.pages) + if (first) setActivePath(first.path) + }) + .catch(error => { + if (!cancelled) toast.error(error instanceof Error ? error.message : String(error)) + }) + .finally(() => { + if (!cancelled) setLoading(false) + }) + return () => { + cancelled = true + } + }, [wiki.id]) + + const activePage = useMemo( + () => (activePath ? findByPath(pages, activePath) : null), + [pages, activePath] + ) + + const handleRegenerate = useCallback(async () => { + setRegenerating(true) + try { + const result = await codeWikiApi.regenerate(wiki.id) + // "Nothing to do" is the answer the caller asked for, not a failure: the + // repository has not moved since the published version. + toast.success( + result.started + ? t('knowledge:codeWiki.reader.started') + : t('knowledge:codeWiki.reader.upToDate') + ) + } catch (error) { + toast.error(error instanceof Error ? error.message : String(error)) + } finally { + setRegenerating(false) + } + }, [wiki.id, t]) + + const knowledgeTeams = useMemo( + () => teams.filter(team => team.bind_mode?.includes('chat') ?? true), + [teams] + ) + + if (loading) { + return ( +
+ +
+ ) + } + + if (pages.length === 0) { + return ( +
+

{t('knowledge:codeWiki.reader.empty')}

+

{t('knowledge:codeWiki.reader.emptyHint')}

+ +
+ ) + } + + return ( +
+
+ { + setActivePath(node.path) + setMode('read') + }} + /> +
+ +
+
+ {mode === 'chat' && ( + + )} + + {activePage?.title ?? wiki.name} + + +
+ + {mode === 'read' ? ( +
+
+ +
+ +
+ ) : ( +
+ +
+ )} +
+ + {mode === 'read' && } +
+ ) +} diff --git a/frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx b/frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx new file mode 100644 index 0000000000..d8cb594f80 --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/CodeWikiSourceFields.tsx @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useCallback, useEffect, useMemo, useState } from 'react' +import { AlertCircle, CheckCircle2, Loader2, Users } from 'lucide-react' +import { Input } from '@/components/ui/input' +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select' +import { RepositorySelector } from '@/features/tasks/components/selector' +import { SimpleConfigRow } from '@/features/settings/components/team-edit/SimpleConfigLayout' +import { useTranslation } from '@/hooks/useTranslation' +import { codeWikiApi } from '@/apis/code-wiki' +import type { GitRepoInfo } from '@/types/api' +import type { CodeWikiResolution, CodeWikiSourceType } from '@/types/code-wiki' + +export interface CodeWikiSource { + source_type: CodeWikiSourceType + source_url: string + language: string + /** Set once resolved; the parent uses it to know the form is usable. */ + resolution: CodeWikiResolution | null +} + +interface CodeWikiSourceFieldsProps { + value: CodeWikiSource + onChange: (next: CodeWikiSource) => void +} + +const SOURCE_TYPES: CodeWikiSourceType[] = ['github', 'gitlab', 'gitea'] + +/** + * Infer which platform hosts a URL, or `null` when it cannot be told. + * + * A self-hosted GitLab or Gitea is not recognisable from its domain, so guessing + * would bind the wiki to a provider that answers meaninglessly about it. Returning + * null makes the caller ask instead. + */ +function inferSourceType(url: string): CodeWikiSourceType | null { + const host = url.match(/^https?:\/\/([^/]+)/)?.[1]?.toLowerCase() + if (!host) return null + if (host === 'github.com') return 'github' + if (host === 'gitlab.com') return 'gitlab' + if (host === 'gitea.com') return 'gitea' + return null +} + +/** + * Where the repository comes from, by picking one or by naming one. + * + * Both are needed. The selector lists only repositories the caller is a member of, + * so a public repository they can read in a browser never appears in it — and a wiki + * that could not be built for one would be more closed than the repository itself. + * + * The shared RepositorySelector is embedded rather than extended: it is used by task + * creation too, and its list is membership-scoped on the server, which is not + * something a component can change. + */ +export function CodeWikiSourceFields({ value, onChange }: CodeWikiSourceFieldsProps) { + const { t } = useTranslation() + const [mode, setMode] = useState<'select' | 'url'>('select') + const [repo, setRepo] = useState(null) + const [urlDraft, setUrlDraft] = useState('') + const [resolving, setResolving] = useState(false) + + const resolution = value.resolution + + const resolve = useCallback( + async (sourceType: CodeWikiSourceType, url: string) => { + if (!url) { + onChange({ ...value, source_url: '', resolution: null }) + return + } + setResolving(true) + try { + const resolved = await codeWikiApi.resolve(sourceType, url) + onChange({ + ...value, + source_type: sourceType, + source_url: url, + resolution: resolved, + }) + } catch { + // A failed probe is not a failed form: the caller may still submit, and the + // create call applies the same check authoritatively. + onChange({ ...value, source_type: sourceType, source_url: url, resolution: null }) + } finally { + setResolving(false) + } + }, + [onChange, value] + ) + + const handleRepoChange = useCallback( + (next: GitRepoInfo | null) => { + setRepo(next) + if (!next) { + onChange({ ...value, source_url: '', resolution: null }) + return + } + const sourceType = (next.type as CodeWikiSourceType) ?? 'github' + void resolve(sourceType, next.git_url) + }, + [onChange, resolve, value] + ) + + // Debounced so that typing a URL costs one probe rather than one per keystroke — + // anonymous GitHub requests are capped at 60 an hour per address. + useEffect(() => { + if (mode !== 'url') return + const inferred = inferSourceType(urlDraft) + const timer = setTimeout(() => { + void resolve(inferred ?? value.source_type, urlDraft.trim()) + }, 600) + return () => clearTimeout(timer) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [urlDraft, mode]) + + const inferred = useMemo(() => inferSourceType(urlDraft), [urlDraft]) + + return ( + <> + + {t('knowledge:codeWiki.create.repository')} * + + } + align="start" + > +
+
+ {(['select', 'url'] as const).map(option => ( + + ))} +
+ + {mode === 'select' ? ( + + ) : ( +
+ {inferred === null && urlDraft.trim() !== '' && ( + + )} + setUrlDraft(e.target.value)} + placeholder="https://github.com/owner/repo" + data-testid="code-wiki-source-url" + /> +
+ )} + + +
+ + + + + + + ) +} + +function SourceStatus({ + resolving, + resolution, +}: { + resolving: boolean + resolution: CodeWikiResolution | null +}) { + const { t } = useTranslation() + + if (resolving) { + return ( +

+ + {t('knowledge:codeWiki.create.checking')} +

+ ) + } + if (!resolution) return null + + if (!resolution.exists) { + return ( +

+ + {t('knowledge:codeWiki.create.notReadable')} +

+ ) + } + + return ( +
+

+ + {resolution.visibility === 'public' + ? t('knowledge:codeWiki.create.publicRepository') + : t('knowledge:codeWiki.create.privateRepository')} + {resolution.default_branch ? ` · ${resolution.default_branch}` : ''} +

+ {resolution.existing_wikis.length > 0 && ( +
+

+ + {t('knowledge:codeWiki.create.alreadyBuilt', { + count: resolution.existing_wikis.length, + })} +

+ {/* Named, not counted: asking for a share needs somebody to ask. One the + caller can already open is a link instead — there is nothing to ask + for. */} +
    + {resolution.existing_wikis.map(wiki => ( +
  • + {wiki.accessible ? ( + + {wiki.name} + + ) : ( + + {wiki.name} + {wiki.owner_name + ? ` — ${t('knowledge:codeWiki.create.ownedBy', { + owner: wiki.owner_name, + })}` + : ''} + + )} +
    • + ))} +
+
+ )} +
+ ) +} diff --git a/frontend/src/features/knowledge/code-wiki/PageOutline.tsx b/frontend/src/features/knowledge/code-wiki/PageOutline.tsx new file mode 100644 index 0000000000..b2f93cfaa3 --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/PageOutline.tsx @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useEffect, useMemo, useRef, useState } from 'react' +import { useTranslation } from '@/hooks/useTranslation' + +interface Heading { + id: string + text: string + /** 2 or 3; deeper levels are folded into 3 so the rail stays narrow. */ + level: number +} + +interface PageOutlineProps { + /** Raw markdown of the page being read. */ + content: string + /** The element headings are rendered into, watched for scroll position. */ + scrollContainer: HTMLElement | null +} + +/** Fenced blocks are skipped: `# ` inside one is a comment, not a heading. */ +const collectHeadings = (markdown: string): Heading[] => { + const headings: Heading[] = [] + const seen = new Map() + let inFence = false + + for (const line of markdown.split('\n')) { + if (line.trimStart().startsWith('```')) { + inFence = !inFence + continue + } + if (inFence) continue + + const match = /^(#{2,6})\s+(.+?)\s*$/.exec(line) + if (!match) continue + + const text = match[2].replace(/[*_`]/g, '').trim() + if (!text) continue + + // Two sections can share a title, and an id that repeats would scroll to the + // first one from every entry. + const base = text.toLowerCase().replace(/[^\w一-龥]+/g, '-') + const count = seen.get(base) ?? 0 + seen.set(base, count + 1) + + headings.push({ + id: count === 0 ? base : `${base}-${count}`, + text, + level: Math.min(match[1].length, 3), + }) + } + + return headings +} + +/** + * The current page's headings, as a rail beside the content. + * + * Separate from the wiki navigation on the left, which moves between pages: this + * moves within one. A generated page runs long enough that reaching a section by + * scrolling is the slow way to do it. + */ +export function PageOutline({ content, scrollContainer }: PageOutlineProps) { + const { t } = useTranslation() + const headings = useMemo(() => collectHeadings(content), [content]) + const [activeId, setActiveId] = useState('') + const observer = useRef(null) + + useEffect(() => { + if (!scrollContainer || headings.length === 0) return + + // rootMargin pulls the trigger line to the top quarter of the viewport, so the + // highlighted entry is the section being read rather than the one just scrolled + // past. + observer.current = new IntersectionObserver( + entries => { + const visible = entries + .filter(entry => entry.isIntersecting) + .sort((a, b) => a.boundingClientRect.top - b.boundingClientRect.top) + if (visible[0]) setActiveId(visible[0].target.id) + }, + { root: scrollContainer, rootMargin: '0px 0px -75% 0px', threshold: 0 } + ) + + for (const heading of headings) { + const element = scrollContainer.querySelector(`#${CSS.escape(heading.id)}`) + if (element) observer.current.observe(element) + } + + return () => observer.current?.disconnect() + }, [headings, scrollContainer]) + + if (headings.length === 0) return null + + const jumpTo = (id: string) => { + const element = scrollContainer?.querySelector(`#${CSS.escape(id)}`) + element?.scrollIntoView({ behavior: 'smooth', block: 'start' }) + } + + return ( + + ) +} diff --git a/frontend/src/features/knowledge/code-wiki/WikiNavigation.tsx b/frontend/src/features/knowledge/code-wiki/WikiNavigation.tsx new file mode 100644 index 0000000000..51bf152b27 --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/WikiNavigation.tsx @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useEffect, useState } from 'react' +import { ChevronDown, ChevronRight } from 'lucide-react' +import { useTranslation } from '@/hooks/useTranslation' +import type { CodeWikiPageNode } from '@/types/code-wiki' + +interface WikiNavigationProps { + pages: CodeWikiPageNode[] + activePath: string + onSelect: (node: CodeWikiPageNode) => void +} + +/** Every ancestor of the active page, so opening a deep page reveals where it sits. */ +const ancestorsOf = (path: string): string[] => { + const parts = path.split('/') + return parts.slice(0, -1).map((_, index) => parts.slice(0, index + 1).join('/')) +} + +interface NodeProps { + node: CodeWikiPageNode + depth: number + activePath: string + expanded: Set + onToggle: (path: string) => void + onSelect: (node: CodeWikiPageNode) => void +} + +function NavigationNode({ node, depth, activePath, expanded, onToggle, onSelect }: NodeProps) { + const { t } = useTranslation() + const hasChildren = node.children.length > 0 + const isOpen = expanded.has(node.path) + const isActive = node.path === activePath + + return ( +
  • +
    + {hasChildren ? ( + + ) : ( + + )} + + +
    + + {hasChildren && isOpen && ( +
      + {node.children.map(child => ( + + ))} +
    + )} +
    • + ) +} + +/** + * The wiki's own structure, derived server-side from page paths. + * + * A read-only relative of the knowledge base folder tree: no upload, rename or + * delete, because everything here is a projection the next publish rewrites. + */ +export function WikiNavigation({ pages, activePath, onSelect }: WikiNavigationProps) { + const [expanded, setExpanded] = useState>(new Set()) + + useEffect(() => { + // Follows the active page rather than replacing the set, so a section the + // reader opened by hand stays open when they move elsewhere. + if (!activePath) return + setExpanded(current => { + const next = new Set(current) + for (const ancestor of ancestorsOf(activePath)) next.add(ancestor) + return next + }) + }, [activePath]) + + const toggle = (path: string) => + setExpanded(current => { + const next = new Set(current) + if (next.has(path)) next.delete(path) + else next.add(path) + return next + }) + + return ( + + ) +} diff --git a/frontend/src/features/knowledge/code-wiki/WikiPageContent.tsx b/frontend/src/features/knowledge/code-wiki/WikiPageContent.tsx new file mode 100644 index 0000000000..0941c2adda --- /dev/null +++ b/frontend/src/features/knowledge/code-wiki/WikiPageContent.tsx @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +'use client' + +import { useEffect, useState } from 'react' +import dynamic from 'next/dynamic' +import { Spinner } from '@/components/ui/spinner' +import { useTheme } from '@/features/theme/ThemeProvider' +import { useTranslation } from '@/hooks/useTranslation' +import { getDocumentContent } from '@/apis/knowledge' +import type { CodeWikiPageNode } from '@/types/code-wiki' + +const EnhancedMarkdown = dynamic(() => import('@/components/common/EnhancedMarkdown'), { + ssr: false, +}) + +interface WikiPageContentProps { + page: CodeWikiPageNode | null + onContentChange: (markdown: string) => void +} + +/** + * One page's body. + * + * Rendered through `EnhancedMarkdown`, the same component the knowledge base + * document viewer uses — which already handles Mermaid with a fallback to the raw + * source when a diagram will not render, and resolves wiki links. The old wiki + * reader had its own parallel implementation of all of it, including a Mermaid + * component with no error handling at all. + */ +export function WikiPageContent({ page, onContentChange }: WikiPageContentProps) { + const { t } = useTranslation() + const { theme } = useTheme() + const [markdown, setMarkdown] = useState('') + const [loading, setLoading] = useState(false) + + useEffect(() => { + if (!page?.document_id) { + setMarkdown('') + onContentChange('') + return + } + + let cancelled = false + setLoading(true) + getDocumentContent(page.document_id, 0, 1) + .then(response => { + if (cancelled) return + const body = response.content ?? '' + setMarkdown(body) + onContentChange(body) + }) + .catch(() => { + if (!cancelled) { + setMarkdown('') + onContentChange('') + } + }) + .finally(() => { + if (!cancelled) setLoading(false) + }) + + return () => { + cancelled = true + } + }, [page?.document_id, onContentChange]) + + if (!page) return null + + if (!page.has_content) { + return ( +

    + {t('knowledge:codeWiki.reader.sectionOnly')} +

    + ) + } + + if (loading) { + return ( +
    + +
    + ) + } + + return ( +
    + +
    + ) +} diff --git a/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx b/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx index 567aec1732..f9ab610950 100644 --- a/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx +++ b/frontend/src/features/knowledge/document/components/CreateKnowledgeBaseDialog.tsx @@ -5,7 +5,7 @@ 'use client' import { useState, useEffect } from 'react' -import { BookOpen, Database, User, Building2, Users, FileText } from 'lucide-react' +import { BookOpen, Code2, Database, User, Building2, Users, FileText } from 'lucide-react' import { Dialog, DialogContent, @@ -24,6 +24,10 @@ import { } from '@/components/ui/select' import { useTranslation } from '@/hooks/useTranslation' import { SimpleConfigRow } from '@/features/settings/components/team-edit/SimpleConfigLayout' +import { + CodeWikiSourceFields, + type CodeWikiSource, +} from '@/features/knowledge/code-wiki/CodeWikiSourceFields' import type { DirectAccessRequirement, KnowledgeBaseCreate, @@ -97,6 +101,13 @@ function createDefaultRetrievalConfig(): RetrievalConfigDraft { } } +/** Documents are uploaded and organised; code is generated from a repository. */ +type KnowledgeBaseKind = 'document' | 'code' + +function createEmptySource(): CodeWikiSource { + return { source_type: 'github', source_url: '', language: 'zh', resolution: null } +} + export function CreateKnowledgeBaseDialog({ open, onOpenChange, @@ -118,6 +129,10 @@ export function CreateKnowledgeBaseDialog({ useState('read') // Selected KB type (can be changed by user) const [selectedKbType, setSelectedKbType] = useState(initialKbType) + // Which kind of knowledge base is being created. Chosen first because it decides + // which fields even apply: a code wiki has a repository and no opening view. + const [kind, setKind] = useState('document') + const [source, setSource] = useState(createEmptySource) // Default enable summary for all KB types const [summaryEnabled, setSummaryEnabled] = useState(true) const [summaryModelRef, setSummaryModelRef] = useState(null) @@ -160,6 +175,8 @@ export function CreateKnowledgeBaseDialog({ useEffect(() => { if (open) { setSelectedKbType(initialKbType) + setKind('document') + setSource(createEmptySource()) setSelectedGroupId(defaultGroupId || 'personal') setDirectAccessRequirement('read') } @@ -175,11 +192,18 @@ export function CreateKnowledgeBaseDialog({ setSummaryModelError('') clearMultimodalError() - if (!name.trim()) { + // A code wiki may be left unnamed: the server fills in the repository's own + // name. Pre-filling the box here instead would read as the caller's own input. + if (!name.trim() && kind !== 'code') { setError(t('knowledge:document.knowledgeBase.nameRequired')) return } + if (kind === 'code' && !source.source_url) { + setError(t('knowledge:codeWiki.create.repositoryRequired')) + return + } + if (name.length > 100) { setError(t('knowledge:document.knowledgeBase.nameTooLong')) return @@ -219,13 +243,28 @@ export function CreateKnowledgeBaseDialog({ max_calls_per_conversation: maxCalls, exempt_calls_before_check: exemptCalls, selectedGroupId: showGroupSelector ? selectedGroupId : undefined, - kb_type: selectedKbType, + kb_type: kind === 'code' ? 'code_wiki' : selectedKbType, + ...(kind === 'code' + ? { + source_type: source.source_type, + source_url: source.source_url, + language: source.language, + // Left blank, the repository's own name is used. Sent from what the + // form already resolved rather than pre-filled into the box, which + // would read as the caller's own input — and rather than resolved + // again on the server, which has asked the provider once already. + resolved_name: source.resolution?.name, + resolved_description: source.resolution?.description, + } + : {}), }) setName('') setDescription('') setDirectAccessRequirement('read') // Reset selectedKbType and keep summaryEnabled as true setSelectedKbType(initialKbType) + setKind('document') + setSource(createEmptySource()) setSummaryEnabled(true) setSummaryModelRef(null) resetMultimodal() @@ -246,6 +285,8 @@ export function CreateKnowledgeBaseDialog({ setDirectAccessRequirement('read') // Reset selectedKbType and keep summaryEnabled as true setSelectedKbType(initialKbType) + setKind('document') + setSource(createEmptySource()) setSummaryEnabled(true) setSummaryModelRef(null) setSummaryModelError('') @@ -279,57 +320,100 @@ export function CreateKnowledgeBaseDialog({
    + {/* Which kind first: it decides which of the fields below even apply. */} +
    + {( + [ + ['document', FileText, 'knowledge:document.knowledgeBase.kindDocument'], + ['code', Code2, 'knowledge:document.knowledgeBase.kindCode'], + ] as const + ).map(([option, Icon, label]) => ( + + ))} +
    - {/* KB Type selector - subtle style */} - -
    -
    - -
    -
    + {kind === 'code' ? ( + + ) : ( + /* KB Type selector - subtle style */ + +
    +
    + +
    - {isNotebook ? ( - - ) : ( - - )} -
    -
    -
    - {isNotebook - ? t('knowledge:document.knowledgeBase.typeNotebook') - : t('knowledge:document.knowledgeBase.typeClassic')} +
    + {isNotebook ? ( + + ) : ( + + )}
    -
    - {isNotebook - ? t('knowledge:document.knowledgeBase.notebookDesc') - : t('knowledge:document.knowledgeBase.classicDesc')} +
    +
    + {isNotebook + ? t('knowledge:document.knowledgeBase.typeNotebook') + : t('knowledge:document.knowledgeBase.typeClassic')} +
    +
    + {isNotebook + ? t('knowledge:document.knowledgeBase.notebookDesc') + : t('knowledge:document.knowledgeBase.classicDesc')} +
    -
    - + + )} {/* Group selector - only show when showGroupSelector is true */} {showGroupSelector && availableGroups && availableGroups.length > 0 && ( !open && setViewingDoc(null)} document={viewingDoc} knowledgeBaseId={knowledgeBase.id} - kbType={knowledgeBase.kb_type} + kbType={documentViewOf(knowledgeBase.kb_type) ?? undefined} canEdit={viewingDoc ? canManageDocument(viewingDoc) : false} knowledgeBaseName={knowledgeBase.name} knowledgeBaseNamespace={knowledgeBase.namespace || 'default'} @@ -1569,7 +1570,7 @@ export function DocumentList({ onUploadComplete={handleUploadComplete} onTableAdd={handleTableAdd} onWebAdd={handleWebAdd} - kbType={knowledgeBase.kb_type} + kbType={documentViewOf(knowledgeBase.kb_type) ?? undefined} folderId={selectedUploadFolderId} folderOptions={folderOptions} onFolderChange={setSelectedUploadFolderId} diff --git a/frontend/src/features/knowledge/document/components/KnowledgeBaseForm.tsx b/frontend/src/features/knowledge/document/components/KnowledgeBaseForm.tsx index 04c421e3ec..ff3f0f80c7 100644 --- a/frontend/src/features/knowledge/document/components/KnowledgeBaseForm.tsx +++ b/frontend/src/features/knowledge/document/components/KnowledgeBaseForm.tsx @@ -32,6 +32,10 @@ import { useMultimodalFeatureEnabled } from '@/features/knowledge/multimodal/hoo interface KnowledgeBaseFormProps { typeSection?: ReactNode + /** False for a code wiki: left blank, the repository's own name is used. */ + nameRequired?: boolean + /** Overrides the placeholder, to say what a blank name will be filled in with. */ + namePlaceholder?: string name: string description: string onNameChange: (value: string) => void @@ -136,6 +140,8 @@ function FormSection({ export function KnowledgeBaseForm({ typeSection, + nameRequired = true, + namePlaceholder, name, description, onNameChange, @@ -305,7 +311,8 @@ export function KnowledgeBaseForm({ - {t('knowledge:document.knowledgeBase.name')} * + {t('knowledge:document.knowledgeBase.name')} + {nameRequired && *} } > @@ -313,7 +320,7 @@ export function KnowledgeBaseForm({ id="knowledge-name" value={name} onChange={e => onNameChange(e.target.value)} - placeholder={t('knowledge:document.knowledgeBase.namePlaceholder')} + placeholder={namePlaceholder ?? t('knowledge:document.knowledgeBase.namePlaceholder')} maxLength={100} data-testid="kb-name-input" className="bg-base" diff --git a/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts b/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts index 3c18c9e804..9b10d97b3f 100644 --- a/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts +++ b/frontend/src/features/knowledge/document/hooks/useKnowledgeBaseDialogs.ts @@ -146,6 +146,27 @@ export function useKnowledgeBaseDialogs({ const kbType = data.kb_type || createKbType + // A code wiki is created through its own endpoint: it needs a repository, + // passes a repository-access gate, and starts generating straight away. + // Everything after that point is the same knowledge base as any other. + if (kbType === 'code_wiki') { + const { codeWikiApi } = await import('@/apis/code-wiki') + const wiki = await codeWikiApi.create({ + name: data.name || data.resolved_name || '', + description: data.description || data.resolved_description, + namespace, + source_type: data.source_type!, + source_url: data.source_url!, + }) + setShowCreateDialog(false) + resetCreateDialogState() + await sidebar.refreshAll() + // Straight into the reader: generation has already started, and the empty + // state there shows its progress. + router.push(`/knowledge/code-wiki/${wiki.id}`) + return + } + const { createKnowledgeBase } = await import('@/apis/knowledge') await createKnowledgeBase({ name: data.name, diff --git a/frontend/src/features/knowledge/document/hooks/useKnowledgeSidebar.ts b/frontend/src/features/knowledge/document/hooks/useKnowledgeSidebar.ts index 219123f207..c208468623 100644 --- a/frontend/src/features/knowledge/document/hooks/useKnowledgeSidebar.ts +++ b/frontend/src/features/knowledge/document/hooks/useKnowledgeSidebar.ts @@ -14,6 +14,7 @@ import { knowledgeBaseApi } from '@/apis/knowledge-base' import { dingtalkDocApi } from '@/apis/dingtalk-doc' import { getKnowledgeBase } from '@/apis/knowledge' import { useUser } from '@/features/common/UserContext' +import { documentViewOf } from '@/types/knowledge' import type { KnowledgeBase, AllGroupedKnowledgeResponse, @@ -460,7 +461,7 @@ export function useKnowledgeSidebar(): UseKnowledgeSidebarReturn { const newItem: RecentAccessItem = { kbId: kb.id, kbName: kb.name, - kbType: (kb.kb_type as 'notebook' | 'classic') || 'notebook', + kbType: documentViewOf(kb.kb_type) ?? 'notebook', namespace: kb.namespace, accessedAt: Date.now(), } @@ -546,7 +547,7 @@ export function useKnowledgeSidebar(): UseKnowledgeSidebarReturn { ? { ...item, kbName: updatedKb.name, - kbType: updatedKb.kb_type || 'notebook', + kbType: documentViewOf(updatedKb.kb_type) ?? 'notebook', namespace: updatedKb.namespace, } : item diff --git a/frontend/src/features/tasks/components/message/MessagesArea.tsx b/frontend/src/features/tasks/components/message/MessagesArea.tsx index 770cb4db09..89d08b6b8e 100644 --- a/frontend/src/features/tasks/components/message/MessagesArea.tsx +++ b/frontend/src/features/tasks/components/message/MessagesArea.tsx @@ -72,6 +72,7 @@ import { MessageLoadingStage } from './MessageLoadingStage' import { SaveToKnowledgeDialog } from './SaveToKnowledgeDialog' import { DocumentDetailDialog } from '@/features/knowledge/document/components/DocumentDetailDialog' import { ToastAction } from '@/components/ui/toast' +import { documentViewOf } from '@/types/knowledge' import type { KnowledgeBaseWithGroupInfo, KnowledgeDocument } from '@/types/knowledge' type SendMessageOptions = { @@ -1483,7 +1484,7 @@ function MessagesArea({ }} document={knowledgeDocumentToView.document} knowledgeBaseId={knowledgeDocumentToView.knowledgeBase.id} - kbType={knowledgeDocumentToView.knowledgeBase.kb_type} + kbType={documentViewOf(knowledgeDocumentToView.knowledgeBase.kb_type) ?? undefined} canEdit={true} knowledgeBaseName={knowledgeDocumentToView.knowledgeBase.name} knowledgeBaseNamespace={knowledgeDocumentToView.knowledgeBase.namespace} diff --git a/frontend/src/i18n/locales/en/knowledge.json b/frontend/src/i18n/locales/en/knowledge.json index 06bf870f97..35244cbda2 100644 --- a/frontend/src/i18n/locales/en/knowledge.json +++ b/frontend/src/i18n/locales/en/knowledge.json @@ -252,7 +252,11 @@ "defaultViewUpdateSuccess": "Default opening view updated", "defaultViewUpdateFailed": "Failed to update default opening view", "targetGroup": "Target Group", - "selectGroup": "Select Group" + "selectGroup": "Select Group", + "kindDocument": "Documents", + "kindDocumentDesc": "Upload and organise material", + "kindCode": "Code", + "kindCodeDesc": "Generated from a repository" }, "breadcrumb": { "root": "All Documents" @@ -1050,5 +1054,51 @@ "description": "You do not have permission to access this knowledge base. If you have any questions, please contact the knowledge base creator or administrator.", "backButton": "Back to Knowledge Base List" } + }, + "codeWiki": { + "outline": { + "title": "On this page", + "label": "Page outline" + }, + "list": { + "empty": "No code wikis yet", + "emptyHint": "Create one from a repository you can read.", + "neverGenerated": "Not generated yet", + "lastGenerated": "Updated {{when}}", + "atCommit": "at {{commit}}", + "pages_one": "{{count}} page", + "pages_other": "{{count}} pages" + }, + "create": { + "title": "New code wiki", + "repository": "Repository", + "name": "Name", + "namePlaceholder": "Leave blank to use the repository name", + "submit": "Create", + "noBoundModel": "The code wiki agent has no model bound, so generation cannot run.", + "repositoryRequired": "Select or enter a repository address", + "fromMyRepositories": "From my repositories", + "byUrl": "By repository address", + "checking": "Checking the repository…", + "notReadable": "Cannot read this repository. Check the address, or add a token for this domain.", + "publicRepository": "Public repository", + "privateRepository": "Private repository", + "language": "Generation language", + "languageZh": "Chinese", + "languageEn": "English", + "alreadyBuilt": "{{count}} other wiki(s) already document this repository — you could ask for a share", + "ownedBy": "owned by {{owner}} — ask them for a share" + }, + "reader": { + "regenerate": "Regenerate", + "regenerating": "Generating…", + "upToDate": "Already up to date with the repository", + "started": "Generation started", + "back": "Back to the page", + "askPlaceholder": "Ask about this wiki", + "sectionOnly": "This section has no page of its own", + "empty": "This wiki has no pages yet", + "emptyHint": "Generate it to see its documentation here." + } } } diff --git a/frontend/src/i18n/locales/zh-CN/knowledge.json b/frontend/src/i18n/locales/zh-CN/knowledge.json index 5bf97d688c..803a2ab8f0 100644 --- a/frontend/src/i18n/locales/zh-CN/knowledge.json +++ b/frontend/src/i18n/locales/zh-CN/knowledge.json @@ -252,7 +252,11 @@ "defaultViewUpdateSuccess": "默认打开方式已更新", "defaultViewUpdateFailed": "默认打开方式更新失败", "targetGroup": "归属", - "selectGroup": "选择分组" + "selectGroup": "选择分组", + "kindDocument": "文档", + "kindDocumentDesc": "上传与整理资料", + "kindCode": "代码", + "kindCodeDesc": "从仓库生成 wiki" }, "breadcrumb": { "root": "全部文档" @@ -1050,5 +1054,51 @@ "description": "您没有该知识库详细信息的访问权限。如有疑问，请联系知识库创建者或管理员。", "backButton": "返回知识库列表" } + }, + "codeWiki": { + "outline": { + "title": "本页目录", + "label": "页面目录" + }, + "list": { + "empty": "还没有代码 Wiki", + "emptyHint": "从一个你有读权限的仓库创建。", + "neverGenerated": "尚未生成", + "lastGenerated": "更新于 {{when}}", + "atCommit": "commit {{commit}}", + "pages_one": "{{count}} 个页面", + "pages_other": "{{count}} 个页面" + }, + "create": { + "title": "新建代码 Wiki", + "repository": "仓库", + "name": "名称", + "namePlaceholder": "留空则使用仓库名", + "submit": "创建", + "noBoundModel": "代码 Wiki 智能体未绑定模型，无法生成。", + "repositoryRequired": "请选择或填写仓库地址", + "fromMyRepositories": "从我的仓库选择", + "byUrl": "输入仓库地址", + "checking": "正在检查仓库…", + "notReadable": "无法读取该仓库，请检查地址或为该域名配置 token", + "publicRepository": "公开仓库", + "privateRepository": "私有仓库", + "language": "生成语言", + "languageZh": "中文", + "languageEn": "English", + "alreadyBuilt": "已有 {{count}} 人为此仓库建过 wiki，可以向其索要分享", + "ownedBy": "归属 {{owner}}，可向其索要分享" + }, + "reader": { + "regenerate": "重新生成", + "regenerating": "生成中…", + "upToDate": "仓库自上次生成后没有变化", + "started": "已开始生成", + "back": "返回文档", + "askPlaceholder": "就这个 wiki 提问", + "sectionOnly": "该章节没有自己的页面", + "empty": "这个 wiki 还没有页面", + "emptyHint": "生成一次即可在此查看文档。" + } } } diff --git a/frontend/src/types/code-wiki.ts b/frontend/src/types/code-wiki.ts new file mode 100644 index 0000000000..34c99bced1 --- /dev/null +++ b/frontend/src/types/code-wiki.ts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: 2026 Weibo, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * A code wiki is a knowledge base an agent writes from a source repository. + * + * It belongs to whoever created it and is governed by the ordinary knowledge-base + * ACL, so it appears in the general list alongside every other kind. These endpoints + * exist only because its list items carry repository fields; they grant nothing the + * general list would not. + */ +export interface CodeWikiSummary { + id: number + name: string + description?: string | null + /** Repository the wiki documents, e.g. `wecode-ai/Wegent`. */ + project_name: string + source_url: string + /** When the live version was published; null when nothing has been. */ + last_published_at?: string | null + /** Commit the live version documents. */ + last_published_commit: string + document_count: number + created_at: string + updated_at: string +} + +export interface CodeWikiListResponse { + items: CodeWikiSummary[] + total: number +} + +export interface CodeWikiCreateRequest { + /** Optional: left blank, the repository's own name is used. */ + name: string + description?: string + /** Where to file it, as for any knowledge base. Defaults to personal. */ + namespace?: string + source_type: CodeWikiSourceType + source_url: string +} + +export type CodeWikiSourceType = 'github' | 'gitlab' | 'gitea' + +/** A wiki of this repository somebody has already built. */ +export interface CodeWikiExisting { + id: number + name: string + /** Who to ask, when it is not accessible. */ + owner_name: string + /** Whether the caller can already open it. */ + accessible: boolean +} + +/** + * What is known about a repository before a wiki is bound to it. + * + * `exists: false` means "not readable with what you have". Private and absent are + * deliberately indistinguishable — telling them apart would disclose which private + * repositories exist. + */ +export interface CodeWikiResolution { + exists: boolean + /** `public` or `private`, when readable. */ + visibility: string + /** Saves listing branches, which has no path for a repository read anonymously. */ + default_branch: string + /** Repository path, offered as the wiki's name when none is given. */ + name: string + description: string + /** `public`, `member`, or `none`. */ + access: string + /** Wikis that already document this repository, whoever owns them. */ + existing_wikis: CodeWikiExisting[] +} + +/** + * What happened when a wiki was asked to regenerate. + * + * `started: false` is a success, not a failure: the repository had not changed since + * the published version, so there was nothing to build. + */ +export interface CodeWikiRunResponse { + started: boolean + /** `full`, `incremental` or `skip`. */ + mode: string + reason: string + generation_id: number + task_id: number +} + +/** + * One node of the reader's navigation. + * + * Every node is a page. The tree is derived from page paths, and a section that holds + * pages but has no page of its own renders as a heading that cannot be opened — + * allowed, and reported by the publish gate as a warning. + */ +export interface CodeWikiPageNode { + /** Stable path, e.g. `architecture/backend`. Identity, not a display value. */ + path: string + /** What the page is called; the document's name. */ + title: string + /** 0 for a section that holds pages but has none of its own. */ + document_id: number + /** False for such a section: a heading that cannot be opened. */ + has_content: boolean + children: CodeWikiPageNode[] +} + +export interface CodeWikiPageTree { + pages: CodeWikiPageNode[] +} diff --git a/frontend/src/types/knowledge.ts b/frontend/src/types/knowledge.ts index 99bd3097ae..e535f8be85 100644 --- a/frontend/src/types/knowledge.ts +++ b/frontend/src/types/knowledge.ts @@ -303,7 +303,28 @@ export interface SummaryModelRef { // opening view, not a resource capability boundary. // - notebook: default to Notebook workspace view // - classic: default to document management view -export type KnowledgeBaseType = 'notebook' | 'classic' +/** + * How a knowledge base is filled and read. + * + * `code_wiki` is generated from a repository by an agent rather than filled by hand, + * and is read through its own three-pane reader. It is otherwise an ordinary + * knowledge base: same ownership, same ACL, same listing. + */ +export type KnowledgeBaseType = 'notebook' | 'classic' | 'code_wiki' + +/** + * The two views a document knowledge base can open in. + * + * Deliberately narrower than `KnowledgeBaseType`: a code wiki has neither view, it + * has a reader of its own. Components that render one or the other take this type so + * that a code wiki cannot be handed to them without the conversion being visible. + */ +export type DocumentViewType = 'notebook' | 'classic' + +/** The view to open a knowledge base in, or `null` when it is not a document one. */ +export function documentViewOf(kbType?: KnowledgeBaseType | null): DocumentViewType | null { + return kbType === 'notebook' || kbType === 'classic' ? kbType : null +} export type KnowledgeView = 'documents' | 'notebook' export type RagConfigMode = 'auto' | 'disabled' export type DirectAccessRequirement = 'read' | 'edit' @@ -361,6 +382,16 @@ export interface KnowledgeBaseCreate { summary_model_ref?: SummaryModelRef | null /** Default opening view: 'notebook' or 'classic' (documents) */ kb_type?: KnowledgeBaseType + /** Only for `kb_type: 'code_wiki'` — which platform hosts the repository. */ + source_type?: 'github' | 'gitlab' | 'gitea' + /** Only for `kb_type: 'code_wiki'` — the repository to document. */ + source_url?: string + /** Only for `kb_type: 'code_wiki'` — what language to generate the pages in. */ + language?: string + /** What the provider calls the repository, used when the name is left blank. */ + resolved_name?: string + /** What the provider says the repository is, used when no description is given. */ + resolved_description?: string /** Guided questions list (max 3) for notebook mode quick user interaction */ guided_questions?: string[] /** Maximum number of knowledge base tool calls allowed per conversation */