Stabilize/integration ready pipeline

.env.example

@@ -3,7 +3,7 @@ OPENAI_API_KEY=sk-...
 OPENAI_MODEL=openai/gpt-4o-mini
 
 #  Parser 
-# Options: nl2pln | canonical_pln | manhin | langextract
+# Options: nl2pln | canonical_pln | manhin | langextract | canonical_langextract
 PARSER=canonical_pln
 PLNRAG_PARSER=canonical_pln
 NL2PLN_MODULE_PATH=data/simba_all.json
@@ -55,6 +55,6 @@ CONCEPTNET_INDEX_ON_STARTUP=true
 CONCEPTNET_MIN_WEIGHT=2.0
 CONCEPTNET_COVERAGE_PERCENT=100.0
 CONCEPTNET_SAMPLE_SEED=42
-CONCEPTNET_AUTO_REBUILD_ON_CHANGE=true
+CONCEPTNET_AUTO_REBUILD_ON_CHANGE=false
 CONCEPTNET_REINDEX_ON_RESET=true
 CONCEPTNET_STARTUP_FAIL_OPEN=true

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 data/atomspace/
 data/faiss/
 *.metta
+!data/conceptnet/conceptnet_background.metta
 docs/reference/
 *matrix*.json
 *_sanity.json

Dockerfile

@@ -26,7 +26,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PETTA_COMMIT=e1490899cefc67c128d5311ff4861f9997674957
-ENV PETTACHAINER_COMMIT=d21b93b5132a7fc8722f64d57b74fb7c3a8d1faa
+ENV PETTACHAINER_COMMIT=6b88df7c903705a38205709151cdd7549fd8d1b0
 
 RUN apt-get update && apt-get install -y \
     software-properties-common \

README.md

@@ -4,6 +4,12 @@ A REST API service for Probabilistic Logic Network (PLN) based retrieval-augment
 Ingests natural language text, converts it to PLN atoms via a pluggable semantic parser,
 stores facts in a PeTTaChainer atomspace, and answers questions via logical proof.
 
+Current service assumptions:
+
+- standalone self-hosted service
+- one shared AtomSpace / knowledge base
+- no multi-user isolation or tenant separation
+
 ## Architecture
 
 ```
@@ -57,12 +63,12 @@ curl -X POST http://localhost:8000/ingest \
   -d '{"texts": ["People who eat fish are smart.", "Kebede eats fish."]}'
 ```
 
-### POST /query
-Ask a question against the knowledge base.
+### POST /reason
+Run a reasoning request against the knowledge base.
 ```bash
-curl -X POST http://localhost:8000/query \
+curl -X POST http://localhost:8000/reason \
   -H "Content-Type: application/json" \
-  -d '{"question": "Is Kebede smart?"}'
+  -d '{"query": "Is Kebede smart?"}'
 ```
 
 ### DELETE /reset
@@ -84,6 +90,11 @@ curl -X DELETE http://localhost:8000/reset \
 curl http://localhost:8000/health
 ```
 
+### GET /ready
+```bash
+curl http://localhost:8000/ready
+```
+
 ## Switching parsers
 
 Set `PARSER` in `.env` for local runs. For Docker Compose, use `PLNRAG_PARSER`
@@ -123,26 +134,42 @@ it may try later fallback candidates produced by the parser.
 ConceptNet can be loaded as readonly background knowledge and indexed into the
 same Qdrant collection as normal ingested facts.
 
+The repository tracks generated ConceptNet runtime artifacts so ConceptNet can be
+enabled without committing the raw ConceptNet dump:
+
+- `data/conceptnet/conceptnet_background.metta`
+- `data/conceptnet/conceptnet_background.jsonl`
+- `data/conceptnet/conceptnet_manifest.json`
+
+The raw dump `data/conceptnet/conceptnet-assertions-5.7.0.csv.gz` is intentionally
+ignored because it is large. It is only needed when rebuilding the generated
+artifacts.
+
 ```bash
 CONCEPTNET_ENABLED=true
 CONCEPTNET_AUTOLOAD=true
 CONCEPTNET_ATOMSPACE_PATH=data/conceptnet/conceptnet_background.metta
 CONCEPTNET_VECTOR_PAYLOAD_PATH=data/conceptnet/conceptnet_background.jsonl
 CONCEPTNET_COVERAGE_PERCENT=100.0
 CONCEPTNET_SAMPLE_SEED=42
-CONCEPTNET_AUTO_REBUILD_ON_CHANGE=true
+CONCEPTNET_AUTO_REBUILD_ON_CHANGE=false
 ```
 
 Background points use the normal `nl` / `pln` payload schema with extra metadata
 such as `source=conceptnet` and `background=true`.
 
-To build the aligned background files from a raw ConceptNet dump:
+To rebuild the aligned background files from a raw ConceptNet dump, first place
+the dump at `data/conceptnet/conceptnet-assertions-5.7.0.csv.gz`, then run:
 
 ```bash
 cp "/path/to/conceptnet-assertions-5.7.0.csv.gz" data/conceptnet/
 python scripts/conceptnet/export_conceptnet.py
 ```
 
+Set `CONCEPTNET_AUTO_REBUILD_ON_CHANGE=true` only on machines that have the raw
+dump available and should regenerate artifacts automatically when the ConceptNet
+export settings change.
+
 Coverage is controlled at export time. For example:
 
 ```bash

api/main.py

@@ -1,31 +1,43 @@
+import logging
 import time
 from contextlib import asynccontextmanager
 from fastapi import FastAPI, HTTPException
 
 from api.models import (
     IngestRequest, IngestResponse,
-    QueryRequest, QueryResponse,
+    ReasonRequest, ReasonResponse,
     ResetRequest, ResetResponse,
-    HealthResponse,
+    HealthResponse, ReadyResponse,
 )
 from core.service import PLNRAGService
 from parsers import get_parser
 from config import get_settings
 
 _start_time = time.time()
 _service: PLNRAGService | None = None
+logger = logging.getLogger(__name__)
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     global _service
     cfg = get_settings()
-    print(f"[Startup] Loading parser: {cfg.parser}")
-    parser = get_parser()
-    _service = PLNRAGService(parser)
-    print("[Startup] Service ready.")
-    yield
-    print("[Shutdown] Cleaning up.")
+    logger.info("Loading parser: %s", cfg.parser)
+    try:
+        parser = get_parser()
+        _service = PLNRAGService(parser)
+        logger.info("Service ready.")
+        yield
+    except Exception:
+        logger.exception("Application startup failed.")
+        raise
+    finally:
+        logger.info("Cleaning up service resources.")
+        if _service is not None:
+            try:
+                _service.close()
+            except Exception:
+                logger.exception("Service cleanup failed.")
 
 
 app = FastAPI(
@@ -38,10 +50,27 @@ async def lifespan(app: FastAPI):
 
 def get_service() -> PLNRAGService:
     if _service is None:
-        raise HTTPException(status_code=503, detail="Service not ready")
+        raise HTTPException(
+            status_code=503,
+            detail={"code": "service_not_ready", "message": "Service not ready"},
+        )
     return _service
 
 
+def ensure_dependencies_ready(svc: PLNRAGService):
+    info = svc.ready()
+    if info["status"] == "unavailable":
+        logger.warning("Request rejected because dependencies are unavailable: %s", info["details"])
+        raise HTTPException(
+            status_code=503,
+            detail={
+                "code": "dependencies_unavailable",
+                "message": "Required dependencies are unavailable",
+                "details": info["details"],
+            },
+        )
+
+
 @app.post("/ingest", response_model=IngestResponse)
 async def ingest(req: IngestRequest):
     """
@@ -51,22 +80,24 @@ async def ingest(req: IngestRequest):
     Processing is sequential — each text sees all previous atoms.
     """
     svc = get_service()
+    ensure_dependencies_ready(svc)
     results = await svc.ingest_batch(req.texts)
     return IngestResponse(
         processed_count=len(results),
         results=results
     )
 
 
-@app.post("/query", response_model=QueryResponse)
-async def query(req: QueryRequest):
+@app.post("/reason", response_model=ReasonResponse)
+async def reason(req: ReasonRequest):
     """
-    Ask a question against the knowledge base.
-    The question is parsed into a PLN query, reasoned over via
+    Run a reasoning request against the knowledge base.
+    The query is parsed into a PLN query, reasoned over via
     PeTTaChainer, and the proof trace is translated to natural language.
     """
     svc = get_service()
-    return await svc.query(req.question)
+    ensure_dependencies_ready(svc)
+    return await svc.reason(req.query)
 
 
 @app.delete("/reset", response_model=ResetResponse)
@@ -84,7 +115,7 @@ async def reset(req: ResetRequest = ResetRequest()):
 
 @app.get("/health", response_model=HealthResponse)
 async def health():
-    """Service health check — returns component status and sizes."""
+    """Liveness check — returns process/component status and sizes."""
     svc = get_service()
     info = svc.health()
     return HealthResponse(
@@ -100,3 +131,11 @@ async def health():
         conceptnet_last_error=info["conceptnet_last_error"],
         uptime_seconds=round(time.time() - _start_time, 1),
     )
+
+
+@app.get("/ready", response_model=ReadyResponse)
+async def ready():
+    """Readiness check — returns dependency availability and degraded state."""
+    svc = get_service()
+    info = svc.ready()
+    return ReadyResponse(**info)

api/models.py

@@ -1,12 +1,21 @@
-from pydantic import BaseModel
-from typing import List, Optional, Literal
+from pydantic import BaseModel, Field, field_validator
+from typing import List, Optional, Literal, Any
 
 
 #  Ingest 
 
 class IngestRequest(BaseModel):
     texts: List[str]
 
+    @field_validator("texts")
+    @classmethod
+    def validate_texts(cls, value: List[str]) -> List[str]:
+        if not value:
+            raise ValueError("texts must contain at least one item")
+        if not any(str(item).strip() for item in value):
+            raise ValueError("texts must contain at least one non-empty item")
+        return value
+
 
 class IngestItemResult(BaseModel):
     text: str
@@ -28,20 +37,27 @@ class IngestResponse(BaseModel):
     results: List[IngestItemResult]
 
 
-#  Query 
+#  Reason 
+
+class ReasonRequest(BaseModel):
+    query: str
 
-class QueryRequest(BaseModel):
-    question: str
+    @field_validator("query")
+    @classmethod
+    def validate_query(cls, value: str) -> str:
+        if not str(value).strip():
+            raise ValueError("query must not be empty")
+        return value
 
 
-class QueryResponse(BaseModel):
-    question: str
+class ReasonResponse(BaseModel):
+    query: str
     pln_query: str
     original_query: str
     executed_query: str
     fallback_used: bool
-    query_status: Literal["well_aligned", "weakly_aligned", "malformed", "no_query"]
-    raw_proof: str
+    query_status: Literal["well_aligned", "weakly_aligned", "no_query"]
+    proof: str
     sources: List[str]       # NL sentences that contributed to the proof
     answer: str
 
@@ -84,3 +100,15 @@ class HealthResponse(BaseModel):
     conceptnet_vectors_expected: int
     conceptnet_last_error: str
     uptime_seconds: float
+
+
+class ReadyResponse(BaseModel):
+    status: Literal["ready", "degraded", "unavailable"]
+    parser: str
+    reasoner_ready: bool
+    qdrant_ready: bool
+    ollama_ready: bool
+    conceptnet_enabled: bool
+    conceptnet_status: str
+    conceptnet_last_error: str
+    details: dict[str, Any] = Field(default_factory=dict)

benchmark_parsers.py

@@ -280,8 +280,8 @@ def _run_parse(parser: object, text: str, context: list[str], is_query: bool):
     return parser.parse(text, context)
 
 
-def _proof_found(raw_proof: str) -> bool:
-    return bool(raw_proof and raw_proof != "[]")
+def _proof_found(proof: str) -> bool:
+    return bool(proof and proof != "[]")
 
 
 def _configure_case_environment(parser_name: str, case_name: str, run_id: str):
@@ -344,11 +344,11 @@ async def _benchmark_case(parser_name: str, case: dict, run_id: str) -> dict:
         ingest_seconds = time.perf_counter() - ingest_started
 
         query_started = time.perf_counter()
-        query_response = await service.query(case["question"])
+        query_response = await service.reason(case["question"])
         query_seconds = time.perf_counter() - query_started
 
         total_seconds = parser_init_seconds + parse_seconds + ingest_seconds + query_seconds
-        found = _proof_found(query_response.raw_proof)
+        found = _proof_found(query_response.proof)
         expected = case.get("expected_proof")
         correct = None if expected is None else (found == expected)
 
@@ -374,13 +374,13 @@ async def _benchmark_case(parser_name: str, case: dict, run_id: str) -> dict:
             "end_to_end": {
                 "ingest": _compact_ingest_results(ingest_results),
                 "query": {
-                    "question": query_response.question,
+                    "query": query_response.query,
                     "pln_query": query_response.pln_query,
                     "original_query": query_response.original_query,
                     "executed_query": query_response.executed_query,
                     "fallback_used": query_response.fallback_used,
                     "query_status": query_response.query_status,
-                    "raw_proof": query_response.raw_proof,
+                    "proof": query_response.proof,
                     "sources": query_response.sources,
                     "answer": query_response.answer,
                     "candidate_count": query_response.candidate_count,
@@ -433,11 +433,11 @@ async def _benchmark_case_with_service(
     ingest_seconds = time.perf_counter() - ingest_started
 
     query_started = time.perf_counter()
-    query_response = await service.query(case["question"])
+    query_response = await service.reason(case["question"])
     query_seconds = time.perf_counter() - query_started
 
     total_seconds = parse_seconds + ingest_seconds + query_seconds
-    found = _proof_found(query_response.raw_proof)
+    found = _proof_found(query_response.proof)
     expected = case.get("expected_proof")
     correct = None if expected is None else (found == expected)
     state_after = _knowledge_state(service)
@@ -466,13 +466,13 @@ async def _benchmark_case_with_service(
         "end_to_end": {
             "ingest": _compact_ingest_results(ingest_results),
             "query": {
-                "question": query_response.question,
+                "query": query_response.query,
                 "pln_query": query_response.pln_query,
                 "original_query": query_response.original_query,
                 "executed_query": query_response.executed_query,
                 "fallback_used": query_response.fallback_used,
                 "query_status": query_response.query_status,
-                "raw_proof": query_response.raw_proof,
+                "proof": query_response.proof,
                 "sources": query_response.sources,
                 "answer": query_response.answer,
                 "candidate_count": query_response.candidate_count,