From 05de127942e0b5255fa079bc4fbffef5466da39b Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Fri, 7 Aug 2026 23:11:40 +0800 Subject: [PATCH 1/2] fix(docker): support PDF scraping by default --- deploy/docker/api.py | 15 +++++++++----- deploy/docker/requirements.txt | 1 + tests/test_issue_2127_docker_pdf.py | 31 +++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 tests/test_issue_2127_docker_pdf.py diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 1756b925f..55d9bdeb6 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -865,15 +865,20 @@ async def handle_stream_crawl_request( # mirroring handle_crawl_request. The streaming path previously skipped # this, leaving /crawl/stream (and /crawl with stream=true) unguarded. urls = _normalize_and_validate_seeds(urls) - browser_config = BrowserConfig.load(browser_config, provenance=Provenance.UNTRUSTED) + browser_config = BrowserConfig.load( + browser_config, provenance=Provenance.UNTRUSTED + ) # browser_config.verbose = True # Set to False or remove for production stress testing browser_config.verbose = False from egress_broker import enforce_egress + enforce_egress(browser_config) - crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED) + crawler_config = CrawlerRunConfig.load( + crawler_config, provenance=Provenance.UNTRUSTED + ) from governor import clamp_deep_crawl + clamp_deep_crawl(crawler_config) - crawler_config.scraping_strategy = LXMLWebScrapingStrategy() crawler_config.stream = True # Deep crawl streaming supports exactly one start URL @@ -941,7 +946,7 @@ async def handle_stream_crawl_request( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) ) - + async def handle_crawl_job( redis, background_tasks: BackgroundTasks, @@ -1023,4 +1028,4 @@ async def _runner(): except HTTPException: await redis.delete(f"task:{task_id}") raise - return {"task_id": task_id} \ No newline at end of file + return {"task_id": task_id} diff --git a/deploy/docker/requirements.txt b/deploy/docker/requirements.txt index 212fcf036..e06cef78d 100644 --- a/deploy/docker/requirements.txt +++ b/deploy/docker/requirements.txt @@ -14,3 +14,4 @@ PyJWT==2.10.1 mcp>=1.18.0 websockets>=15.0.1 httpx[http2]>=0.27.2 +pypdf diff --git a/tests/test_issue_2127_docker_pdf.py b/tests/test_issue_2127_docker_pdf.py new file mode 100644 index 000000000..3fe0e5694 --- /dev/null +++ b/tests/test_issue_2127_docker_pdf.py @@ -0,0 +1,31 @@ +import ast +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def test_default_docker_dependencies_include_pypdf(): + requirements = ( + (ROOT / "deploy" / "docker" / "requirements.txt").read_text().splitlines() + ) + + assert "pypdf" in requirements + + +def test_stream_handler_preserves_requested_scraping_strategy(): + tree = ast.parse((ROOT / "deploy" / "docker" / "api.py").read_text()) + handler = next( + node + for node in tree.body + if isinstance(node, ast.AsyncFunctionDef) + and node.name == "handle_stream_crawl_request" + ) + assigned_attributes = { + target.attr + for node in ast.walk(handler) + if isinstance(node, ast.Assign) + for target in node.targets + if isinstance(target, ast.Attribute) + } + + assert "scraping_strategy" not in assigned_attributes From 10130de3d72b2fd2c593e17b2e52aa248493679f Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Wed, 12 Aug 2026 23:10:53 +0800 Subject: [PATCH 2/2] test(docker): address PDF review feedback Signed-off-by: nightcityblade --- deploy/docker/requirements.txt | 2 +- tests/test_issue_2127_docker_pdf.py | 80 +++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 21 deletions(-) diff --git a/deploy/docker/requirements.txt b/deploy/docker/requirements.txt index e06cef78d..7ae06a543 100644 --- a/deploy/docker/requirements.txt +++ b/deploy/docker/requirements.txt @@ -14,4 +14,4 @@ PyJWT==2.10.1 mcp>=1.18.0 websockets>=15.0.1 httpx[http2]>=0.27.2 -pypdf +pypdf>=6.0.0 diff --git a/tests/test_issue_2127_docker_pdf.py b/tests/test_issue_2127_docker_pdf.py index 3fe0e5694..1fe187622 100644 --- a/tests/test_issue_2127_docker_pdf.py +++ b/tests/test_issue_2127_docker_pdf.py @@ -1,31 +1,71 @@ -import ast +import importlib from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest +from packaging.requirements import InvalidRequirement, Requirement + +from crawl4ai.processors.pdf import PDFContentScrapingStrategy ROOT = Path(__file__).resolve().parent.parent def test_default_docker_dependencies_include_pypdf(): - requirements = ( - (ROOT / "deploy" / "docker" / "requirements.txt").read_text().splitlines() - ) + lines = (ROOT / "deploy" / "docker" / "requirements.txt").read_text().splitlines() + names = set() + for line in lines: + line = line.strip() + if not line or line.startswith(("#", "-")): + continue + try: + names.add(Requirement(line).name) + except InvalidRequirement: + continue - assert "pypdf" in requirements + assert "pypdf" in names -def test_stream_handler_preserves_requested_scraping_strategy(): - tree = ast.parse((ROOT / "deploy" / "docker" / "api.py").read_text()) - handler = next( - node - for node in tree.body - if isinstance(node, ast.AsyncFunctionDef) - and node.name == "handle_stream_crawl_request" - ) - assigned_attributes = { - target.attr - for node in ast.walk(handler) - if isinstance(node, ast.Assign) - for target in node.targets - if isinstance(target, ast.Attribute) +@pytest.mark.asyncio +async def test_stream_handler_preserves_requested_scraping_strategy(monkeypatch): + docker_dir = ROOT / "deploy" / "docker" + monkeypatch.syspath_prepend(str(docker_dir)) + + api = importlib.import_module("api") + crawler_pool = importlib.import_module("crawler_pool") + egress_broker = importlib.import_module("egress_broker") + governor = importlib.import_module("governor") + + crawler = MagicMock() + crawler.arun_many = AsyncMock(return_value=MagicMock()) + monkeypatch.setattr(api, "_normalize_and_validate_seeds", lambda urls: urls) + monkeypatch.setattr(egress_broker, "enforce_egress", lambda _: None) + monkeypatch.setattr(governor, "clamp_deep_crawl", lambda _: None) + monkeypatch.setattr(crawler_pool, "get_crawler", AsyncMock(return_value=crawler)) + + crawler_config = { + "type": "CrawlerRunConfig", + "params": { + "cache_mode": "bypass", + "stream": False, + "scraping_strategy": { + "type": "PDFContentScrapingStrategy", + "params": {"extract_images": False, "batch_size": 8}, + }, + }, + } + config = { + "crawler": { + "memory_threshold_percent": 90, + "rate_limiter": {"base_delay": [0.1, 0.3]}, + } } - assert "scraping_strategy" not in assigned_attributes + await api.handle_stream_crawl_request( + urls=["https://example.com/document.pdf"], + browser_config={"type": "BrowserConfig", "params": {}}, + crawler_config=crawler_config, + config=config, + ) + + effective_config = crawler.arun_many.await_args.kwargs["config"] + assert isinstance(effective_config.scraping_strategy, PDFContentScrapingStrategy)