Merge pull request #2117 from nightcityblade/fix/issue-2116

fix(docker): preserve single-url crawl failure details

Author: ntohidi

deploy/docker/api.py

@@ -92,6 +92,15 @@ def _attach_declarative_hooks(crawler, hooks_config: dict) -> dict:
 
 logger = logging.getLogger(__name__)
 
+
+def _raise_for_crawl_failure(result):
+    if not result.success:
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=result.error_message,
+        )
+
+
 # --- Helper to get memory ---
 def _get_memory_mb():
     try:
@@ -147,11 +156,7 @@ async def handle_llm_qa(
         enforce_egress(browser_cfg)
         crawler = await get_crawler(browser_cfg)
         result = await crawler.arun(url)
-        if not result.success:
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail=result.error_message
-            )
+        _raise_for_crawl_failure(result)
         content = result.markdown.fit_markdown or result.markdown.raw_markdown
 
         # Create prompt and get LLM response
@@ -179,6 +184,8 @@ async def handle_llm_qa(
         )
 
         return response.choices[0].message.content
+    except HTTPException:
+        raise
     except LLMProviderNotAllowed as e:
         raise HTTPException(status_code=400, detail=str(e))
     except Exception as e:
@@ -389,11 +396,7 @@ async def handle_markdown_request(
             )
         )
 
-        if not result.success:
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail=result.error_message
-            )
+        _raise_for_crawl_failure(result)
 
         return (result.markdown.raw_markdown
                if filter_type == FilterType.RAW
@@ -1023,4 +1026,4 @@ async def _runner():
     except HTTPException:
         await redis.delete(f"task:{task_id}")
         raise
-    return {"task_id": task_id}
+    return {"task_id": task_id}

deploy/docker/tests/test_api_crawl_failures.py

@@ -0,0 +1,58 @@
+from types import SimpleNamespace
+
+import pytest
+from fastapi import HTTPException
+
+from api import _raise_for_crawl_failure
+
+
+def test_crawl_failure_is_reported_as_bad_gateway():
+    result = SimpleNamespace(success=False, error_message="Blocked by anti-bot protection: challenge")
+
+    with pytest.raises(HTTPException) as raised:
+        _raise_for_crawl_failure(result)
+
+    assert raised.value.status_code == 502
+    assert raised.value.detail == result.error_message
+
+
+@pytest.mark.parametrize(
+    ("method", "path", "payload"),
+    [
+        ("post", "/md", {"url": "https://example.com", "f": "raw"}),
+        ("get", "/llm/example.com?q=summarize", None),
+    ],
+)
+def test_single_url_crawl_failure_reaches_client(
+    stock_client, server_module, monkeypatch, method, path, payload
+):
+    error_message = "Blocked by anti-bot protection: challenge"
+    failed_result = SimpleNamespace(success=False, error_message=error_message)
+
+    class FailedCrawler:
+        async def arun(self, *args, **kwargs):
+            return failed_result
+
+    async def get_failed_crawler(*args, **kwargs):
+        return FailedCrawler()
+
+    async def release_crawler(*args, **kwargs):
+        return None
+
+    import api
+    import crawler_pool
+    from auth import create_access_token
+
+    monkeypatch.setattr(api, "validate_url_destination", lambda url: None)
+    monkeypatch.setattr(crawler_pool, "get_crawler", get_failed_crawler)
+    monkeypatch.setattr(crawler_pool, "release_crawler", release_crawler)
+
+    token = create_access_token({"sub": "test@example.com"})
+    request = getattr(stock_client, method)
+    kwargs = {"json": payload} if payload is not None else {}
+    response = request(
+        path, headers={"Authorization": f"Bearer {token}"}, **kwargs
+    )
+
+    assert response.status_code == 502
+    assert response.json() == {"detail": error_message}