From 1e055ab15c936e0891f9a18e2ce5cd574875ed2d Mon Sep 17 00:00:00 2001 From: phernandez Date: Mon, 7 Sep 2026 23:29:37 -0500 Subject: [PATCH 01/10] feat(core): ingest docx, pptx, and csv into document sidecar notes Closes #1512. Documents other than PDF now get the same sidecar Markdown note a PDF gets (`report.docx.md` next to `report.docx`), through the existing parser-neutral document contract. - Add `ExtractedDocument` and a `DocumentExtractor` protocol; the new `RawDocumentRuntime` dispatches on the indexed media type. The PDF runtime and `build_raw_document_artifacts` keep their signatures and delegate, so Cloud's wiring is unchanged. - Extract the killable, byte-capped child-process mechanics from the pdf-inspector adapter into `bounded_process.py` and the rlimit helpers into `worker_limits.py`; both workers use them. - Add a markitdown worker that calls `DocxConverter` / `PptxConverter` directly. The sniffing `MarkItDown.convert()` loop is never used, so magika, URL converters, zip recursion, and the plain-text and ASCII-decode failures seen on real files are out of the picture. Picture links are dropped, keeping alt text only when descriptive. - Render CSV with the stdlib as a bounded preview (header, first N rows, row count) with strict UTF-8 decoding instead of a charset guess. - Add a local project-directory runtime (API-backed source resolver, checksum-as-generation reader, sidecar + run-note writer that refuses to overwrite hand-written or enriched notes) and `bm import document`. - Ship markitdown[docx,pptx] behind a `basic-memory[documents]` extra. Verified end to end in an isolated home: docx, pptx, csv, and an 87-page PDF each produced an indexed, searchable sidecar and a run note; a second run reported both as already current. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01VPRRyMREwCHSLnY3TCBr7s Signed-off-by: phernandez --- CHANGELOG.md | 11 + pyproject.toml | 4 + .../cli/commands/import_document.py | 107 ++++++ src/basic_memory/cli/main.py | 1 + .../document_ingestion/bounded_process.py | 166 +++++++++ .../document_ingestion/csv_extractor.py | 146 ++++++++ .../document_ingestion/local_runtime.py | 218 ++++++++++++ .../markitdown_extractor.py | 241 ++++++++++++++ .../document_ingestion/markitdown_worker.py | 134 ++++++++ .../document_ingestion/pdf_inspector.py | 147 ++------ .../pdf_inspector_worker.py | 30 +- .../document_ingestion/raw_document.py | 249 ++++++++++---- .../document_ingestion/worker_limits.py | 35 ++ tests/cli/test_import_document.py | 163 +++++++++ tests/document_ingestion/office_fixtures.py | 81 +++++ tests/document_ingestion/process_fakes.py | 80 +++++ .../document_ingestion/test_csv_extractor.py | 117 +++++++ .../test_document_extractors.py | 229 +++++++++++++ .../document_ingestion/test_local_runtime.py | 315 ++++++++++++++++++ .../test_markitdown_extractor.py | 257 ++++++++++++++ .../test_markitdown_worker.py | 161 +++++++++ .../document_ingestion/test_pdf_inspector.py | 78 +---- .../test_pdf_inspector_worker.py | 30 +- uv.lock | 251 +++++++++++++- 24 files changed, 2956 insertions(+), 295 deletions(-) create mode 100644 src/basic_memory/cli/commands/import_document.py create mode 100644 src/basic_memory/document_ingestion/bounded_process.py create mode 100644 src/basic_memory/document_ingestion/csv_extractor.py create mode 100644 src/basic_memory/document_ingestion/local_runtime.py create mode 100644 src/basic_memory/document_ingestion/markitdown_extractor.py create mode 100644 src/basic_memory/document_ingestion/markitdown_worker.py create mode 100644 src/basic_memory/document_ingestion/worker_limits.py create mode 100644 tests/cli/test_import_document.py create mode 100644 tests/document_ingestion/office_fixtures.py create mode 100644 tests/document_ingestion/process_fakes.py create mode 100644 tests/document_ingestion/test_csv_extractor.py create mode 100644 tests/document_ingestion/test_document_extractors.py create mode 100644 tests/document_ingestion/test_local_runtime.py create mode 100644 tests/document_ingestion/test_markitdown_extractor.py create mode 100644 tests/document_ingestion/test_markitdown_worker.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e90d16b11..239f084dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ ### Features +- **#1512**: Word, PowerPoint, and CSV files get the same sidecar Markdown note a + PDF gets. `bm import document ` indexes the project, extracts the file, + and writes `..md` next to it plus a run note under + `document-ingestion-runs/`, all through the existing parser-neutral document + contract. Office formats run through Microsoft's `markitdown` converters + (`basic-memory[documents]`) inside the same killable, byte-capped worker + process as pdf-inspector; CSV renders a bounded stdlib preview with strict + UTF-8. Re-running is a no-op while the source is unchanged, and the command + refuses to overwrite a hand-written or enriched note at the sidecar path. + PDF keeps pdf-inspector and works through the same command. + - **#610**: The manual's SYNOPSIS blocks are now generated from the tool registry. `just man-regen` renders the MCP call on every section-3 page from the schema clients actually receive (required parameters first, then defaults, in schema diff --git a/pyproject.toml b/pyproject.toml index e97ed17ca..24936db0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,9 @@ redis = [ pdf = [ "pdf-inspector>=0.2.6,<2", ] +documents = [ + "markitdown[docx,pptx]>=0.1.7,<0.2", +] [project.urls] Homepage = "https://github.com/basicmachines-co/basic-memory" @@ -149,6 +152,7 @@ dev = [ "libcst>=1.8.6", "pytest-timeout>=2.4.0", "pytest-split>=0.11.0", + "markitdown[docx,pptx]>=0.1.7,<0.2", ] [tool.hatch.version] diff --git a/src/basic_memory/cli/commands/import_document.py b/src/basic_memory/cli/commands/import_document.py new file mode 100644 index 000000000..ce6d8ca2d --- /dev/null +++ b/src/basic_memory/cli/commands/import_document.py @@ -0,0 +1,107 @@ +"""Import command: extract one document file in a project into a sidecar Markdown note.""" + +# PEP 563 lazy annotations keep the ingestion stack out of module import (#886). +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Annotated, Optional + +import typer +from loguru import logger +from rich.console import Console +from rich.panel import Panel + +from basic_memory.cli.app import import_app +from basic_memory.cli.commands.command_utils import run_with_cleanup +from basic_memory.config import ConfigManager + +if TYPE_CHECKING: + from basic_memory.document_ingestion.raw_document import RawDocumentWriteResult + +console = Console() + + +async def import_document(path: Path, project: str | None) -> tuple[str, RawDocumentWriteResult]: + """Index the source, extract it, and write its sidecar note into the project.""" + # Deferred: the ingestion stack and API client load only when the command runs (#886). + from basic_memory.document_ingestion.local_runtime import ( + ApiDocumentSourceEntityResolver, + LocalDocumentSourceReader, + LocalRawDocumentWriter, + default_document_extractors, + ) + from basic_memory.document_ingestion.raw_document import RawDocumentRuntime + from basic_memory.markdown import EntityParser, MarkdownProcessor + from basic_memory.mcp.async_client import get_client + from basic_memory.mcp.clients import KnowledgeClient, ProjectClient + from basic_memory.mcp.project_context import get_active_project + from basic_memory.services.file_service import FileService + + config_manager = ConfigManager() + project_name = project or config_manager.default_project + async with get_client(project_name=project_name) as client: + project_item = await get_active_project(client, project_name, None) + project_home = Path(project_item.path).expanduser().resolve() + source = path.expanduser().resolve() + if not source.is_file(): + raise typer.BadParameter(f"File not found: {path}") + if not source.is_relative_to(project_home): + raise typer.BadParameter( + f"{path} is not inside project {project_item.name!r} ({project_home})" + ) + relative_path = source.relative_to(project_home).as_posix() + + # The source must be an indexed file entity before it can own a document + # note, and a fresh drop may not have been seen by the watcher yet. + await ProjectClient(client).index( + project_item.external_id, force_full=False, run_in_background=False + ) + + knowledge = KnowledgeClient(client, project_item.external_id) + app_config = config_manager.config + markdown_processor = MarkdownProcessor(EntityParser(project_home), app_config=app_config) + file_service = FileService(project_home, markdown_processor, app_config=app_config) + runtime = RawDocumentRuntime( + source_resolver=ApiDocumentSourceEntityResolver(knowledge), + source_reader=LocalDocumentSourceReader(project_home), + extractors=default_document_extractors(), + writer=LocalRawDocumentWriter(file_service, knowledge), + ) + result = await runtime.ingest(file_path=relative_path, observed_etag=None) + return project_item.name, result + + +@import_app.command( + name="document", + help="Extract a PDF, docx, pptx, or csv file in a project into a sidecar Markdown note.", +) +def document( + path: Annotated[Path, typer.Argument(help="Path to a file inside the project directory")], + project: Annotated[ + Optional[str], + typer.Option("--project", "-p", help="Project name (defaults to the default project)"), + ] = None, +) -> None: + """Write ``.md`` next to the source and a run note under document-ingestion-runs/.""" + try: + project_name, result = run_with_cleanup(import_document(path, project)) + except typer.BadParameter as error: + typer.echo(f"Error: {error}", err=True) + raise typer.Exit(1) + except Exception as error: + logger.error("Document import failed") + typer.echo(f"Error during import: {error}", err=True) + raise typer.Exit(1) + + document_state = "created" if result.document_created else "already current" + run_state = "created" if result.run_created else "already recorded" + console.print( + Panel( + f"[green]Document import complete![/green]\n\n" + f"Project: {project_name}\n" + f"Document note: {result.document_file_path} ({document_state})\n" + f"Run note: {result.run_file_path} ({run_state})\n" + f"Run id: {result.run_id}", + expand=False, + ) + ) diff --git a/src/basic_memory/cli/main.py b/src/basic_memory/cli/main.py index d7043ba2c..afaa3d4b5 100644 --- a/src/basic_memory/cli/main.py +++ b/src/basic_memory/cli/main.py @@ -24,6 +24,7 @@ def _version_only_invocation(argv: list[str]) -> bool: import_chatgpt, import_claude_conversations, import_claude_projects, + import_document, import_memory_json, inspect, install, diff --git a/src/basic_memory/document_ingestion/bounded_process.py b/src/basic_memory/document_ingestion/bounded_process.py new file mode 100644 index 000000000..c92018092 --- /dev/null +++ b/src/basic_memory/document_ingestion/bounded_process.py @@ -0,0 +1,166 @@ +"""Run one extractor over untrusted bytes in a killable, byte-capped child process. + +Every native or third-party document parser Basic Memory runs is synchronous and +parses attacker-controlled input. The parent therefore never calls a parser +in-process: it feeds the bytes to ``python -m `` on stdin, drains both +pipes with a hard ceiling, and kills the child at the deadline. The worker +applies POSIX rlimits to itself before reading any input. + +This module owns only the process mechanics. Each extractor owns its argv, +its output contract, and the error types its callers see. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Sequence + + +class BoundedProcessError(RuntimeError): + """Base failure for one bounded child process run.""" + + +class BoundedProcessSpawnError(BoundedProcessError): + """The running event loop cannot spawn subprocesses.""" + + +class BoundedProcessTimeoutError(BoundedProcessError): + """The child was killed after exceeding its wall-clock deadline.""" + + +class BoundedProcessOutputLimitError(BoundedProcessError): + """A child pipe crossed the output byte ceiling while streaming.""" + + def __init__(self, stream_name: str) -> None: + super().__init__(f"child process exceeded the output byte limit on {stream_name}") + self.stream_name = stream_name + + +class BoundedProcessExitError(BoundedProcessError): + """The child exited with a non-zero status.""" + + def __init__(self, returncode: int, detail: str) -> None: + super().__init__(f"child process failed with exit code {returncode}: {detail}") + self.returncode = returncode + self.detail = detail + + +async def run_bounded_process( + argv: Sequence[str], + stdin_bytes: bytes, + *, + timeout_seconds: float, + max_output_bytes: int, +) -> bytes: + """Run ``argv`` with ``stdin_bytes`` on stdin and return its capped stdout. + + The caller bounds concurrency; this function bounds one run. It raises a + :class:`BoundedProcessError` subclass for every failure mode so the caller + can name the failure in its own domain terms. + """ + try: + process = await asyncio.create_subprocess_exec( + *argv, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except NotImplementedError as error: + # Trigger: the running loop cannot spawn subprocesses. Basic Memory + # installs the selector loop on Windows for aiosqlite (see db.py), + # and that loop has no subprocess transport. + # Outcome: a named error instead of a bare NotImplementedError. + raise BoundedProcessSpawnError( + "child process needs an event loop with subprocess support; " + "Basic Memory runs the selector loop on Windows" + ) from error + try: + async with asyncio.timeout(timeout_seconds): + stdout, stderr = await _communicate_capped(process, stdin_bytes, cap=max_output_bytes) + except asyncio.CancelledError: + await kill_and_wait(process) + raise + except TimeoutError as error: + await kill_and_wait(process) + raise BoundedProcessTimeoutError( + "child process exceeded the configured deadline" + ) from error + except BoundedProcessOutputLimitError: + # A pipe crossed its ceiling while the child may still be writing. + await kill_and_wait(process) + raise + + if process.returncode != 0: + detail = stderr.decode("utf-8", errors="replace")[-2000:].strip() + raise BoundedProcessExitError( + process.returncode if process.returncode is not None else -1, + detail or "no error detail", + ) + return stdout + + +async def kill_and_wait(process: asyncio.subprocess.Process) -> None: + """Terminate and reap one child before releasing worker capacity.""" + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + # The child can exit between the returncode check and the signal. + # Waiting still reaps it and preserves the caller's original outcome. + pass + await process.wait() + + +# Read pipes in modest chunks so the ceiling applies as bytes arrive rather than +# after an entire stream has already been buffered. +_PIPE_CHUNK_BYTES = 64 * 1024 + + +async def _communicate_capped( + process: asyncio.subprocess.Process, + stdin_bytes: bytes, + *, + cap: int, +) -> tuple[bytes, bytes]: + """Feed stdin and drain both pipes with a hard byte ceiling on each. + + ``Process.communicate()`` buffers stdout and stderr without limit before the + caller can look at their size, so a hostile document that makes the worker + emit a huge field, or a runaway error stream, would land in parent memory. + This variant raises as soon as either pipe crosses ``cap``; the caller kills + the child. + """ + stdin, stdout, stderr = process.stdin, process.stdout, process.stderr + if stdin is None or stdout is None or stderr is None: # pragma: no cover - PIPE at spawn + raise RuntimeError("child process was started without pipes") + try: + async with asyncio.TaskGroup() as group: + stdout_task = group.create_task(_read_capped(stdout, cap=cap, stream_name="stdout")) + stderr_task = group.create_task(_read_capped(stderr, cap=cap, stream_name="stderr")) + group.create_task(_feed_stdin(stdin, stdin_bytes)) + except* BoundedProcessOutputLimitError as overflow: + raise overflow.exceptions[0] from None + await process.wait() + return stdout_task.result(), stderr_task.result() + + +async def _read_capped(stream: asyncio.StreamReader, *, cap: int, stream_name: str) -> bytes: + buffer = bytearray() + while chunk := await stream.read(_PIPE_CHUNK_BYTES): + buffer.extend(chunk) + if len(buffer) > cap: + raise BoundedProcessOutputLimitError(stream_name) + return bytes(buffer) + + +async def _feed_stdin(stdin: asyncio.StreamWriter, stdin_bytes: bytes) -> None: + try: + stdin.write(stdin_bytes) + await stdin.drain() + except (BrokenPipeError, ConnectionResetError): + # The child exited before consuming its input. Its exit status and + # stderr carry the failure; ``Process.communicate()`` ignores the same + # pair for the same reason. + pass + finally: + stdin.close() diff --git a/src/basic_memory/document_ingestion/csv_extractor.py b/src/basic_memory/document_ingestion/csv_extractor.py new file mode 100644 index 000000000..1b0c7ae0d --- /dev/null +++ b/src/basic_memory/document_ingestion/csv_extractor.py @@ -0,0 +1,146 @@ +"""In-process CSV to Markdown preview extraction. + +CSV needs no third-party parser: the stdlib reader is pure Python over bytes the +runtime has already bounded, so it runs in a worker thread rather than a child +process. Decoding is strict UTF-8 (BOM tolerated). A guessed charset is how +markitdown produced mojibake on a valid UTF-8 export, so a non-UTF-8 file fails +the extraction instead of being silently mangled. + +The Markdown is a preview, not a dump: header, the first ``max_rows`` rows, and +the total row count. A 7,000-row export rendered in full is search noise, not +knowledge; the source file stays in the project for anyone who needs every row. +""" + +from __future__ import annotations + +import asyncio +import csv +import io +import time +from collections.abc import Sequence +from dataclasses import dataclass, field +from datetime import UTC, datetime + +from pydantic import BaseModel, ConfigDict, Field + +from basic_memory import __version__ as basic_memory_version +from basic_memory.document_ingestion.raw_document import ( + ExtractedDocument, + extraction_options_checksum, +) +from basic_memory.schemas.document import DocumentExtractionStatus, DocumentExtractionV1 + +CSV_ENGINE = "basic-memory/csv" +CSV_MEDIA_TYPE = "text/csv" +CSV_RAW_PIPELINE_VERSION = "csv-raw-v1" +CSV_EXTRACTION_PROFILE = "csv-preview-v1" + + +class CsvLimits(BaseModel): + """Bounds on one CSV preview extraction.""" + + model_config = ConfigDict(frozen=True, strict=True) + + max_source_bytes: int = Field(default=25 * 1024 * 1024, gt=0) + max_rows: int = Field(default=200, gt=0) + max_output_bytes: int = Field(default=5 * 1024 * 1024, gt=0) + + +class CsvExtractionError(RuntimeError): + """A CSV preview extraction failed.""" + + +class CsvSourceTooLargeError(CsvExtractionError): + """Raised before parsing when the source exceeds the configured byte limit.""" + + +class CsvDecodeError(CsvExtractionError): + """Raised when the source is not UTF-8.""" + + +@dataclass(frozen=True, slots=True) +class CsvPreview: + """Rendered preview plus the counts that describe what it left out.""" + + markdown: str + row_count: int + shown_rows: int + + +@dataclass(frozen=True, slots=True) +class CsvExtractor: + """Render a bounded Markdown preview of one CSV file.""" + + limits: CsvLimits = field(default_factory=CsvLimits) + + async def extract(self, content: bytes, *, file_name: str) -> ExtractedDocument: + if len(content) > self.limits.max_source_bytes: + raise CsvSourceTooLargeError("CSV source exceeds the configured extraction byte limit") + started = time.perf_counter() + preview = await asyncio.to_thread( + render_csv_preview, content, max_rows=self.limits.max_rows + ) + if len(preview.markdown.encode("utf-8")) > self.limits.max_output_bytes: + raise CsvExtractionError("CSV preview exceeds the configured output byte limit") + extraction = DocumentExtractionV1( + engine=CSV_ENGINE, + engine_version=basic_memory_version, + profile=CSV_EXTRACTION_PROFILE, + options_hash=extraction_options_checksum(self.limits), + classification="csv", + status=DocumentExtractionStatus.complete, + extracted_at=datetime.now(tz=UTC), + duration_ms=int((time.perf_counter() - started) * 1000), + page_count=0, + extracted_page_count=0, + requires_ocr=False, + ocr_page_count=0, + has_tables=preview.shown_rows > 0, + ) + return ExtractedDocument( + extraction=extraction, + markdown=preview.markdown, + kind="csv", + pipeline_version=CSV_RAW_PIPELINE_VERSION, + ) + + +def render_csv_preview(content: bytes, *, max_rows: int) -> CsvPreview: + """Render the header, the first ``max_rows`` rows, and the total row count.""" + try: + text = content.decode("utf-8-sig") + except UnicodeDecodeError as error: + raise CsvDecodeError("CSV source is not UTF-8 encoded") from error + + reader = csv.reader(io.StringIO(text, newline="")) + header = next(reader, None) + if header is None: + return CsvPreview(markdown="_Empty CSV file._\n", row_count=0, shown_rows=0) + + shown: list[Sequence[str]] = [] + row_count = 0 + for row in reader: + row_count += 1 + if len(shown) < max_rows: + shown.append(row) + + width = len(header) + lines = [ + _table_row(header, width), + "| " + " | ".join("---" for _ in range(width)) + " |", + *(_table_row(row, width) for row in shown), + "", + ] + if row_count > len(shown): + lines.append(f"_Showing {len(shown)} of {row_count} rows._") + else: + lines.append(f"_{row_count} rows._") + return CsvPreview(markdown="\n".join(lines) + "\n", row_count=row_count, shown_rows=len(shown)) + + +def _table_row(cells: Sequence[str], width: int) -> str: + # Ragged rows are common in hand-edited exports: pad short rows and drop + # cells past the header width so every line stays a valid table row. + padded = [*cells[:width], *([""] * (width - len(cells)))] + escaped = (cell.replace("\n", " ").replace("|", "\\|") for cell in padded) + return "| " + " | ".join(escaped) + " |" diff --git a/src/basic_memory/document_ingestion/local_runtime.py b/src/basic_memory/document_ingestion/local_runtime.py new file mode 100644 index 000000000..a1bc22396 --- /dev/null +++ b/src/basic_memory/document_ingestion/local_runtime.py @@ -0,0 +1,218 @@ +"""Local project-directory runtime for raw document ingestion. + +Cloud reads sources from object storage and accepts notes through a queue. The +local runtime reads the file under the project directory, writes the sidecar +note back into it, and asks the API to index the new Markdown so the knowledge +graph sees it without waiting for the watcher. The file bytes' SHA-256 stands in +for a storage ETag: a local file has no version id, so the checksum is the only +generation marker available. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import sys +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Protocol +from uuid import UUID + +from basic_memory.document_ingestion.csv_extractor import CSV_MEDIA_TYPE, CsvExtractor +from basic_memory.file_utils import ParseError +from basic_memory.document_ingestion.markitdown_extractor import markitdown_extractors +from basic_memory.document_ingestion.pdf_inspector import PdfInspector +from basic_memory.document_ingestion.raw_document import ( + PDF_MEDIA_TYPE, + DocumentExtractor, + DocumentSourceChangedError, + DocumentSourceEntity, + DocumentSourceSnapshot, + PdfDocumentExtractor, + RawDocumentArtifacts, + RawDocumentWriteResult, + build_raw_ingestion_run_markdown, + canonical_db_checksum, + raw_document_matches, +) +from basic_memory.schemas.document import DocumentIngestionStage, parse_document_markdown +from basic_memory.schemas.v2.entity import EntityResolveResponse, EntityResponseV2 +from basic_memory.services.file_service import FileService + + +class DocumentSourceResolutionError(RuntimeError): + """The requested path did not resolve to its own indexed file entity.""" + + +class DocumentSidecarConflictError(RuntimeError): + """The sidecar path holds content this runtime must not overwrite.""" + + +class DocumentKnowledgeApi(Protocol): + """The three knowledge-API calls the local runtime needs (satisfied by KnowledgeClient).""" + + async def resolve_entity_response( + self, identifier: str, *, strict: bool = False + ) -> EntityResolveResponse: ... + + async def get_entity(self, entity_id: str) -> EntityResponseV2: ... + + async def index_file(self, file_path: str) -> object: + """Index one Markdown file; only the side effect matters here.""" + + +def default_document_extractors( + *, python_executable: str = sys.executable +) -> dict[str, DocumentExtractor]: + """Every extractor the local runtime can dispatch to, keyed by media type.""" + extractors: dict[str, DocumentExtractor] = { + PDF_MEDIA_TYPE: PdfDocumentExtractor(PdfInspector(python_executable=python_executable)), + CSV_MEDIA_TYPE: CsvExtractor(), + } + extractors.update(markitdown_extractors(python_executable=python_executable)) + return extractors + + +def sha256_checksum(content: bytes) -> str: + return f"sha256:{hashlib.sha256(content).hexdigest()}" + + +@dataclass(frozen=True, slots=True) +class ApiDocumentSourceEntityResolver: + """Resolve a project-relative path to its indexed file entity through the API.""" + + knowledge: DocumentKnowledgeApi + + async def resolve(self, file_path: str) -> DocumentSourceEntity: + resolved = await self.knowledge.resolve_entity_response(file_path, strict=True) + # Strict resolution still tries ``.md`` when the path itself is not + # indexed, so an existing sidecar would be mistaken for its own source. + if resolved.file_path != file_path: + raise DocumentSourceResolutionError( + f"{file_path!r} resolved to {resolved.file_path!r}; " + "the source file itself is not indexed" + ) + entity = await self.knowledge.get_entity(resolved.external_id) + return DocumentSourceEntity( + entity_id=entity.id, + external_id=UUID(entity.external_id), + file_path=entity.file_path, + media_type=entity.content_type, + ) + + +@dataclass(frozen=True, slots=True) +class LocalDocumentSourceReader: + """Read source bytes from the project directory; the checksum is the generation.""" + + project_home: Path + + async def read( + self, + entity: DocumentSourceEntity, + *, + observed_etag: str | None, + ) -> DocumentSourceSnapshot: + content = await asyncio.to_thread((self.project_home / entity.file_path).read_bytes) + checksum = sha256_checksum(content) + if observed_etag is not None and observed_etag != checksum: + raise DocumentSourceChangedError( + f"{entity.file_path} changed since it was observed ({observed_etag} -> {checksum})" + ) + return DocumentSourceSnapshot( + entity=entity, + content=content, + checksum=checksum, + size_bytes=len(content), + storage_etag=checksum, + ) + + async def require_current(self, snapshot: DocumentSourceSnapshot) -> None: + current = await asyncio.to_thread( + (self.project_home / snapshot.entity.file_path).read_bytes + ) + if sha256_checksum(current) != snapshot.checksum: + raise DocumentSourceChangedError( + f"{snapshot.entity.file_path} changed on disk during extraction" + ) + + +@dataclass(frozen=True, slots=True) +class LocalRawDocumentWriter: + """Write the sidecar and run notes into the project directory and index them.""" + + file_service: FileService + knowledge: DocumentKnowledgeApi + + async def write(self, artifacts: RawDocumentArtifacts) -> RawDocumentWriteResult: + document_checksum, document_created = await accept_document_note( + self.file_service, self.knowledge, artifacts + ) + run_created = await accept_run_note( + self.file_service, self.knowledge, artifacts, raw_checksum=document_checksum + ) + return RawDocumentWriteResult( + document_external_id=artifacts.document_external_id, + document_file_path=artifacts.document_file_path, + document_db_checksum=document_checksum, + run_id=artifacts.run_id, + run_file_path=artifacts.run_file_path, + document_created=document_created, + run_created=run_created, + ) + + +async def accept_document_note( + file_service: FileService, + knowledge: DocumentKnowledgeApi, + artifacts: RawDocumentArtifacts, +) -> tuple[str, bool]: + """Write the sidecar note unless an identical raw projection already exists. + + Returns the canonical checksum of the note on disk and whether it was written. + """ + path = artifacts.document_file_path + if await file_service.exists(path): + existing_markdown = await file_service.read_file_content(path) + try: + existing = parse_document_markdown(existing_markdown) + except (ParseError, ValueError) as error: + # No frontmatter (ParseError) or frontmatter that fails the contract + # (pydantic's ValidationError is a ValueError): the file at the sidecar + # path is a hand-written note, not a generated projection. + raise DocumentSidecarConflictError( + f"{path} exists and is not a generated document note; " + f"move it aside to ingest {artifacts.source.file_path}" + ) from error + if existing.frontmatter.ingestion.stage is not DocumentIngestionStage.raw: + raise DocumentSidecarConflictError( + f"{path} has been enriched past the raw stage; refusing to overwrite it" + ) + if raw_document_matches(existing, artifacts): + checksum = await file_service.compute_checksum(path) + return canonical_db_checksum(checksum), False + # Same source path, different bytes or engine: the raw projection is + # derived state and is rebuilt from the new run. + checksum = await file_service.write_file(path, artifacts.document_markdown) + await knowledge.index_file(path) + return canonical_db_checksum(checksum), True + + +async def accept_run_note( + file_service: FileService, + knowledge: DocumentKnowledgeApi, + artifacts: RawDocumentArtifacts, + *, + raw_checksum: str, +) -> bool: + """Write the run note for this deterministic run id unless it already exists.""" + path = artifacts.run_file_path + if await file_service.exists(path): + return False + markdown = build_raw_ingestion_run_markdown( + artifacts, raw_checksum=raw_checksum, raw_created_at=datetime.now(tz=UTC) + ) + await file_service.write_file(path, markdown) + await knowledge.index_file(path) + return True diff --git a/src/basic_memory/document_ingestion/markitdown_extractor.py b/src/basic_memory/document_ingestion/markitdown_extractor.py new file mode 100644 index 000000000..d968ada02 --- /dev/null +++ b/src/basic_memory/document_ingestion/markitdown_extractor.py @@ -0,0 +1,241 @@ +"""Bounded subprocess adapter for Microsoft's markitdown Office converters. + +markitdown is a synchronous, in-process library with no resource limits that +parses attacker-controlled Office archives (zip plus XML through lxml, images +through Pillow). Every conversion therefore runs in a killable child process +with explicit source, output, memory, CPU, wall-clock, and concurrency +ceilings, the same shape as the pdf-inspector adapter. + +The worker calls one specific converter chosen from the indexed media type. It +never uses markitdown's content-sniffing ``MarkItDown.convert()`` loop, so +magika, the URL converters, zip recursion, and the plain-text fallthrough are +never in play. The ``markitdown`` package is imported only by the worker +module, so ``basic-memory[documents]`` stays optional for callers that merely +reference the contracts. +""" + +from __future__ import annotations + +import asyncio +import importlib.util +import sys +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import UTC, datetime +from enum import StrEnum + +from pydantic import BaseModel, ConfigDict, Field + +from basic_memory.document_ingestion.bounded_process import ( + BoundedProcessExitError, + BoundedProcessOutputLimitError, + BoundedProcessSpawnError, + BoundedProcessTimeoutError, + run_bounded_process, +) +from basic_memory.document_ingestion.raw_document import ( + DocumentExtractor, + ExtractedDocument, + extraction_options_checksum, +) +from basic_memory.schemas.document import DocumentExtractionStatus, DocumentExtractionV1 + +MARKITDOWN_ENGINE = "microsoft/markitdown" +MARKITDOWN_WORKER_MODULE = "basic_memory.document_ingestion.markitdown_worker" +MARKITDOWN_RAW_PIPELINE_VERSION = "markitdown-raw-v1" + +DOCX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" +PPTX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.presentationml.presentation" + + +class MarkitdownFormat(StrEnum): + """Office formats routed to one specific markitdown converter.""" + + docx = "docx" + pptx = "pptx" + + +# Dispatch key is the exact media type the indexer records for the source entity. +MARKITDOWN_FORMATS_BY_MEDIA_TYPE: Mapping[str, MarkitdownFormat] = { + DOCX_MEDIA_TYPE: MarkitdownFormat.docx, + PPTX_MEDIA_TYPE: MarkitdownFormat.pptx, +} +MEDIA_TYPES_BY_MARKITDOWN_FORMAT: Mapping[MarkitdownFormat, str] = { + fmt: media_type for media_type, fmt in MARKITDOWN_FORMATS_BY_MEDIA_TYPE.items() +} + + +class MarkitdownLimits(BaseModel): + """Resource limits enforced around one markitdown conversion.""" + + model_config = ConfigDict(frozen=True, strict=True) + + max_source_bytes: int = Field(default=25 * 1024 * 1024, gt=0) + max_output_bytes: int = Field(default=5 * 1024 * 1024, gt=0) + # lxml, Pillow, and python-pptx need more address space than the Rust PDF + # parser; 512 MiB made valid decks fail before conversion began. + max_memory_bytes: int = Field(default=1024 * 1024 * 1024, gt=0) + timeout_seconds: float = Field(default=60.0, gt=0) + cpu_seconds: int = Field(default=50, gt=0) + max_concurrency: int = Field(default=1, gt=0) + + +class MarkitdownOutput(BaseModel): + """Validated, serialization-safe output from the markitdown worker process.""" + + model_config = ConfigDict(frozen=True, strict=True) + + engine: str = MARKITDOWN_ENGINE + engine_version: str + format: MarkitdownFormat + markdown: str + processing_time_ms: int = Field(ge=0) + # Slides are the pptx page unit; docx has no page model, so it reports None. + slide_count: int | None = Field(default=None, ge=0) + + +class MarkitdownExtractionError(RuntimeError): + """A bounded markitdown conversion failed.""" + + +class MarkitdownSourceTooLargeError(MarkitdownExtractionError): + """Raised before spawning when the source exceeds the configured byte limit.""" + + +class MarkitdownNotInstalledError(MarkitdownExtractionError): + """Raised before spawning when the optional ``documents`` extra is missing.""" + + +@dataclass(slots=True) +class MarkitdownExtractor: + """Run one markitdown converter in killable, concurrency-bounded subprocesses.""" + + format: MarkitdownFormat + limits: MarkitdownLimits = field(default_factory=MarkitdownLimits) + python_executable: str = sys.executable + _semaphore: asyncio.Semaphore = field(init=False, repr=False) + + def __post_init__(self) -> None: + self._semaphore = asyncio.Semaphore(self.limits.max_concurrency) + + async def extract(self, content: bytes, *, file_name: str) -> ExtractedDocument: + """Convert one Office document without blocking the caller's event loop.""" + if len(content) > self.limits.max_source_bytes: + raise MarkitdownSourceTooLargeError( + "Office document source exceeds the configured extraction byte limit" + ) + # Fail here with an install hint instead of a child-process ImportError + # traceback: the worker is the only module that imports markitdown. + if importlib.util.find_spec("markitdown") is None: + raise MarkitdownNotInstalledError( + "Office document extraction needs the markitdown package; " + "install basic-memory[documents]" + ) + + argv = ( + self.python_executable, + "-m", + MARKITDOWN_WORKER_MODULE, + "--format", + self.format.value, + "--file-name", + file_name, + "--max-output-bytes", + str(self.limits.max_output_bytes), + "--max-memory-bytes", + str(self.limits.max_memory_bytes), + "--cpu-seconds", + str(self.limits.cpu_seconds), + ) + async with self._semaphore: + try: + stdout = await run_bounded_process( + argv, + content, + timeout_seconds=self.limits.timeout_seconds, + max_output_bytes=self.limits.max_output_bytes, + ) + except BoundedProcessSpawnError as error: + raise MarkitdownExtractionError( + "Office document extraction needs an event loop with subprocess support; " + "Basic Memory runs the selector loop on Windows" + ) from error + except BoundedProcessTimeoutError as error: + raise MarkitdownExtractionError( + "Office document extraction exceeded the configured deadline" + ) from error + except BoundedProcessOutputLimitError as error: + raise MarkitdownExtractionError( + "Office document extraction exceeded the configured output byte limit " + f"on {error.stream_name}" + ) from error + except BoundedProcessExitError as error: + raise MarkitdownExtractionError( + f"Office document extraction failed with exit code {error.returncode}: " + f"{error.detail}" + ) from error + + try: + output = MarkitdownOutput.model_validate_json(stdout, strict=True) + except ValueError as error: + raise MarkitdownExtractionError( + "Office document extraction returned an invalid result" + ) from error + if output.format is not self.format: + raise MarkitdownExtractionError( + f"Office document extraction converted {output.format.value}, " + f"expected {self.format.value}" + ) + return markitdown_extracted_document( + output, limits=self.limits, extracted_at=datetime.now(tz=UTC) + ) + + +def markitdown_extracted_document( + output: MarkitdownOutput, + *, + limits: MarkitdownLimits, + extracted_at: datetime, +) -> ExtractedDocument: + """Map worker output into the parser-neutral extraction contract.""" + # markitdown emits one marker per slide, so slides fill the page diagnostics + # for pptx. docx reports zero pages, which the contract reads as "not + # paginated" for a complete extraction. + page_count = output.slide_count or 0 + extraction = DocumentExtractionV1( + engine=output.engine, + engine_version=output.engine_version, + profile=f"markitdown-{output.format.value}-v1", + options_hash=extraction_options_checksum(limits), + classification=output.format.value, + status=DocumentExtractionStatus.complete, + extracted_at=extracted_at, + duration_ms=output.processing_time_ms, + page_count=page_count, + extracted_page_count=page_count, + requires_ocr=False, + ocr_page_count=0, + pages_needing_ocr=(), + has_tables=any(line.startswith("|") for line in output.markdown.splitlines()), + ) + return ExtractedDocument( + extraction=extraction, + markdown=output.markdown, + kind=output.format.value, + pipeline_version=MARKITDOWN_RAW_PIPELINE_VERSION, + ) + + +def markitdown_extractors( + limits: MarkitdownLimits | None = None, + *, + python_executable: str = sys.executable, +) -> dict[str, DocumentExtractor]: + """Return one bounded extractor per supported Office media type.""" + shared_limits = limits or MarkitdownLimits() + return { + media_type: MarkitdownExtractor( + format=fmt, limits=shared_limits, python_executable=python_executable + ) + for media_type, fmt in MARKITDOWN_FORMATS_BY_MEDIA_TYPE.items() + } diff --git a/src/basic_memory/document_ingestion/markitdown_worker.py b/src/basic_memory/document_ingestion/markitdown_worker.py new file mode 100644 index 000000000..2daf1739a --- /dev/null +++ b/src/basic_memory/document_ingestion/markitdown_worker.py @@ -0,0 +1,134 @@ +"""One-shot markitdown conversion process used by the async adapter. + +Runs as ``python -m basic_memory.document_ingestion.markitdown_worker`` with the +Office file bytes on stdin and one validated JSON ``MarkitdownOutput`` on stdout. +Resource limits are applied before any source bytes are read so a malformed +archive cannot exhaust the parent process. + +The worker calls the converter for the requested format directly. That skips +markitdown's content sniffing and its try-every-converter loop, which is where +the plain-text fallthrough and the ASCII-decode crash live. +""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import io +import re +import sys +import time +from typing import assert_never + +from markitdown import DocumentConverter, StreamInfo +from markitdown.converters import DocxConverter, PptxConverter + +from basic_memory.document_ingestion.markitdown_extractor import ( + MEDIA_TYPES_BY_MARKITDOWN_FORMAT, + MarkitdownFormat, + MarkitdownOutput, +) +from basic_memory.document_ingestion.worker_limits import apply_cpu_limit, apply_memory_limit + +SLIDE_MARKER = "" in extracted.markdown + extraction = extracted.extraction + assert extraction.engine == MARKITDOWN_ENGINE + assert extraction.engine_version == importlib.metadata.version("markitdown") + assert extraction.profile == "markitdown-pptx-v1" + assert extraction.classification == "pptx" + assert extraction.status is DocumentExtractionStatus.complete + assert extraction.page_count == 2 + assert extraction.extracted_page_count == 2 + assert extraction.has_tables is True + + +@pytest.mark.asyncio +async def test_markitdown_extractor_passes_format_and_limits_to_the_worker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + process = FakeProcess(stdout=valid_output_json()) + install_fake_process(monkeypatch, process) + extractor = MarkitdownExtractor( + format=MarkitdownFormat.docx, + limits=MarkitdownLimits(max_output_bytes=1000, max_memory_bytes=2000, cpu_seconds=3), + python_executable="python-x", + ) + + extracted = await extractor.extract(b"docx-bytes", file_name="plan.docx") + + assert process.argv == ( + "python-x", + "-m", + MARKITDOWN_WORKER_MODULE, + "--format", + "docx", + "--file-name", + "plan.docx", + "--max-output-bytes", + "1000", + "--max-memory-bytes", + "2000", + "--cpu-seconds", + "3", + ) + assert process.stdin.written == b"docx-bytes" + assert extracted.kind == "docx" + assert extracted.markdown == "# Goals" + assert extracted.extraction.page_count == 0 + assert extracted.extraction.has_tables is False + + +@pytest.mark.asyncio +async def test_markitdown_extractor_rejects_an_oversized_source_before_spawning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def unexpected_subprocess(*args: object, **kwargs: object) -> None: + raise AssertionError("must not spawn for an oversized source") + + monkeypatch.setattr(asyncio, "create_subprocess_exec", unexpected_subprocess) + extractor = MarkitdownExtractor( + format=MarkitdownFormat.docx, limits=MarkitdownLimits(max_source_bytes=3) + ) + + with pytest.raises(MarkitdownSourceTooLargeError): + await extractor.extract(b"docx-bytes", file_name="plan.docx") + + +@pytest.mark.asyncio +async def test_markitdown_extractor_names_the_missing_extra( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(importlib.util, "find_spec", lambda name: None) + + with pytest.raises(MarkitdownNotInstalledError, match=r"basic-memory\[documents\]"): + await MarkitdownExtractor(format=MarkitdownFormat.docx).extract( + b"docx-bytes", file_name="plan.docx" + ) + + +@pytest.mark.asyncio +async def test_markitdown_extractor_names_a_loop_without_subprocess_support( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def selector_loop_spawn(*args: object, **kwargs: object) -> None: + raise NotImplementedError + + monkeypatch.setattr(asyncio, "create_subprocess_exec", selector_loop_spawn) + + with pytest.raises(MarkitdownExtractionError, match="subprocess support"): + await MarkitdownExtractor(format=MarkitdownFormat.docx).extract( + b"docx-bytes", file_name="plan.docx" + ) + + +@pytest.mark.asyncio +async def test_markitdown_extractor_kills_the_child_on_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + process = FakeProcess(returncode=None, hang=True) + install_fake_process(monkeypatch, process) + extractor = MarkitdownExtractor( + format=MarkitdownFormat.docx, limits=MarkitdownLimits(timeout_seconds=0.01) + ) + + with pytest.raises(MarkitdownExtractionError, match="deadline"): + await extractor.extract(b"docx-bytes", file_name="plan.docx") + + assert process.killed + assert process.waited + + +@pytest.mark.asyncio +async def test_markitdown_extractor_reports_a_failed_child( + monkeypatch: pytest.MonkeyPatch, +) -> None: + install_fake_process(monkeypatch, FakeProcess(returncode=3, stderr=b"boom\n")) + + with pytest.raises(MarkitdownExtractionError, match="exit code 3: boom"): + await MarkitdownExtractor(format=MarkitdownFormat.docx).extract( + b"docx-bytes", file_name="plan.docx" + ) + + +@pytest.mark.asyncio +async def test_markitdown_extractor_kills_a_child_that_overruns_a_pipe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + process = FakeProcess(returncode=None, stdout=b"x" * 64, hang=True) + install_fake_process(monkeypatch, process) + extractor = MarkitdownExtractor( + format=MarkitdownFormat.docx, limits=MarkitdownLimits(max_output_bytes=16) + ) + + with pytest.raises(MarkitdownExtractionError, match="output byte limit on stdout"): + await extractor.extract(b"docx-bytes", file_name="plan.docx") + + assert process.killed + + +@pytest.mark.asyncio +async def test_markitdown_extractor_rejects_an_invalid_child_result( + monkeypatch: pytest.MonkeyPatch, +) -> None: + install_fake_process(monkeypatch, FakeProcess(stdout=b'{"engine_version": 1}')) + + with pytest.raises(MarkitdownExtractionError, match="invalid result"): + await MarkitdownExtractor(format=MarkitdownFormat.docx).extract( + b"docx-bytes", file_name="plan.docx" + ) + + +@pytest.mark.asyncio +async def test_markitdown_extractor_rejects_a_worker_that_converted_another_format( + monkeypatch: pytest.MonkeyPatch, +) -> None: + install_fake_process( + monkeypatch, FakeProcess(stdout=valid_output_json(format=MarkitdownFormat.pptx)) + ) + + with pytest.raises(MarkitdownExtractionError, match="converted pptx, expected docx"): + await MarkitdownExtractor(format=MarkitdownFormat.docx).extract( + b"docx-bytes", file_name="plan.docx" + ) + + +def test_markitdown_extracted_document_uses_slides_as_pages() -> None: + output = MarkitdownOutput( + engine_version="0.1.7", + format=MarkitdownFormat.pptx, + markdown="\n# A\n\n\n# B\n\n", + processing_time_ms=3, + slide_count=3, + ) + + extracted = markitdown_extracted_document( + output, limits=MarkitdownLimits(), extracted_at=datetime(2026, 9, 7, tzinfo=UTC) + ) + + assert extracted.extraction.page_count == 3 + assert extracted.extraction.extracted_page_count == 3 + assert extracted.extraction.requires_ocr is False + + +def test_markitdown_extractors_cover_docx_and_pptx_with_shared_limits() -> None: + limits = MarkitdownLimits(timeout_seconds=5.0) + + extractors = markitdown_extractors(limits, python_executable="python-x") + + assert set(extractors) == {DOCX_MEDIA_TYPE, PPTX_MEDIA_TYPE} + docx = extractors[DOCX_MEDIA_TYPE] + pptx = extractors[PPTX_MEDIA_TYPE] + assert isinstance(docx, MarkitdownExtractor) and docx.format is MarkitdownFormat.docx + assert isinstance(pptx, MarkitdownExtractor) and pptx.format is MarkitdownFormat.pptx + assert docx.limits is limits and pptx.limits is limits + assert docx.python_executable == "python-x" + + +def test_markitdown_limits_are_strict() -> None: + with pytest.raises(ValidationError): + MarkitdownLimits.model_validate({"max_source_bytes": "1"}) diff --git a/tests/document_ingestion/test_markitdown_worker.py b/tests/document_ingestion/test_markitdown_worker.py new file mode 100644 index 000000000..5cadb2e4d --- /dev/null +++ b/tests/document_ingestion/test_markitdown_worker.py @@ -0,0 +1,161 @@ +"""Tests for the one-shot markitdown worker process.""" + +from __future__ import annotations + +import importlib.metadata +import io +import json +import zipfile +from types import SimpleNamespace + +import pytest + +from basic_memory.document_ingestion import markitdown_worker, worker_limits +from basic_memory.document_ingestion.markitdown_extractor import ( + MARKITDOWN_ENGINE, + MarkitdownFormat, + MarkitdownOutput, +) +from tests.document_ingestion.office_fixtures import ( + minimal_docx, + minimal_pptx, + pptx_with_picture, +) + + +def test_convert_office_bytes_renders_docx_headings() -> None: + output = markitdown_worker.convert_office_bytes( + minimal_docx(), + format=MarkitdownFormat.docx, + file_name="plan.docx", + max_output_bytes=1_000_000, + ) + + assert output.engine == MARKITDOWN_ENGINE + assert output.engine_version == importlib.metadata.version("markitdown") + assert output.format is MarkitdownFormat.docx + assert output.slide_count is None + assert output.markdown == "# Goals\n\nShip document ingestion for Office formats." + + +def test_convert_office_bytes_counts_pptx_slides_and_keeps_notes_and_tables() -> None: + output = markitdown_worker.convert_office_bytes( + minimal_pptx(), + format=MarkitdownFormat.pptx, + file_name="deck.pptx", + max_output_bytes=1_000_000, + ) + + assert output.slide_count == 2 + assert "" in output.markdown + assert "Speaker note: mention the sidecar pattern" in output.markdown + assert "| pdf | pdf-inspector |" in output.markdown + # markitdown's own convert loop collapses blank runs; the direct call must too. + assert "\n\n\n" not in output.markdown + + +def test_convert_office_bytes_drops_picture_references_with_filename_alt_text() -> None: + output = markitdown_worker.convert_office_bytes( + pptx_with_picture(), + format=MarkitdownFormat.pptx, + file_name="deck.pptx", + max_output_bytes=1_000_000, + ) + + assert output.markdown == "\n# Architecture" + + +@pytest.mark.parametrize( + ("markdown", "expected"), + [ + ("Before ![Revenue by region](Picture3.jpg) after", "Before Revenue by region after"), + ("![image.png](Picture2.jpg)", ""), + ("![](data:image/png;base64,AAAA)", ""), + ("![ Chart 1.PNG ](x)", ""), + ("No pictures here", "No pictures here"), + ], +) +def test_strip_image_references_keeps_only_descriptive_alt_text( + markdown: str, expected: str +) -> None: + assert markitdown_worker.strip_image_references(markdown) == expected + + +def test_convert_office_bytes_enforces_the_output_limit() -> None: + with pytest.raises(ValueError, match="output byte limit"): + markitdown_worker.convert_office_bytes( + minimal_docx(), + format=MarkitdownFormat.docx, + file_name="plan.docx", + max_output_bytes=10, + ) + + +def test_convert_office_bytes_fails_on_bytes_that_are_not_an_office_archive() -> None: + with pytest.raises(zipfile.BadZipFile): + markitdown_worker.convert_office_bytes( + b"not a zip archive", + format=MarkitdownFormat.docx, + file_name="plan.docx", + max_output_bytes=1_000_000, + ) + + +def run_main( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + *, + data: bytes, + max_output_bytes: int, +) -> tuple[str, list[tuple[str, int]]]: + applied: list[tuple[str, int]] = [] + monkeypatch.setattr( + markitdown_worker, "apply_memory_limit", lambda n: applied.append(("memory", n)) + ) + monkeypatch.setattr(markitdown_worker, "apply_cpu_limit", lambda n: applied.append(("cpu", n))) + monkeypatch.setattr( + markitdown_worker.sys, + "argv", + [ + "markitdown_worker", + "--format", + "docx", + "--file-name", + "plan.docx", + "--max-output-bytes", + str(max_output_bytes), + "--max-memory-bytes", + "1024", + "--cpu-seconds", + "5", + ], + ) + monkeypatch.setattr(markitdown_worker.sys, "stdin", SimpleNamespace(buffer=io.BytesIO(data))) + markitdown_worker.main() + return capsys.readouterr().out, applied + + +def test_main_applies_limits_then_writes_one_validated_json_result( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + payload, applied = run_main( + monkeypatch, capsys, data=minimal_docx(), max_output_bytes=1_000_000 + ) + + assert applied == [("memory", 1024), ("cpu", 5)] + output = MarkitdownOutput.model_validate_json(payload, strict=True) + assert output.format is MarkitdownFormat.docx + assert json.loads(payload)["markdown"].startswith("# Goals") + + +def test_main_bounds_the_whole_envelope( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + # The body alone fits under the cap; the JSON envelope around it does not. + with pytest.raises(ValueError, match="result exceeds"): + run_main(monkeypatch, capsys, data=minimal_docx(), max_output_bytes=120) + + +def test_worker_limits_module_is_the_single_owner_of_rlimits() -> None: + assert markitdown_worker.apply_cpu_limit is worker_limits.apply_cpu_limit + assert markitdown_worker.apply_memory_limit is worker_limits.apply_memory_limit diff --git a/tests/document_ingestion/test_pdf_inspector.py b/tests/document_ingestion/test_pdf_inspector.py index abdb6d99a..8aed2de8a 100644 --- a/tests/document_ingestion/test_pdf_inspector.py +++ b/tests/document_ingestion/test_pdf_inspector.py @@ -10,6 +10,8 @@ import pytest from pydantic import ValidationError +from basic_memory.document_ingestion.bounded_process import kill_and_wait +from tests.document_ingestion.process_fakes import FakeProcess, install_fake_process from basic_memory.document_ingestion.pdf_inspector import ( PDF_INSPECTOR_ENGINE, PdfInspector, @@ -19,7 +21,6 @@ PdfInspectorProcessError, PdfInspectorSourceTooLargeError, PdfInspectorTimeoutError, - _kill_and_wait, ) @@ -59,77 +60,6 @@ def minimal_text_pdf() -> bytes: return bytes(document) -class FakeStdin: - """Scripted stdin writer; flags when the adapter starts feeding the child.""" - - def __init__(self, *, started: asyncio.Event, broken: bool) -> None: - self._started = started - self._broken = broken - self.written = b"" - self.closed = False - - def write(self, data: bytes) -> None: - self._started.set() - self.written += data - - async def drain(self) -> None: - if self._broken: - raise BrokenPipeError - - def close(self) -> None: - self.closed = True - - -def pipe(data: bytes, *, eof: bool = True) -> asyncio.StreamReader: - reader = asyncio.StreamReader() - reader.feed_data(data) - if eof: - reader.feed_eof() - return reader - - -class FakeProcess: - """Stand-in for asyncio's subprocess handle with scripted pipes.""" - - def __init__( - self, - *, - returncode: int | None = 0, - stdout: bytes = b"", - stderr: bytes = b"", - hang: bool = False, - kill_raises: bool = False, - stdin_broken: bool = False, - ) -> None: - self.returncode = returncode - self._kill_raises = kill_raises - self.communicate_started = asyncio.Event() - self.stdin = FakeStdin(started=self.communicate_started, broken=stdin_broken) - # A hanging child never closes stdout, so the capped read waits forever. - self.stdout = pipe(stdout, eof=not hang) - self.stderr = pipe(stderr) - self.killed = False - self.waited = False - - def kill(self) -> None: - if self._kill_raises: - raise ProcessLookupError - self.killed = True - self.returncode = -9 - - async def wait(self) -> int: - self.waited = True - return self.returncode if self.returncode is not None else -9 - - -def install_fake_process(monkeypatch: pytest.MonkeyPatch, process: FakeProcess) -> None: - async def create_subprocess(*args: object, **kwargs: object) -> FakeProcess: - _ = (args, kwargs) - return process - - monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess) - - @pytest.mark.skipif( sys.platform == "win32", reason="Basic Memory pins the selector event loop on Windows, which cannot spawn subprocesses", @@ -309,11 +239,11 @@ async def test_pdf_inspector_rejects_invalid_child_result( @pytest.mark.asyncio async def test_kill_and_wait_tolerates_a_child_that_already_exited() -> None: raced = FakeProcess(returncode=None, kill_raises=True) - await _kill_and_wait(cast(asyncio.subprocess.Process, raced)) + await kill_and_wait(cast(asyncio.subprocess.Process, raced)) assert raced.waited is True finished = FakeProcess(returncode=0) - await _kill_and_wait(cast(asyncio.subprocess.Process, finished)) + await kill_and_wait(cast(asyncio.subprocess.Process, finished)) assert finished.killed is False assert finished.waited is True diff --git a/tests/document_ingestion/test_pdf_inspector_worker.py b/tests/document_ingestion/test_pdf_inspector_worker.py index f09eba5dc..ea8db094c 100644 --- a/tests/document_ingestion/test_pdf_inspector_worker.py +++ b/tests/document_ingestion/test_pdf_inspector_worker.py @@ -13,7 +13,7 @@ import pytest -from basic_memory.document_ingestion import pdf_inspector_worker +from basic_memory.document_ingestion import pdf_inspector_worker, worker_limits from basic_memory.document_ingestion.pdf_inspector import PDF_INSPECTOR_ENGINE, PdfInspectorOutput from basic_memory.document_ingestion.pdf_inspector import PdfInspectorLimits from basic_memory.document_ingestion.raw_document import ( @@ -184,40 +184,40 @@ def fake_posix_resource() -> SimpleNamespace: def test_worker_bounds_linux_address_space(monkeypatch: pytest.MonkeyPatch) -> None: resource = fake_posix_resource() - monkeypatch.setattr(pdf_inspector_worker.sys, "platform", "linux") - monkeypatch.setattr(pdf_inspector_worker, "resource", resource) + monkeypatch.setattr(worker_limits.sys, "platform", "linux") + monkeypatch.setattr(worker_limits, "resource", resource) - pdf_inspector_worker._apply_memory_limit(512 * 1024 * 1024) + worker_limits.apply_memory_limit(512 * 1024 * 1024) resource.setrlimit.assert_called_once_with("RLIMIT_AS", (512 * 1024 * 1024, 512 * 1024 * 1024)) def test_worker_skips_address_space_limit_off_linux(monkeypatch: pytest.MonkeyPatch) -> None: resource = fake_posix_resource() - monkeypatch.setattr(pdf_inspector_worker.sys, "platform", "darwin") - monkeypatch.setattr(pdf_inspector_worker, "resource", resource) + monkeypatch.setattr(worker_limits.sys, "platform", "darwin") + monkeypatch.setattr(worker_limits, "resource", resource) - pdf_inspector_worker._apply_memory_limit(512 * 1024 * 1024) + worker_limits.apply_memory_limit(512 * 1024 * 1024) resource.setrlimit.assert_not_called() def test_worker_bounds_cpu_time(monkeypatch: pytest.MonkeyPatch) -> None: resource = fake_posix_resource() - monkeypatch.setattr(pdf_inspector_worker, "resource", resource) + monkeypatch.setattr(worker_limits, "resource", resource) - pdf_inspector_worker._apply_cpu_limit(25) + worker_limits.apply_cpu_limit(25) resource.setrlimit.assert_called_once_with("RLIMIT_CPU", (25, 26)) def test_worker_skips_rlimits_without_posix_resource(monkeypatch: pytest.MonkeyPatch) -> None: """Windows has no rlimits; the parent's deadline is the only ceiling there.""" - monkeypatch.setattr(pdf_inspector_worker, "resource", None) - monkeypatch.setattr(pdf_inspector_worker.sys, "platform", "linux") + monkeypatch.setattr(worker_limits, "resource", None) + monkeypatch.setattr(worker_limits.sys, "platform", "linux") - pdf_inspector_worker._apply_cpu_limit(25) - pdf_inspector_worker._apply_memory_limit(1024) + worker_limits.apply_cpu_limit(25) + worker_limits.apply_memory_limit(1024) def run_worker_main( @@ -226,10 +226,10 @@ def run_worker_main( applied: list[tuple[str, int]] = [] monkeypatch.setattr(pdf_inspector_worker, "pdf_inspector", engine) monkeypatch.setattr( - pdf_inspector_worker, "_apply_memory_limit", lambda n: applied.append(("memory", n)) + pdf_inspector_worker, "apply_memory_limit", lambda n: applied.append(("memory", n)) ) monkeypatch.setattr( - pdf_inspector_worker, "_apply_cpu_limit", lambda n: applied.append(("cpu", n)) + pdf_inspector_worker, "apply_cpu_limit", lambda n: applied.append(("cpu", n)) ) monkeypatch.setattr(sys, "stdin", SimpleNamespace(buffer=io.BytesIO(b"%PDF-test"))) monkeypatch.setattr( diff --git a/uv.lock b/uv.lock index e16b89711..e633fe32c 100644 --- a/uv.lock +++ b/uv.lock @@ -327,6 +327,9 @@ dependencies = [ ] [package.optional-dependencies] +documents = [ + { name = "markitdown", extra = ["docx", "pptx"] }, +] milvus = [ { name = "pymilvus" }, { name = "pymilvus", extra = ["milvus-lite"], marker = "sys_platform != 'win32'" }, @@ -346,6 +349,7 @@ dev = [ { name = "icecream" }, { name = "libcst" }, { name = "logfire" }, + { name = "markitdown", extra = ["docx", "pptx"] }, { name = "pdf-inspector" }, { name = "psycopg" }, { name = "pyright" }, @@ -381,6 +385,7 @@ requires-dist = [ { name = "logfire", specifier = ">=4.19.0" }, { name = "loguru", specifier = ">=0.7.3" }, { name = "markdown-it-py", specifier = ">=3.0.0" }, + { name = "markitdown", extras = ["docx", "pptx"], marker = "extra == 'documents'", specifier = ">=0.1.7,<0.2" }, { name = "mcp", specifier = ">=2,<3" }, { name = "mdformat", specifier = ">=0.7.22" }, { name = "mdformat-frontmatter", specifier = ">=2.0.8" }, @@ -412,7 +417,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32'", specifier = ">=0.21.0" }, { name = "watchfiles", specifier = ">=1.0.4" }, ] -provides-extras = ["milvus", "pdf", "redis"] +provides-extras = ["documents", "milvus", "pdf", "redis"] [package.metadata.requires-dev] dev = [ @@ -422,6 +427,7 @@ dev = [ { name = "icecream", specifier = ">=2.1.3" }, { name = "libcst", specifier = ">=1.8.6" }, { name = "logfire", specifier = ">=4.19.0" }, + { name = "markitdown", extras = ["docx", "pptx"], specifier = ">=0.1.7,<0.2" }, { name = "pdf-inspector", specifier = ">=0.2.6,<2" }, { name = "psycopg", specifier = ">=3.2.0" }, { name = "pyright", specifier = ">=1.1.408" }, @@ -448,6 +454,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl", hash = "sha256:d16c9bbc61ea14637596c5f6fbff2ee99cbe3573e46a716401734ef50c3060c2", size = 1333658, upload-time = "2025-12-13T06:50:28.266Z" }, ] +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + [[package]] name = "cachetools" version = "7.1.4" @@ -658,6 +677,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] +[[package]] +name = "cobble" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/7a/a507c709be2c96e1bb6102eb7b7f4026c5e5e223ef7d745a17d239e9d844/cobble-0.1.4.tar.gz", hash = "sha256:de38be1539992c8a06e569630717c485a5f91be2192c461ea2b220607dfa78aa", size = 3805, upload-time = "2024-06-01T18:11:09.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/e1/3714a2f371985215c219c2a70953d38e3eed81ef165aed061d21de0e998b/cobble-0.1.4-py3-none-any.whl", hash = "sha256:36c91b1655e599fd428e2b95fdd5f0da1ca2e9f1abb0bc871dec21a0e78a2b44", size = 3984, upload-time = "2024-06-01T18:11:07.911Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -829,6 +857,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/7c/2e5dcf53909deddd0bf38cbe277ad9806be038276b1c6c436561b4d9b2e2/dateparser-1.4.1-py3-none-any.whl", hash = "sha256:f25d4e051a84be27a35bd297e3e1dc59ff78373701b89be352ba80372d22d0d0", size = 300503, upload-time = "2026-06-15T08:45:45.951Z" }, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + [[package]] name = "detect-installer" version = "0.1.0" @@ -2032,6 +2069,134 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/f0/92f2d609d6642b5f30cb50a885d2bf1483301c69d5786286500d15651ef2/lsprotocol-2025.0.0-py3-none-any.whl", hash = "sha256:f9d78f25221f2a60eaa4a96d3b4ffae011b107537facee61d3da3313880995c7", size = 76250, upload-time = "2025-06-17T21:30:19.455Z" }, ] +[[package]] +name = "lxml" +version = "6.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/ad/28ecd7cb894d172f3c9c80a075eeeb2017ac62e3632cee05a5f9493547eb/lxml-6.1.3.tar.gz", hash = "sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21", size = 4211198, upload-time = "2026-09-02T14:48:02.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/1f/a180b57d9eeabaab77f9d5aa30356898ea749c4795596a8f66d1eb6bef2e/lxml-6.1.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c0710ac085a157b593c38fbcacd950f15c4afa8e2057527185875ab302752bc", size = 8602094, upload-time = "2026-09-02T14:47:26.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/25/070c92013a1c029a602b03560d68772313d918268667fa993da7961759c9/lxml-6.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:623c8799c17128753c65699f1c3aa32402657393a9ad6db09ed8b98ddf76611d", size = 4638308, upload-time = "2026-09-02T14:47:29.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1c/722e88883173097a1a375153e3c2447eba3060d0231522cf6596e99f4195/lxml-6.1.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f683dc6300317700025e41d89a43e0276692ded16113a3c43eab704d605c58e5", size = 4939696, upload-time = "2026-09-02T14:47:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/db/36/aa413bc214dc4f785ad2b2ddd8cc99aae7062d49ab155e91e6011af00daf/lxml-6.1.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11", size = 5105247, upload-time = "2026-09-02T14:47:36.734Z" }, + { url = "https://files.pythonhosted.org/packages/a3/a0/a1f7f1313795bfec67b77f01ef3b1128d49f2d7f66a8413fa55d47f4e25f/lxml-6.1.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b37772102d44bb6628186accca3a121b1fa3a6b3d97518a8c29a5229ca4c0d0a", size = 5011915, upload-time = "2026-09-02T14:47:39.846Z" }, + { url = "https://files.pythonhosted.org/packages/b9/78/840e7e3f1d0cc7a5cfac5d8505b97e25b6427fd774ac4bae672aaebfb4b5/lxml-6.1.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddcf547bea2aee967d6a77779376a45e77e610e8465147a1f3d7e20d539d6e32", size = 5638175, upload-time = "2026-09-02T14:47:43.644Z" }, + { url = "https://files.pythonhosted.org/packages/0a/20/e022dbc6b4753a9bc9fc5fb28a27163430c1731b9913997f6544c1b2518c/lxml-6.1.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c", size = 5244675, upload-time = "2026-09-02T14:47:47.635Z" }, + { url = "https://files.pythonhosted.org/packages/99/83/82cde81d2b5eb38d1539fdfdf318abdd014a7e604f4df01c9cd3deb18f2a/lxml-6.1.3-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:a5c18810318303ce9afb3f95e2ddb54834f96fa699a8600433fd5a93dcf44c56", size = 5358205, upload-time = "2026-09-02T14:47:50.306Z" }, + { url = "https://files.pythonhosted.org/packages/d2/a1/f3b057371c8cb29f2a9c9c44ea320592446e40b74a4b0af68c3d8e65bc73/lxml-6.1.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:3e42265103fb385d8642a78672edf376c6f7e1d3598a7a4f9cb1278f2f6b5f6f", size = 4704495, upload-time = "2026-09-02T14:47:53.251Z" }, + { url = "https://files.pythonhosted.org/packages/1a/a4/230eb28be5d412152ffc3c679b51fe1aeede5a53f3a8eb6e9748f2f4754f/lxml-6.1.3-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:21402998e4b78e7cce237d2788841aaa21ac9a4d1574d04dc2d12ee41ae807b5", size = 5255117, upload-time = "2026-09-02T14:47:55.963Z" }, + { url = "https://files.pythonhosted.org/packages/a3/18/1969f56763af24ce42ea156007b0b2d73fddea552e283b2010416394f0f4/lxml-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:38fc4e4e4e084e0bd491949482527d406788045c546d4f8789e93fc527b91385", size = 5054424, upload-time = "2026-09-02T14:47:58.131Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/2a90acc1f6fabaa3a8db9340437822bd8d041b205d626a4b3e8621aaa390/lxml-6.1.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5609efdb0d3c95499c00046bc53648b3482ec2175b5503d6e611b3f0555dc71d", size = 4785572, upload-time = "2026-09-02T14:48:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/a5/1e/b90e845b1dcd0f2f3f26b98283d857f25909223aacd265eee032c34ab8b1/lxml-6.1.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:97ce49699d87ebf8aad631b55d65b33219a4f1bfefbbf5bff19dc9af160aeaf9", size = 5656516, upload-time = "2026-09-02T14:48:03.419Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ab/0a1b802c57f3fba5c4efd77d5c6b78adaa8f7b681f0c90456b140fe8bf6c/lxml-6.1.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:48542c9acba9ff9450bd18d871d2c2c8787fdb283572b623d206f1b927cd7d9e", size = 5245982, upload-time = "2026-09-02T14:48:06.109Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/2c016fbceb3778137459292538d9dfa7e3ad9070fe409c15254ddd90d2cc/lxml-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c55e71a9b1db1f107efb60da49c093689b74c5c31a708e5379e2fd9439d4fbb5", size = 5267340, upload-time = "2026-09-02T14:48:08.374Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b1/736d18fd6f0835761923b7bac1f0c27d60c1200384e9093f05d8c5100525/lxml-6.1.3-cp312-cp312-win32.whl", hash = "sha256:b3ff39654f0ce6ebd4db154211136dbe7e8157bcc3bed2344c87f32c7c6ecb6c", size = 3602606, upload-time = "2026-09-02T14:48:10.384Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5b/6ed903e4e6278a020c8a6f0dbbe78030d041840a6b4a64ea441a1e414077/lxml-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:3e9a00d1c2c30936f7add097c41afc5da6556c580909104aafd382cac92a855c", size = 4005999, upload-time = "2026-09-02T14:48:12.51Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1b/7bcebb7b6332cb3ae85e9c13b139adb6f23f75c71d84041c56a5005d9a29/lxml-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:1aeca87830c4fe649dcf93fe2b059525b71c72587f21be4ae4af7103082a79fa", size = 3666631, upload-time = "2026-09-02T14:48:14.567Z" }, + { url = "https://files.pythonhosted.org/packages/52/05/3ef45db776baea068044c799bbba68f3ca00a440c0e930a17c572f3d9639/lxml-6.1.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:3a48093cdb058a93af842ede9703520e810b05dcd0fc6d7190a06376c3bfb6bd", size = 8590357, upload-time = "2026-09-02T14:48:17.413Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a5/eee2fc77eee5ea68e4a4334b1def1781a3beaeefd3d98e81b4a38dc447b7/lxml-6.1.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:887c021d9a977cff89cb273047c1352997b772a8908a25c21836861f69b92be1", size = 4632616, upload-time = "2026-09-02T14:48:20.745Z" }, + { url = "https://files.pythonhosted.org/packages/35/42/df27b56848acd29d8a720acc28977911aab36f2a09df4208d5502e887415/lxml-6.1.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:611a51e61c92f62345a50b0035df6fc0d678f9299f33728826d831598862f59d", size = 4936186, upload-time = "2026-09-02T14:48:22.94Z" }, + { url = "https://files.pythonhosted.org/packages/ab/8d/8a7b91df0b54d09d25f5f44885d6b3e0a6d6643a8c070191580318d20c42/lxml-6.1.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b477912f42c5c33405a10c759d22f80cf5af043ae02d95b9d8e5e5bc555739ed", size = 5093324, upload-time = "2026-09-02T14:48:25.132Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7e/8f340ddcd43790332fb0de8a26628d571a492da3300cd191821698407c96/lxml-6.1.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cffe18571ccc51d742cd08cbb3f8b756de9311d18c7ea98f5d92f37b8fb60c2", size = 4998850, upload-time = "2026-09-02T14:48:27.394Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c1/9c5bb572f1f09ec9e4322bd4a4e9f4ad48347fc56ef94cf4df58a5279dc8/lxml-6.1.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:75cc6569e86be5785b6188ef1642670c6adbc984e81ec35e224842ecd9eefcc8", size = 5626813, upload-time = "2026-09-02T14:48:29.61Z" }, + { url = "https://files.pythonhosted.org/packages/ac/7d/8bf1fd8bae8247743968bb76d027a1ac5bd2c4b44495fba6a71b30d10706/lxml-6.1.3-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d85dfab42dd672f87a7f76e9de7172962aee69fa12044f0d6e1a23cbd53fb80e", size = 5232385, upload-time = "2026-09-02T14:48:31.969Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/6cef69ed81cb7df0d03b0dd09d08e6e2cf5061a743ff6f42f0b741548e9b/lxml-6.1.3-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:42632b4024ab24a6b488f559ac851312509888b6b80ae2aa11cf29a646a0d245", size = 5347088, upload-time = "2026-09-02T14:48:34.13Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e1/8e5fd8ddc8c7d685badb0f2db149e3c9da84eefc2827c01c658df2c4e3cb/lxml-6.1.3-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:febd35ef45f603c2d74b74655efdbf45e14f55fc0aef4ac82b663ca829b283e0", size = 4707227, upload-time = "2026-09-02T14:48:36.62Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7e/00041382a11be40a88bf405ebff11c8efabd3de79f2691e1638b1c47a8a0/lxml-6.1.3-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a43b3bdf11e477dc7770609d3477316f974354dfc8425d596f64f471cc8daf6e", size = 5240208, upload-time = "2026-09-02T14:48:38.893Z" }, + { url = "https://files.pythonhosted.org/packages/fd/fe/316538b5cff0936fa63d45d421c655730fcbb5a28dcac728c175083002bc/lxml-6.1.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d582042c69857c364e8153de6e18e0da9b7b515a6a8113caf69a6ec8e0520f2", size = 5050271, upload-time = "2026-09-02T14:48:41.213Z" }, + { url = "https://files.pythonhosted.org/packages/c9/91/455bcccb3ac725373007344d351151810cd19762d1673b64b811f4359a42/lxml-6.1.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8e49a646acfab83c68974f4aa1d0a2acca9e88d7d627ae0fc13201b14b76d310", size = 4780433, upload-time = "2026-09-02T14:48:43.779Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f6/580440e2f52cf00bba5c5e1080bfa88cdfcde73be71a11d95170ddbb663f/lxml-6.1.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0dee106e9aa97fb00541b1ed7827070564d0549c3d3fba8920e6b20fd980f748", size = 5645928, upload-time = "2026-09-02T14:48:46.187Z" }, + { url = "https://files.pythonhosted.org/packages/f6/dc/d123c1f244306543d545f62443f794959e4f1ea709fe100f8740d514e74a/lxml-6.1.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:dd5e90f34cffcfed97f36cf066325773d2b6021c60c29942e53a18b028501b1d", size = 5231184, upload-time = "2026-09-02T14:48:48.691Z" }, + { url = "https://files.pythonhosted.org/packages/c3/3c/fe55b2bd5c6113c906511cd88f6a470195c5fbff1124f19970ab706c3477/lxml-6.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9b3e7d71bf6acff341233417abbdface29c647e3113892d9aaedc02eb4aa2bc", size = 5255814, upload-time = "2026-09-02T14:48:50.948Z" }, + { url = "https://files.pythonhosted.org/packages/e7/a7/485df55acf55dc35e4ca89d2f48f03889e5a3241826b18b85102b32ce9d8/lxml-6.1.3-cp313-cp313-win32.whl", hash = "sha256:160fcf381f76c3aeac28a756bec44f48942a8f7245a87aa28e3a523b4d90cd87", size = 3602214, upload-time = "2026-09-02T14:48:53.236Z" }, + { url = "https://files.pythonhosted.org/packages/c0/28/e46a7702bd95e9043291f7c3539b6184cba66f96cea9936f20939b284eeb/lxml-6.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:e477aca0bc0d19f3b4ae9e4f2a1cfd687c31bf772d78734910658186b40b2477", size = 4004091, upload-time = "2026-09-02T14:48:55.699Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1d/154c78e20479a43916e63f19cb720d83f44f024b03228be44c92d9a97b24/lxml-6.1.3-cp313-cp313-win_arm64.whl", hash = "sha256:b1cc980905221a5d8b3c476330730b3adb40ff80add71ffbdb6215ba055656f1", size = 3665468, upload-time = "2026-09-02T14:48:57.703Z" }, + { url = "https://files.pythonhosted.org/packages/0c/15/fc75a70b0af6021d0ea16811f1fc71cc42cd06ce90fe10f007a69b2eed84/lxml-6.1.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2bec13085dc8ef48a3fe62f7dfcacfeda2c785cdf19cc8eeda2bb9ed081da165", size = 8609725, upload-time = "2026-09-02T14:49:00.156Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/398fcf9018f881ec9aeaafae1ddd6586dfb13314a35d35e899de373dcae0/lxml-6.1.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4f4db7c7e954d289d71878938348b3d91b904a3e8210a11939359fb758a58e7d", size = 4639629, upload-time = "2026-09-02T14:49:02.81Z" }, + { url = "https://files.pythonhosted.org/packages/a7/2d/49b6a6ad7ce8f64b07b9fe852ff0c6d3fcbb26db61bee4f63d4120180a1c/lxml-6.1.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2cae5d5c90a62d9139c512a0cb1aad1d182b022b5740daea2617eb5bf7fc658e", size = 4965074, upload-time = "2026-09-02T14:49:05.133Z" }, + { url = "https://files.pythonhosted.org/packages/66/bc/6230cf80e4331c33383b0b6b73dc31a393dd76edd4cb73d761de5123034d/lxml-6.1.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c6c0c13128a32eb04a51357e56a094e13aa8e6d3d1884de2e9ae923f6915e1a8", size = 5099355, upload-time = "2026-09-02T14:49:07.343Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cf/d1143d9b7717e07a82f158a1fc9ce6e581fdad1226734950af869e3ffde4/lxml-6.1.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2221e88679d1351e9a40aaee54bc65679b9795bbd0160bc3d5e36b163344eb75", size = 5036795, upload-time = "2026-09-02T14:49:09.65Z" }, + { url = "https://files.pythonhosted.org/packages/31/6f/194bb00ffb89712c30f5a7e1b8e685590e140fad6c8261fec172c09a3dc0/lxml-6.1.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cfb398886a7eb4c719161c3efcff2a1248febc53a4d8e5072d2d8a87fed84ac9", size = 5658740, upload-time = "2026-09-02T14:49:11.9Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/27e3cee3dcdb3b7bc09727b642bdbfcd098490ea77df04611db9060d7722/lxml-6.1.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7eb78ba28b187e1e9203a55c60fcf70df2d22cb205fe6d51b9383d6097419f0", size = 5245991, upload-time = "2026-09-02T14:49:14.154Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/8312560579fc980bbd2233a8a673cc46f7d613d3633f2bf08a21e8f4ad13/lxml-6.1.3-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:ea6b1e9105b4b24a34c722432d9fb578f9ed83af21fa1abda639011e0f22bbb6", size = 5354136, upload-time = "2026-09-02T14:49:16.459Z" }, + { url = "https://files.pythonhosted.org/packages/74/d8/eda60f4f73a9c780b5d6e1175484f66e6c81a2c93346e2906a1fec9c7a02/lxml-6.1.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:e8b17e23df3e827a69d25af70990ca2420e92668aaffaeeb3cd2351d7916a023", size = 4704379, upload-time = "2026-09-02T14:49:19.032Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c8/c9cc60057be78ac34bd2b842e45e6e88edbfe5e532e82c3b82381b7aab49/lxml-6.1.3-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b7c37339d7e75cab9a123a04248e243cefefb302ad6db566ea0c77cbcde421e", size = 5258676, upload-time = "2026-09-02T14:49:21.306Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/66894008fee8d1785b8db129747ae963fd427b68f456918df7f2f24a8b98/lxml-6.1.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:83e3a51e7933db700a0da0db31849db3a24022d9970da9bb73001e1d0326fd92", size = 5090069, upload-time = "2026-09-02T14:49:23.562Z" }, + { url = "https://files.pythonhosted.org/packages/8b/31/c1b60404859f4c3cd1f41f29c65a24e25cea78fde822d9574a21f66810be/lxml-6.1.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9bde9ae026a55b9a192078dfa6e27dd0ca4a050171ab6272e92f97b757dfdf48", size = 4741958, upload-time = "2026-09-02T14:49:26.037Z" }, + { url = "https://files.pythonhosted.org/packages/23/b8/6285f0cf546f14da2554cabdeaf7c2c2ff3190c74807f0de2e8810a786f9/lxml-6.1.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1a635e837b50a1819bebfedaac5916498ea024120969da8790500148fb0a894d", size = 5683245, upload-time = "2026-09-02T14:49:28.438Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f6/2168cab44336dcb15fed0f0b78577225b83297cdf0dee349c95420c3dcb0/lxml-6.1.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d0c5c362bc94f1929dc7e96e715bbe7bd17037f802e6d8f0d1545df9133c0559", size = 5246087, upload-time = "2026-09-02T14:49:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/f5/89/32f5de69a0a31f30e6164981851f87b37ecb2c4ee838e504b88d49d4818e/lxml-6.1.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c59e4265608da6a041f54646ecc0c9ecdbb19aaf14c4c684bb6c2114998cc415", size = 5269352, upload-time = "2026-09-02T14:49:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a1/741d952ed3a7ef7a50055c6415aec3f067015e97f72f4389ce77b09657ba/lxml-6.1.3-cp314-cp314-win32.whl", hash = "sha256:2e62c569ec7531b679b184cbfe335c501c1d13c4b363560013019962eb630e6d", size = 3662783, upload-time = "2026-09-02T14:50:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bc/5811cc73cac05e324e05ba9b0924e1a163a317a167ede8a9c748b11db30a/lxml-6.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:66299564c046bc7e0cc5de5106601eae907e9fa5904cd68a323380a8502f7861", size = 4073951, upload-time = "2026-09-02T14:50:26.348Z" }, + { url = "https://files.pythonhosted.org/packages/92/18/3768c8b01ac3a9bed1914715e6011711b00e2a11628ffa6f7fa37f8e0269/lxml-6.1.3-cp314-cp314-win_arm64.whl", hash = "sha256:ebd054ad1737a68fb7c5c073d405cef2b88bb824e294de3b4a4e995b47f0e376", size = 3749279, upload-time = "2026-09-02T14:50:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/72/38/84684784738d9451db2b330de2483f496690c3a5c642071df24135739b37/lxml-6.1.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5a143e6207579de8baeded4eaac9134413200359f1969d636f0bfb98ee8c3c8f", size = 8860296, upload-time = "2026-09-02T14:49:36.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/b7/fc4c50bb1b38e864010ea396046cabe85129bf9e65b11edcfbc37d356241/lxml-6.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a1cec0f99b9b914d39176347a93b7610dc09324491aee1cbc57cd291a41a1d55", size = 4755190, upload-time = "2026-09-02T14:49:39.872Z" }, + { url = "https://files.pythonhosted.org/packages/94/e2/ee9aa6ed2b666b2db1f6f7fd48964ff9da39ebe827ef5eac0ab881f639d9/lxml-6.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f6b9d2aad499c769ee8287609ab0e6de99d8bcea99c6e6c2e64945259fd52fb2", size = 4979517, upload-time = "2026-09-02T14:49:42.153Z" }, + { url = "https://files.pythonhosted.org/packages/29/e3/e7763d1661b283ddd4fa36f91b9a497db6b8d2aff55028b16c7f642e0755/lxml-6.1.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a23fefdb345b2d4d0ff2860571b5ff9a89a28b6a120f720e8fb0324d346626", size = 5115270, upload-time = "2026-09-02T14:49:44.493Z" }, + { url = "https://files.pythonhosted.org/packages/2d/cd/22205d5b4d177e3f4156f780412426ee7c7f8107809f119f0dcc40fa51e3/lxml-6.1.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:545ccc14fb05485f48b4439ec35beb16d5b5280eb6c81c658bd4707a2a119414", size = 5032449, upload-time = "2026-09-02T14:49:46.841Z" }, + { url = "https://files.pythonhosted.org/packages/da/43/06a4626c3bb79ef8c501b674afab8100d64e798665bb2a97d1c960636a49/lxml-6.1.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:93476b6514b373fc6ca67d26c442784f7807c86f00635bfe79f935c3eab2af17", size = 5603325, upload-time = "2026-09-02T14:49:49.664Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/733682a0c2de9f5779ba207bbb3f3f6be8c6bda863fc01739b186b38783a/lxml-6.1.3-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8db38ff3fb7aee7d6a82ae4da2eef1178656fe1216841fbd24870062a9d60473", size = 5229023, upload-time = "2026-09-02T14:49:52.447Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8a/e69cdaca3fd33a647942925664f01b20908d41a6968c182305be9c38fb11/lxml-6.1.3-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:25f4118c438f96bb466e83108506d03d5c31b1bd2387e83e5b070bda6ded9c37", size = 5317811, upload-time = "2026-09-02T14:49:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b2/0c397588174403c2ab68fc464abf97e03e7324f9c6cb6a99023104707195/lxml-6.1.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:1beb0f9909b26cee938df9ba56b15252a84429b1fc30ce6fca161390b9789a70", size = 4646516, upload-time = "2026-09-02T14:49:57.761Z" }, + { url = "https://files.pythonhosted.org/packages/56/7e/cfea25afafbe49db8b225764f7f74bb37c2a7f5e717d917d3d4a5e098ed4/lxml-6.1.3-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3a27ac6c780c8b8a1cd231b58407634cafc1c4cc28cd6c7141362df0f36351e7", size = 5240626, upload-time = "2026-09-02T14:50:00.279Z" }, + { url = "https://files.pythonhosted.org/packages/a1/75/7a587771bb52ebb0e2c57b6dbe9fd96a70fbb54d72ddd97d54c5f8ec18d5/lxml-6.1.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1932d7ce78a561367512c594fe66eac2b2ec9b9264cfd9b5f950622f4a116e2", size = 5086619, upload-time = "2026-09-02T14:50:03.245Z" }, + { url = "https://files.pythonhosted.org/packages/1e/01/94c0ebe6d831861542d251e038052e52bf6d33f1d18f1cfffdc82851065a/lxml-6.1.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7d0f5976aa2701996f759b30172925829867547bb073af0ae67d1307a0f0262c", size = 4758828, upload-time = "2026-09-02T14:50:05.873Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/938d67bd0e5b1fdfa52be28aefdffbad57e1f6b8e921c2aab88542c75f40/lxml-6.1.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e7ce578aa8a80910a72a8ca0bbea3baae10100827249001999726a788456d8", size = 5627083, upload-time = "2026-09-02T14:50:08.555Z" }, + { url = "https://files.pythonhosted.org/packages/d8/65/4e51522f6c214650db0abb7b16ccd11b1238b8a05a8d59aa4ebed59c9f67/lxml-6.1.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d97c5227621af74b111882a290b10f371780a38eef9d9e730408fba2259b52fb", size = 5235170, upload-time = "2026-09-02T14:50:11.255Z" }, + { url = "https://files.pythonhosted.org/packages/92/c2/e73d19365665f6b16ef84df21199befc3b06e4c539046ad2d9595f6fb9ea/lxml-6.1.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da707f14ea3c35ee463d50acd596d6488e4b2b4ae7cf77a5bf93f55c023d63e8", size = 5252273, upload-time = "2026-09-02T14:50:13.782Z" }, + { url = "https://files.pythonhosted.org/packages/48/a9/7f386c84c9fe2854e1ca6e231c285e1c8f392971ac353c6865e6ec49faff/lxml-6.1.3-cp314-cp314t-win32.whl", hash = "sha256:9efe56a68179f3adc4de41861c9358931db03837c48dd5e1c78077b84dd07f3a", size = 3902712, upload-time = "2026-09-02T14:50:16.171Z" }, + { url = "https://files.pythonhosted.org/packages/82/a6/8a3eb793f7900ef01c7f99e6f5fcbcfbdff35251cfaef66b32a4c16352d6/lxml-6.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c9389b3784b56c58d933b5e0aecdf28f901b073ff385358d8a7d40907f6e14b2", size = 4400979, upload-time = "2026-09-02T14:50:18.621Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c4/3807bea283b4fe9e9d9f5dde46a73df91178472b335d2778e10b2a37aa22/lxml-6.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32a409be3190b088f960ac92bfedfbef2f86c49ff940765e1548177592d20026", size = 3823401, upload-time = "2026-09-02T14:50:21.119Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8e/4614fcd65496054cfb7172662f3576a59200278739506433b8c241ea422a/lxml-6.1.3-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:6ea2f13dce778ca072ccee598bca46a092ce192e8fd907b6c1f0e52c800529a0", size = 8609378, upload-time = "2026-09-02T14:50:31.772Z" }, + { url = "https://files.pythonhosted.org/packages/f2/51/2cdce3c65fa99a6195dd8fbd512d33407c1000ad99f63e0a285b63d7a8eb/lxml-6.1.3-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c581b1d68b3845fb86c6b2983e755b29bf001461c59fa411d2c26a911b6559a9", size = 4640022, upload-time = "2026-09-02T14:50:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/52/09/0b30084e9eb1c546a4be3d9c56df70058d116b1a320400a59b0f7da87bf0/lxml-6.1.3-cp315-cp315-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e01125896585139453cab8cb235893644d8815d7509520da95ae3ee8d1c1f79", size = 5037928, upload-time = "2026-09-02T14:50:37.007Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0e/5c37275a3e361f6138dc06db748ea565c1fe8a5f4ee5e2ddd80047c81a89/lxml-6.1.3-cp315-cp315-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:290f66b97ede0e552e1cb44a0fd8a74f9753ee635b50830a0b122fb72788d015", size = 5661932, upload-time = "2026-09-02T14:50:39.777Z" }, + { url = "https://files.pythonhosted.org/packages/70/c5/b71ffb289b15e2642e2a3cf6d468c44da39ea119061a99e5b05e3d10f217/lxml-6.1.3-cp315-cp315-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73fc05988ed20809450474ba760a87c8ad4e455fc09783c02195e56ec634b41a", size = 5249209, upload-time = "2026-09-02T14:50:42.141Z" }, + { url = "https://files.pythonhosted.org/packages/81/ea/9910da149a23932f9301652e57661cd9e42b0df18f12be21159b7255f92b/lxml-6.1.3-cp315-cp315-manylinux_2_31_armv7l.whl", hash = "sha256:dc3a44689eea43eab836e5c98a8ab015dc2419987d1ea6eafc7c590cdff86bed", size = 4704543, upload-time = "2026-09-02T14:50:44.634Z" }, + { url = "https://files.pythonhosted.org/packages/76/07/9290329cd188c62e22021f79df04ee0cc33d9a93b0d38bd65ccd452ad9d0/lxml-6.1.3-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:209c3ccbfe35a04ac6d24f0611f9d1cbf8025d49991b14acd935236234d6c156", size = 5261298, upload-time = "2026-09-02T14:50:47.301Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0c/aba78bd3401cd99b73a0aed8e2b9b43e14be94fab3603d4bbc8a62365f2a/lxml-6.1.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2f5b2a2b9811b853b39bfa41367c6d78747b8e3e80e07fc5a24aae295c1a4d7d", size = 5090453, upload-time = "2026-09-02T14:50:49.952Z" }, + { url = "https://files.pythonhosted.org/packages/8d/dc/fa4426c3355aa0216cbeb3911495b5f65a26e0df85859a89928fe28f0396/lxml-6.1.3-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:6a406d0b3cb207b0fa460ed4dc93e866f44f105da0169361cb18ff998a44c7f0", size = 4744709, upload-time = "2026-09-02T14:50:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/be/2b/224fe7918658ab7c532ac2412f3c1eb28f71e6364fb07566262d0cc6a7b6/lxml-6.1.3-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:53258656846f5c48996b882fb4b135885e088a3ad3d96b4bc0530f95124d1f69", size = 5685802, upload-time = "2026-09-02T14:50:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/21/44/7d480819b9adcae5f84dd8ac529132c6b7a578544398225cd20321adcd91/lxml-6.1.3-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:aa633613ff907ea91b9b0489a1f0da1b8725d8c6ccec6b77e8a1c9c235044bb0", size = 5249019, upload-time = "2026-09-02T14:50:57.985Z" }, + { url = "https://files.pythonhosted.org/packages/72/83/385a267ea1b6b283f2249dd827ef360a295e9db14e13ef4665a120c60d64/lxml-6.1.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:90f709b9accab6b2e4d14f5c8718203877a0486bcb3afd74d8b539ecd1e961d4", size = 5271886, upload-time = "2026-09-02T14:51:01.667Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0d/f967b0eb172ae876855a402d6d9b11fa86e3e0c89ca9bbfeadf7ffbfa719/lxml-6.1.3-cp315-cp315-win32.whl", hash = "sha256:b4fc6b03b9d9d90557274f571ab30e7fbbfc527955536935d96f98b6817a86e4", size = 3662894, upload-time = "2026-09-02T14:51:45.173Z" }, + { url = "https://files.pythonhosted.org/packages/f4/48/d8a8c4160a29e663109ad520bac2deb37fcd014756d024561e8bc3e611ec/lxml-6.1.3-cp315-cp315-win_amd64.whl", hash = "sha256:33cadd956b667997e4de1635fce9541f2e8ede2038fcde8cf55aa14d571d1bad", size = 4074626, upload-time = "2026-09-02T14:51:47.77Z" }, + { url = "https://files.pythonhosted.org/packages/25/20/3e1395d34d19f9254625d0b567b81cf70d37d3417be074f4d63b94a2be3c/lxml-6.1.3-cp315-cp315-win_arm64.whl", hash = "sha256:8a330c0ee5fa318c7b5cbbaad882baeca3f570357e7eb25ab34bf31008150758", size = 3749495, upload-time = "2026-09-02T14:51:50.663Z" }, + { url = "https://files.pythonhosted.org/packages/8f/c6/7465ffd9c43883526a382df6fa4846c9d8d419214f7effbf65270e795471/lxml-6.1.3-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:0bf5a3e397df2ec4258eb5eea4c1ac6cf013ca1abd04a176903bff20a70021fe", size = 8857677, upload-time = "2026-09-02T14:51:05.109Z" }, + { url = "https://files.pythonhosted.org/packages/ed/eb/1f3a917e299df43c8162c3e6f64fc2cea3bcf277910f35bff5b8e5d39901/lxml-6.1.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:13d22c0d57355366b393936acf6b98a5e0edeadddd3fccbc6a846c50a76b8741", size = 4754522, upload-time = "2026-09-02T14:51:08.137Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f9/f81b4bdb6efb7a596be29603d8758154d00a5f545db9f3cef9d9041c8f64/lxml-6.1.3-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cad7617727a96d189bd6f979d0fadf765198c7934e85f4edaba9bf3ad919a300", size = 5033744, upload-time = "2026-09-02T14:51:10.633Z" }, + { url = "https://files.pythonhosted.org/packages/c8/0f/26d9bfaacb319c86e0eca8a1a0bf1130d36a7afbd318883e23caea63763d/lxml-6.1.3-cp315-cp315t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cae82b5ca24b0c2beedb269f6e2a96f466acd926879ab00ae19f1a65cbf9ffb0", size = 5615269, upload-time = "2026-09-02T14:51:13.357Z" }, + { url = "https://files.pythonhosted.org/packages/5d/90/73675f3f4141350ed65d6fec533b107d4e802c5caa340cf111771edd86e0/lxml-6.1.3-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:69cafd61aea04ebb3502c93c2aaa568b12931ca0802231e0b5de76bf8b6e74bd", size = 5236280, upload-time = "2026-09-02T14:51:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/fd/be/ed260767e7977de463a0f91f3f4fffcab85c0a2a024a21ffe1fa442c2c79/lxml-6.1.3-cp315-cp315t-manylinux_2_31_armv7l.whl", hash = "sha256:dc205732d593118cf701d986f40e9de7801bb2e371cb189ddbda9b7348f4d97e", size = 4650718, upload-time = "2026-09-02T14:51:19.102Z" }, + { url = "https://files.pythonhosted.org/packages/d0/fd/e9839d03b1e767f2725cf7d7d81b80d5f3f9fdc10ad8827e2479311b046e/lxml-6.1.3-cp315-cp315t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88e719b9437f148f7e1465df845c758dd1598618cbea3a2fd1e61a715542f2b2", size = 5243376, upload-time = "2026-09-02T14:51:21.606Z" }, + { url = "https://files.pythonhosted.org/packages/34/a5/4606e347e2788c301f677004aa83e28d24da9fe663a24380122af57be6fc/lxml-6.1.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:40983eabefd13da003e68170928c7acc011f0d095eefce5871a3c71c9385fb9a", size = 5092340, upload-time = "2026-09-02T14:51:24.21Z" }, + { url = "https://files.pythonhosted.org/packages/ea/99/3314a8661cdf30f493c55a87db283961dfaae08451976a2ca418958e1804/lxml-6.1.3-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:fad67b12ffe0f71e02b4932b04883cbc76a9072bbd30731409d3523cf058b011", size = 4758768, upload-time = "2026-09-02T14:51:26.813Z" }, + { url = "https://files.pythonhosted.org/packages/30/58/3bdc577f78ea8b7d72d39a84506f7001d5b28728f43e5b84891e3b7d9a4a/lxml-6.1.3-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6cd11e7550d89e551a87dcec30f04b1fca32e86b68708aa01a4daa455d8605e5", size = 5649546, upload-time = "2026-09-02T14:51:29.453Z" }, + { url = "https://files.pythonhosted.org/packages/6a/e4/652633de1a2395949ebb7a8fc7d089aba12a2b45f0fefbc9d29e3e3ab3cf/lxml-6.1.3-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ca0ec532ad2f5ba1e5ec120ac157769c57f01855b3d8bf37213f5d88abd9ba0a", size = 5234874, upload-time = "2026-09-02T14:51:32.262Z" }, + { url = "https://files.pythonhosted.org/packages/65/a6/c4581d171de30449304b4859bbd3607e9b40da13c0f88b68e6097c8d785e/lxml-6.1.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e99e09ab7741f1281e2677f4c0058c7f5267d182530b09c87e4f6aa26adf3887", size = 5260043, upload-time = "2026-09-02T14:51:34.841Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/ed6ee6186a89e69ca4ea9658b2a278f46a5efe8b5d4db56c7197f18653fe/lxml-6.1.3-cp315-cp315t-win32.whl", hash = "sha256:ace1d2c83b2bd24db5940600541140e87a325e119cb32d5fa9ad720d7e76648e", size = 3901093, upload-time = "2026-09-02T14:51:37.234Z" }, + { url = "https://files.pythonhosted.org/packages/67/9d/11d10257a4a048d04195d638bb61f0246ce2448eb05f682bcbab25a257a8/lxml-6.1.3-cp315-cp315t-win_amd64.whl", hash = "sha256:b49638355ea3bebba70da783ccbc630fd72afa16bc46c54474bfa1f9a915bbc6", size = 4395446, upload-time = "2026-09-02T14:51:39.884Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b7/44edd7de434181c582892e68d1ffe6775ca403ce14aea07cb5a218a936cf/lxml-6.1.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5a721a98c649855963811b59b55755b30566e7f7fc40bdc9803d66dee9f811cf", size = 3822836, upload-time = "2026-09-02T14:51:42.471Z" }, +] + +[[package]] +name = "magika" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "numpy" }, + { name = "onnxruntime" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/b6/8fdd991142ad3e037179a494b153f463024e5a211ef3ad948b955c26b4de/magika-0.6.2.tar.gz", hash = "sha256:37eb6ae8020f6e68f231bc06052c0a0cbe8e6fa27492db345e8dc867dbceb067", size = 3036634, upload-time = "2025-05-02T14:54:18.88Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/07/4f7748f34279f2852068256992377474f9700b6fbad6735d6be58605178f/magika-0.6.2-py3-none-any.whl", hash = "sha256:5ef72fbc07723029b3684ef81454bc224ac5f60986aa0fc5a28f4456eebcb5b2", size = 2967609, upload-time = "2025-05-02T14:54:09.696Z" }, + { url = "https://files.pythonhosted.org/packages/64/6d/0783af677e601d8a42258f0fbc47663abf435f927e58a8d2928296743099/magika-0.6.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9109309328a1553886c8ff36c2ee9a5e9cfd36893ad81b65bf61a57debdd9d0e", size = 12404787, upload-time = "2025-05-02T14:54:16.963Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ad/42e39748ddc4bbe55c2dc1093ce29079c04d096ac0d844f8ae66178bc3ed/magika-0.6.2-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:57cd1d64897634d15de552bd6b3ae9c6ff6ead9c60d384dc46497c08288e4559", size = 15091089, upload-time = "2025-05-02T14:54:11.59Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1f/28e412d0ccedc068fbccdae6a6233faaa97ec3e5e2ffd242e49655b10064/magika-0.6.2-py3-none-win_amd64.whl", hash = "sha256:711f427a633e0182737dcc2074748004842f870643585813503ff2553b973b9f", size = 12385740, upload-time = "2025-05-02T14:54:14.096Z" }, +] + [[package]] name = "mako" version = "1.3.12" @@ -2044,6 +2209,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, ] +[[package]] +name = "mammoth" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cobble" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/3c/a58418d2af00f2da60d4a51e18cd0311307b72d48d2fffec36a97b4a5e44/mammoth-1.11.0.tar.gz", hash = "sha256:a0f59e442f34d5b6447f4b0999306cbf3e67aaabfa8cb516f878fb1456744637", size = 53142, upload-time = "2025-09-19T10:35:20.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/54/2e39566a131b13f6d8d193f974cb6a34e81bb7cc2fa6f7e03de067b36588/mammoth-1.11.0-py2.py3-none-any.whl", hash = "sha256:c077ab0d450bd7c0c6ecd529a23bf7e0fa8190c929e28998308ff4eada3f063b", size = 54752, upload-time = "2025-09-19T10:35:18.699Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -2056,6 +2233,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, ] +[[package]] +name = "markdownify" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/ab/d1297139c0e2ceb151ae564c8c4f57ac0155d8f1f8b4cbd5d6523c82ea36/markdownify-1.2.3.tar.gz", hash = "sha256:1a176f05522c8a2cb1dd3ab9d307dcdadbed5c26ae717855bfc42b3b6d38d937", size = 18852, upload-time = "2026-06-30T20:27:39.06Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/10/fa543d484e8b1199243fe20eedd02cc5af050edebce98a7293a5773df592/markdownify-1.2.3-py3-none-any.whl", hash = "sha256:a189a0bedfd14009030fde5f85bb6f77c56897cb839b5c25315dd7d4e3e290ba", size = 15732, upload-time = "2026-06-30T20:27:38.094Z" }, +] + +[[package]] +name = "markitdown" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "charset-normalizer" }, + { name = "defusedxml" }, + { name = "magika" }, + { name = "markdownify" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/93/e8a4af0c47551beb6383e226e840cbc811a577b8096eb385251b3fcc8f62/markitdown-0.1.7.tar.gz", hash = "sha256:4d1f3c69cd43b82288fdc3653686d759dcf355ee7c681aa6a855aed98a1e4f44", size = 51767, upload-time = "2026-07-29T18:20:31.496Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/16/51d269a754d690ec31d3faa0686c8c14ac955dbc0580c358f256ba3391ec/markitdown-0.1.7-py3-none-any.whl", hash = "sha256:4eca912c87c6aa6897284a7f4bf6769a23bccf8544530f5d8b175fbe3797c916", size = 71093, upload-time = "2026-07-29T18:20:30.226Z" }, +] + +[package.optional-dependencies] +docx = [ + { name = "lxml" }, + { name = "mammoth" }, +] +pptx = [ + { name = "python-pptx" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -3497,6 +3713,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] +[[package]] +name = "python-pptx" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "pillow" }, + { name = "typing-extensions" }, + { name = "xlsxwriter" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/a9/0c0db8d37b2b8a645666f7fd8accea4c6224e013c42b1d5c17c93590cd06/python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095", size = 10109297, upload-time = "2024-08-07T17:33:37.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788, upload-time = "2024-08-07T17:33:28.192Z" }, +] + [[package]] name = "pytz" version = "2026.2" @@ -4030,6 +4261,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "soupsieve" +version = "2.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/99/a6ca3beb3ccacb41fb3321d8a60e5566f9e6467601ef8eba6a17e1b89778/soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74", size = 122445, upload-time = "2026-08-07T00:57:24.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823", size = 37370, upload-time = "2026-08-07T00:57:23.524Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.51" @@ -4629,6 +4869,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, ] +[[package]] +name = "xlsxwriter" +version = "3.2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940, upload-time = "2025-09-16T00:16:21.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315, upload-time = "2025-09-16T00:16:20.108Z" }, +] + [[package]] name = "yarl" version = "1.24.2" From 58b425a6b16ce93da0da89512242a8d2878429b1 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 8 Sep 2026 10:17:50 -0500 Subject: [PATCH 02/10] fix(core): address review findings for document ingestion - `bm import document` pins local API routing. The runtime reads the source and writes the sidecar in the local project directory, so a cloud-configured project must not route its index calls to a server that cannot see those files. - The local writer verifies an existing run note against the accepted sidecar checksum and rewrites it when the two disagree, so overlapping imports of the same source converge instead of freezing a mismatch. - CSV cells escape backslashes before pipes so `a\|b` survives the Markdown table round trip. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01VPRRyMREwCHSLnY3TCBr7s Signed-off-by: phernandez --- .../cli/commands/import_document.py | 9 ++++- .../document_ingestion/csv_extractor.py | 4 +- .../document_ingestion/local_runtime.py | 28 +++++++++++-- tests/cli/test_import_document.py | 12 +++++- .../document_ingestion/test_csv_extractor.py | 7 ++++ .../document_ingestion/test_local_runtime.py | 40 +++++++++++++++++++ 6 files changed, 93 insertions(+), 7 deletions(-) diff --git a/src/basic_memory/cli/commands/import_document.py b/src/basic_memory/cli/commands/import_document.py index ce6d8ca2d..385baa4fe 100644 --- a/src/basic_memory/cli/commands/import_document.py +++ b/src/basic_memory/cli/commands/import_document.py @@ -13,6 +13,7 @@ from basic_memory.cli.app import import_app from basic_memory.cli.commands.command_utils import run_with_cleanup +from basic_memory.cli.commands.routing import force_routing from basic_memory.config import ConfigManager if TYPE_CHECKING: @@ -84,7 +85,13 @@ def document( ) -> None: """Write ``.md`` next to the source and a run note under document-ingestion-runs/.""" try: - project_name, result = run_with_cleanup(import_document(path, project)) + # Trigger: the project may be configured for cloud routing. + # Why: this runtime reads the source and writes the sidecar in the local + # project directory, so the API it indexes through must be the local + # one; a cloud client would be asked to index files it cannot see. + # Outcome: local ASGI routing for the whole command, whatever the project mode. + with force_routing(local=True): + project_name, result = run_with_cleanup(import_document(path, project)) except typer.BadParameter as error: typer.echo(f"Error: {error}", err=True) raise typer.Exit(1) diff --git a/src/basic_memory/document_ingestion/csv_extractor.py b/src/basic_memory/document_ingestion/csv_extractor.py index 1b0c7ae0d..568a06b3a 100644 --- a/src/basic_memory/document_ingestion/csv_extractor.py +++ b/src/basic_memory/document_ingestion/csv_extractor.py @@ -142,5 +142,7 @@ def _table_row(cells: Sequence[str], width: int) -> str: # Ragged rows are common in hand-edited exports: pad short rows and drop # cells past the header width so every line stays a valid table row. padded = [*cells[:width], *([""] * (width - len(cells)))] - escaped = (cell.replace("\n", " ").replace("|", "\\|") for cell in padded) + # Backslashes first: a literal `\` before a `|` would otherwise turn the pipe + # escape into an escaped backslash followed by a live cell separator. + escaped = (cell.replace("\n", " ").replace("\\", "\\\\").replace("|", "\\|") for cell in padded) return "| " + " | ".join(escaped) + " |" diff --git a/src/basic_memory/document_ingestion/local_runtime.py b/src/basic_memory/document_ingestion/local_runtime.py index a1bc22396..4ff53b8f7 100644 --- a/src/basic_memory/document_ingestion/local_runtime.py +++ b/src/basic_memory/document_ingestion/local_runtime.py @@ -20,7 +20,6 @@ from uuid import UUID from basic_memory.document_ingestion.csv_extractor import CSV_MEDIA_TYPE, CsvExtractor -from basic_memory.file_utils import ParseError from basic_memory.document_ingestion.markitdown_extractor import markitdown_extractors from basic_memory.document_ingestion.pdf_inspector import PdfInspector from basic_memory.document_ingestion.raw_document import ( @@ -36,7 +35,12 @@ canonical_db_checksum, raw_document_matches, ) -from basic_memory.schemas.document import DocumentIngestionStage, parse_document_markdown +from basic_memory.file_utils import ParseError +from basic_memory.schemas.document import ( + DocumentIngestionStage, + parse_document_ingestion_run_markdown, + parse_document_markdown, +) from basic_memory.schemas.v2.entity import EntityResolveResponse, EntityResponseV2 from basic_memory.services.file_service import FileService @@ -206,10 +210,26 @@ async def accept_run_note( *, raw_checksum: str, ) -> bool: - """Write the run note for this deterministic run id unless it already exists.""" + """Write the run note for this run id unless it already names the accepted sidecar bytes.""" path = artifacts.run_file_path if await file_service.exists(path): - return False + try: + existing = parse_document_ingestion_run_markdown( + await file_service.read_file_content(path) + ) + except (ParseError, ValueError) as error: + raise DocumentSidecarConflictError( + f"{path} exists and is not a generated ingestion run note" + ) from error + output = existing.frontmatter.output + recorded_checksum = output.raw.checksum if output and output.raw else None + if recorded_checksum == raw_checksum: + return False + # Trigger: the run note names different sidecar bytes than the note on disk. + # Why: two imports of the same unchanged source can interleave (same run id, + # different extracted_at), leaving one run's sidecar beside the other + # run's note; reusing the stale note would freeze that mismatch. + # Outcome: rewrite the run note so provenance converges on this pass. markdown = build_raw_ingestion_run_markdown( artifacts, raw_checksum=raw_checksum, raw_created_at=datetime.now(tz=UTC) ) diff --git a/tests/cli/test_import_document.py b/tests/cli/test_import_document.py index e0cbc43d3..b06bbb7eb 100644 --- a/tests/cli/test_import_document.py +++ b/tests/cli/test_import_document.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os from contextlib import asynccontextmanager from pathlib import Path from types import SimpleNamespace @@ -42,7 +43,13 @@ @patch("basic_memory.cli.commands.import_document.import_document", new_callable=AsyncMock) def test_import_document_reports_the_written_notes(mock_import: AsyncMock) -> None: - mock_import.return_value = ("main", RESULT) + routing_seen: list[str | None] = [] + + async def record_routing(path: Path, project: str | None) -> tuple[str, RawDocumentWriteResult]: + routing_seen.append(os.environ.get("BASIC_MEMORY_FORCE_LOCAL")) + return ("main", RESULT) + + mock_import.side_effect = record_routing result = runner.invoke(cli_app, ["import", "document", "docs/plan.docx", "--project", "main"]) @@ -50,6 +57,9 @@ def test_import_document_reports_the_written_notes(mock_import: AsyncMock) -> No assert "docs/plan.docx.md (created)" in result.output assert "already recorded" in result.output mock_import.assert_awaited_once_with(Path("docs/plan.docx"), "main") + # The local runtime reads and writes the project directory, so the command + # must pin local API routing even for a cloud-configured project. + assert routing_seen == ["true"] @patch("basic_memory.cli.commands.import_document.import_document", new_callable=AsyncMock) diff --git a/tests/document_ingestion/test_csv_extractor.py b/tests/document_ingestion/test_csv_extractor.py index 4b94acd4b..c820dbf35 100644 --- a/tests/document_ingestion/test_csv_extractor.py +++ b/tests/document_ingestion/test_csv_extractor.py @@ -37,6 +37,13 @@ def test_render_csv_preview_escapes_cells_and_reports_the_row_count() -> None: ) +def test_render_csv_preview_escapes_backslashes_before_pipes() -> None: + preview = render_csv_preview(b"path\nC:\\dir\\file\na\\|b\n", max_rows=200) + + assert "| C:\\\\dir\\\\file |" in preview.markdown + assert "| a\\\\\\|b |" in preview.markdown + + def test_render_csv_preview_truncates_to_max_rows_but_counts_every_row() -> None: content = b"n\n" + b"".join(f"{index}\n".encode() for index in range(10)) diff --git a/tests/document_ingestion/test_local_runtime.py b/tests/document_ingestion/test_local_runtime.py index f059e6f98..4ab6d7d9f 100644 --- a/tests/document_ingestion/test_local_runtime.py +++ b/tests/document_ingestion/test_local_runtime.py @@ -27,6 +27,7 @@ ExtractedDocument, RawDocumentArtifacts, build_raw_document_artifacts_from_extracted, + build_raw_ingestion_run_markdown, ) from basic_memory.schemas.document import ( DocumentExtractionStatus, @@ -274,6 +275,45 @@ async def test_writer_rebuilds_the_raw_projection_when_the_source_changed( assert knowledge.indexed.count(second.document_file_path) == 2 +@pytest.mark.asyncio +async def test_writer_rewrites_a_run_note_that_names_other_sidecar_bytes( + file_service: FileService, +) -> None: + knowledge = knowledge_api() + writer = LocalRawDocumentWriter(file_service, knowledge) + built = artifacts() + first = await writer.write(built) + # Simulate an overlapping import that replaced the sidecar after this run + # note was written: the note now names bytes that are no longer on disk. + stale_note = build_raw_ingestion_run_markdown( + built, raw_checksum="sha256:" + "d" * 64, raw_created_at=NOW + ) + await file_service.write_file(built.run_file_path, stale_note) + + second = await writer.write(built) + + assert second.document_created is False + assert second.run_created is True + run_note = parse_document_ingestion_run_markdown( + (file_service.base_path / built.run_file_path).read_text(encoding="utf-8") + ) + assert run_note.frontmatter.output is not None + assert run_note.frontmatter.output.raw is not None + assert run_note.frontmatter.output.raw.checksum == first.document_db_checksum + assert knowledge.indexed.count(built.run_file_path) == 2 + + +@pytest.mark.asyncio +async def test_writer_refuses_a_hand_written_run_note(file_service: FileService) -> None: + knowledge = knowledge_api() + writer = LocalRawDocumentWriter(file_service, knowledge) + built = artifacts() + await file_service.write_file(built.run_file_path, "# Not a run note\n") + + with pytest.raises(DocumentSidecarConflictError, match="not a generated ingestion run note"): + await writer.write(built) + + @pytest.mark.asyncio async def test_writer_refuses_a_hand_written_sidecar(file_service: FileService) -> None: built = artifacts() From ebd87bfc3c9e1739906572d81c73ad27a0c2b852 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 8 Sep 2026 10:34:13 -0500 Subject: [PATCH 03/10] fix(core): guard raw sidecars before rebuilding them A raw sidecar whose source changed is rebuilt only while it is still the projection an earlier run wrote. The writer looks up that run's note, compares the recorded checksum with the note on disk (allowing only the permalink the indexer adds), and refuses with a conflict when the bytes differ, when the run note is missing, or when the note changes between the check and the replacing write. Note content is canonical; a raw note a person has annotated is never replaced silently. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01VPRRyMREwCHSLnY3TCBr7s Signed-off-by: phernandez --- .../document_ingestion/local_runtime.py | 56 +++++++++- .../document_ingestion/test_local_runtime.py | 102 ++++++++++++++++++ 2 files changed, 155 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/document_ingestion/local_runtime.py b/src/basic_memory/document_ingestion/local_runtime.py index 4ff53b8f7..dbd05facd 100644 --- a/src/basic_memory/document_ingestion/local_runtime.py +++ b/src/basic_memory/document_ingestion/local_runtime.py @@ -38,6 +38,10 @@ from basic_memory.file_utils import ParseError from basic_memory.schemas.document import ( DocumentIngestionStage, + DocumentMarkdownV1, + assemble_document_markdown, + derive_document_ingestion_run_path, + document_markdown_checksum, parse_document_ingestion_run_markdown, parse_document_markdown, ) @@ -179,6 +183,7 @@ async def accept_document_note( path = artifacts.document_file_path if await file_service.exists(path): existing_markdown = await file_service.read_file_content(path) + existing_checksum = canonical_db_checksum(await file_service.compute_checksum(path)) try: existing = parse_document_markdown(existing_markdown) except (ParseError, ValueError) as error: @@ -194,15 +199,60 @@ async def accept_document_note( f"{path} has been enriched past the raw stage; refusing to overwrite it" ) if raw_document_matches(existing, artifacts): - checksum = await file_service.compute_checksum(path) - return canonical_db_checksum(checksum), False + return existing_checksum, False # Same source path, different bytes or engine: the raw projection is - # derived state and is rebuilt from the new run. + # rebuilt from the new run, but only while it is still the projection an + # earlier run wrote. Note content is canonical, and a raw note a person + # has annotated must not be replaced silently. + await require_untouched_raw_projection(file_service, existing, path=path) + # Compare-and-swap: the note must still be the bytes judged untouched above. + if canonical_db_checksum(await file_service.compute_checksum(path)) != existing_checksum: + raise DocumentSidecarConflictError( + f"{path} changed while it was being replaced; re-run the import" + ) checksum = await file_service.write_file(path, artifacts.document_markdown) await knowledge.index_file(path) return canonical_db_checksum(checksum), True +async def require_untouched_raw_projection( + file_service: FileService, + existing: DocumentMarkdownV1, + *, + path: str, +) -> None: + """Fail unless the raw sidecar on disk is still what its own run note recorded. + + The indexer adds a ``permalink`` to a freshly written note; that is the one + change a pristine projection is allowed to carry. Anything else means a person + edited the note, and the rebuild must stop instead of discarding their words. + """ + run_id = existing.frontmatter.ingestion.run_id + run_path = derive_document_ingestion_run_path(run_id) + if not await file_service.exists(run_path): + raise DocumentSidecarConflictError( + f"{path} was written by run {run_id} but that run note is missing; " + "move the note aside to rebuild it" + ) + try: + run = parse_document_ingestion_run_markdown(await file_service.read_file_content(run_path)) + except (ParseError, ValueError) as error: + raise DocumentSidecarConflictError( + f"{run_path} exists and is not a generated ingestion run note" + ) from error + output = run.frontmatter.output + recorded_checksum = output.raw.checksum if output and output.raw else None + pristine_frontmatter = existing.frontmatter.model_copy(update={"permalink": None}) + pristine = assemble_document_markdown( + existing.model_copy(update={"frontmatter": pristine_frontmatter}) + ) + if recorded_checksum != document_markdown_checksum(pristine): + raise DocumentSidecarConflictError( + f"{path} has been edited since run {run_id} wrote it; " + "move it aside or finish enriching it before importing the source again" + ) + + async def accept_run_note( file_service: FileService, knowledge: DocumentKnowledgeApi, diff --git a/tests/document_ingestion/test_local_runtime.py b/tests/document_ingestion/test_local_runtime.py index 4ab6d7d9f..b318f0c45 100644 --- a/tests/document_ingestion/test_local_runtime.py +++ b/tests/document_ingestion/test_local_runtime.py @@ -275,6 +275,108 @@ async def test_writer_rebuilds_the_raw_projection_when_the_source_changed( assert knowledge.indexed.count(second.document_file_path) == 2 +def with_permalink(markdown: str, permalink: str) -> str: + """Reproduce the indexer adding a permalink to a freshly written sidecar.""" + document = parse_document_markdown(markdown) + frontmatter = document.frontmatter.model_copy(update={"permalink": permalink}) + return assemble_document_markdown(document.model_copy(update={"frontmatter": frontmatter})) + + +@pytest.mark.asyncio +async def test_writer_rebuilds_an_untouched_sidecar_that_the_indexer_annotated( + file_service: FileService, +) -> None: + writer = LocalRawDocumentWriter(file_service, knowledge_api()) + first = artifacts(checksum_char="a") + await writer.write(first) + sidecar = file_service.base_path / first.document_file_path + sidecar.write_text( + with_permalink(sidecar.read_text(encoding="utf-8"), "main/data/riders.csv"), + encoding="utf-8", + ) + + second = await writer.write(artifacts(checksum_char="b")) + + assert second.document_created is True + rebuilt = parse_document_markdown(sidecar.read_text(encoding="utf-8")) + assert rebuilt.frontmatter.source.checksum == "sha256:" + "b" * 64 + + +@pytest.mark.asyncio +async def test_writer_refuses_to_replace_an_edited_raw_sidecar(file_service: FileService) -> None: + writer = LocalRawDocumentWriter(file_service, knowledge_api()) + first = artifacts(checksum_char="a") + await writer.write(first) + sidecar = file_service.base_path / first.document_file_path + sidecar.write_text( + sidecar.read_text(encoding="utf-8") + "\nMy annotation about row three.\n", + encoding="utf-8", + ) + + with pytest.raises(DocumentSidecarConflictError, match="edited since"): + await writer.write(artifacts(checksum_char="b")) + + assert "My annotation about row three." in sidecar.read_text(encoding="utf-8") + + +@pytest.mark.asyncio +async def test_writer_refuses_to_replace_a_raw_sidecar_without_its_run_note( + file_service: FileService, +) -> None: + writer = LocalRawDocumentWriter(file_service, knowledge_api()) + first = artifacts(checksum_char="a") + await writer.write(first) + (file_service.base_path / first.run_file_path).unlink() + + with pytest.raises(DocumentSidecarConflictError, match="run note is missing"): + await writer.write(artifacts(checksum_char="b")) + + +@pytest.mark.asyncio +async def test_writer_refuses_to_replace_a_raw_sidecar_whose_run_note_is_not_generated( + file_service: FileService, +) -> None: + writer = LocalRawDocumentWriter(file_service, knowledge_api()) + first = artifacts(checksum_char="a") + await writer.write(first) + (file_service.base_path / first.run_file_path).write_text( + "# Someone's note\n", encoding="utf-8" + ) + + with pytest.raises(DocumentSidecarConflictError, match="not a generated ingestion run note"): + await writer.write(artifacts(checksum_char="b")) + + +@pytest.mark.asyncio +async def test_writer_refuses_when_the_sidecar_changes_before_the_replacing_write( + file_service: FileService, monkeypatch: pytest.MonkeyPatch +) -> None: + writer = LocalRawDocumentWriter(file_service, knowledge_api()) + first = artifacts(checksum_char="a") + await writer.write(first) + sidecar = file_service.base_path / first.document_file_path + real_compute_checksum = file_service.compute_checksum + calls = 0 + + async def edit_between_checks(path: Path | str) -> str: + # The second checksum read is the compare-and-swap guard: mutate the file + # right before it so the guard sees bytes that differ from the decision. + nonlocal calls + calls += 1 + if calls == 2: + sidecar.write_text( + sidecar.read_text(encoding="utf-8") + "\nLate edit.\n", encoding="utf-8" + ) + return await real_compute_checksum(path) + + monkeypatch.setattr(file_service, "compute_checksum", edit_between_checks) + + with pytest.raises(DocumentSidecarConflictError, match="changed while"): + await writer.write(artifacts(checksum_char="b")) + + assert "Late edit." in sidecar.read_text(encoding="utf-8") + + @pytest.mark.asyncio async def test_writer_rewrites_a_run_note_that_names_other_sidecar_bytes( file_service: FileService, From f904444c75d5fe5a9a05b329cce9758d2fc2f18a Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 8 Sep 2026 11:22:21 -0500 Subject: [PATCH 04/10] fix(core): record raw projection checksums independent of indexer edits Every checksum the local document writer records or compares is now the checksum of the raw projection as its run wrote it: the sidecar re-assembled without the permalink the indexer appends. An unchanged re-import therefore leaves the run note alone, and a later source change still recognizes the annotated sidecar as untouched and rebuilds it. This also makes the comparison independent of the platform newline convention the atomic writer uses, which is what failed the Windows unit job. CSV cells flatten CRLF and lone CR as well as LF, so a quoted multiline field cannot split a table row once note assembly normalizes carriage returns. The reader raises the process-wide field size limit to the configured `max_field_bytes` (never lowering it) and reports parser failures as `CsvExtractionError` instead of a bare `_csv.Error`. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01VPRRyMREwCHSLnY3TCBr7s Signed-off-by: phernandez --- .../document_ingestion/csv_extractor.py | 54 ++++++++++++++----- .../document_ingestion/local_runtime.py | 29 ++++++---- .../document_ingestion/test_csv_extractor.py | 51 ++++++++++++++++++ .../document_ingestion/test_local_runtime.py | 28 ++++++++++ 4 files changed, 139 insertions(+), 23 deletions(-) diff --git a/src/basic_memory/document_ingestion/csv_extractor.py b/src/basic_memory/document_ingestion/csv_extractor.py index 568a06b3a..e3e71c7b5 100644 --- a/src/basic_memory/document_ingestion/csv_extractor.py +++ b/src/basic_memory/document_ingestion/csv_extractor.py @@ -16,6 +16,7 @@ import asyncio import csv import io +import re import time from collections.abc import Sequence from dataclasses import dataclass, field @@ -44,6 +45,13 @@ class CsvLimits(BaseModel): max_source_bytes: int = Field(default=25 * 1024 * 1024, gt=0) max_rows: int = Field(default=200, gt=0) max_output_bytes: int = Field(default=5 * 1024 * 1024, gt=0) + # The stdlib parser rejects any field over its own limit (128 KiB by default), + # which is far below the source ceiling; exported text or JSON columns hit it. + max_field_bytes: int = Field(default=25 * 1024 * 1024, gt=0) + + +_DEFAULT_LIMITS = CsvLimits() +_LINE_BREAKS = re.compile(r"\r\n|\r|\n") class CsvExtractionError(RuntimeError): @@ -78,7 +86,10 @@ async def extract(self, content: bytes, *, file_name: str) -> ExtractedDocument: raise CsvSourceTooLargeError("CSV source exceeds the configured extraction byte limit") started = time.perf_counter() preview = await asyncio.to_thread( - render_csv_preview, content, max_rows=self.limits.max_rows + render_csv_preview, + content, + max_rows=self.limits.max_rows, + max_field_bytes=self.limits.max_field_bytes, ) if len(preview.markdown.encode("utf-8")) > self.limits.max_output_bytes: raise CsvExtractionError("CSV preview exceeds the configured output byte limit") @@ -105,24 +116,37 @@ async def extract(self, content: bytes, *, file_name: str) -> ExtractedDocument: ) -def render_csv_preview(content: bytes, *, max_rows: int) -> CsvPreview: +def render_csv_preview( + content: bytes, + *, + max_rows: int, + max_field_bytes: int = _DEFAULT_LIMITS.max_field_bytes, +) -> CsvPreview: """Render the header, the first ``max_rows`` rows, and the total row count.""" try: text = content.decode("utf-8-sig") except UnicodeDecodeError as error: raise CsvDecodeError("CSV source is not UTF-8 encoded") from error - reader = csv.reader(io.StringIO(text, newline="")) - header = next(reader, None) - if header is None: - return CsvPreview(markdown="_Empty CSV file._\n", row_count=0, shown_rows=0) + # ``csv.field_size_limit`` is process-wide and has no per-reader form. Only + # ever raise it: a higher ceiling cannot break another caller, and a source + # already bounded by ``max_source_bytes`` must not fail on one long column. + if csv.field_size_limit() < max_field_bytes: + csv.field_size_limit(max_field_bytes) + reader = csv.reader(io.StringIO(text, newline="")) shown: list[Sequence[str]] = [] row_count = 0 - for row in reader: - row_count += 1 - if len(shown) < max_rows: - shown.append(row) + try: + header = next(reader, None) + if header is None: + return CsvPreview(markdown="_Empty CSV file._\n", row_count=0, shown_rows=0) + for row in reader: + row_count += 1 + if len(shown) < max_rows: + shown.append(row) + except csv.Error as error: + raise CsvExtractionError(f"CSV source could not be parsed: {error}") from error width = len(header) lines = [ @@ -142,7 +166,11 @@ def _table_row(cells: Sequence[str], width: int) -> str: # Ragged rows are common in hand-edited exports: pad short rows and drop # cells past the header width so every line stays a valid table row. padded = [*cells[:width], *([""] * (width - len(cells)))] - # Backslashes first: a literal `\` before a `|` would otherwise turn the pipe - # escape into an escaped backslash followed by a live cell separator. - escaped = (cell.replace("\n", " ").replace("\\", "\\\\").replace("|", "\\|") for cell in padded) + # Every line-break form is flattened, CRLF included: canonical note assembly + # turns a stray carriage return into a newline, which would split the row. + # Backslashes are escaped before pipes: a literal `\` before a `|` would + # otherwise turn the pipe escape into an escaped backslash and a live separator. + escaped = ( + _LINE_BREAKS.sub(" ", cell).replace("\\", "\\\\").replace("|", "\\|") for cell in padded + ) return "| " + " | ".join(escaped) + " |" diff --git a/src/basic_memory/document_ingestion/local_runtime.py b/src/basic_memory/document_ingestion/local_runtime.py index dbd05facd..4662cbebd 100644 --- a/src/basic_memory/document_ingestion/local_runtime.py +++ b/src/basic_memory/document_ingestion/local_runtime.py @@ -183,7 +183,7 @@ async def accept_document_note( path = artifacts.document_file_path if await file_service.exists(path): existing_markdown = await file_service.read_file_content(path) - existing_checksum = canonical_db_checksum(await file_service.compute_checksum(path)) + on_disk_checksum = canonical_db_checksum(await file_service.compute_checksum(path)) try: existing = parse_document_markdown(existing_markdown) except (ParseError, ValueError) as error: @@ -199,20 +199,33 @@ async def accept_document_note( f"{path} has been enriched past the raw stage; refusing to overwrite it" ) if raw_document_matches(existing, artifacts): - return existing_checksum, False + return raw_projection_checksum(existing), False # Same source path, different bytes or engine: the raw projection is # rebuilt from the new run, but only while it is still the projection an # earlier run wrote. Note content is canonical, and a raw note a person # has annotated must not be replaced silently. await require_untouched_raw_projection(file_service, existing, path=path) # Compare-and-swap: the note must still be the bytes judged untouched above. - if canonical_db_checksum(await file_service.compute_checksum(path)) != existing_checksum: + if canonical_db_checksum(await file_service.compute_checksum(path)) != on_disk_checksum: raise DocumentSidecarConflictError( f"{path} changed while it was being replaced; re-run the import" ) - checksum = await file_service.write_file(path, artifacts.document_markdown) + await file_service.write_file(path, artifacts.document_markdown) await knowledge.index_file(path) - return canonical_db_checksum(checksum), True + return document_markdown_checksum(artifacts.document_markdown), True + + +def raw_projection_checksum(document: DocumentMarkdownV1) -> str: + """Checksum of a raw sidecar as its run wrote it, before the indexer's permalink. + + Every checksum the local runtime records or compares is this one, so a note + the indexer has annotated still matches the run note that produced it. + """ + pristine_frontmatter = document.frontmatter.model_copy(update={"permalink": None}) + pristine = assemble_document_markdown( + document.model_copy(update={"frontmatter": pristine_frontmatter}) + ) + return document_markdown_checksum(pristine) async def require_untouched_raw_projection( @@ -242,11 +255,7 @@ async def require_untouched_raw_projection( ) from error output = run.frontmatter.output recorded_checksum = output.raw.checksum if output and output.raw else None - pristine_frontmatter = existing.frontmatter.model_copy(update={"permalink": None}) - pristine = assemble_document_markdown( - existing.model_copy(update={"frontmatter": pristine_frontmatter}) - ) - if recorded_checksum != document_markdown_checksum(pristine): + if recorded_checksum != raw_projection_checksum(existing): raise DocumentSidecarConflictError( f"{path} has been edited since run {run_id} wrote it; " "move it aside or finish enriching it before importing the source again" diff --git a/tests/document_ingestion/test_csv_extractor.py b/tests/document_ingestion/test_csv_extractor.py index c820dbf35..b638a57b9 100644 --- a/tests/document_ingestion/test_csv_extractor.py +++ b/tests/document_ingestion/test_csv_extractor.py @@ -2,6 +2,8 @@ from __future__ import annotations +import csv + import pytest from basic_memory import __version__ as basic_memory_version @@ -69,6 +71,55 @@ def test_render_csv_preview_flattens_multiline_cells() -> None: assert "| line one line two |" in preview.markdown +def test_render_csv_preview_flattens_crlf_and_lone_cr_inside_cells() -> None: + preview = render_csv_preview(b'note\r\n"line one\r\nline two\rline three"\r\n', max_rows=200) + + assert "| line one line two line three |" in preview.markdown + assert "\r" not in preview.markdown + + +def test_render_csv_preview_accepts_fields_larger_than_the_stdlib_default() -> None: + wide = b"x" * 200_000 + preview = render_csv_preview(b"blob\n" + wide + b"\n", max_rows=200) + + assert preview.row_count == 1 + assert wide.decode() in preview.markdown + + +def test_render_csv_preview_names_a_parser_failure(monkeypatch: pytest.MonkeyPatch) -> None: + # The lenient reader only fails on a field above the process-wide size limit, + # which a bounded source cannot produce under default limits; drive the + # translation directly. Real readers fail while iterating, not on construction. + class FailingReader: + def __iter__(self) -> FailingReader: + return self + + def __next__(self) -> list[str]: + raise csv.Error("field larger than field limit (131072)") + + monkeypatch.setattr(csv, "reader", lambda *args, **kwargs: FailingReader()) + + with pytest.raises(CsvExtractionError, match="could not be parsed: field larger"): + render_csv_preview(b"a,b\n1,2\n", max_rows=200) + + +def test_render_csv_preview_only_raises_the_field_size_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: list[int] = [] + + def record_limit(new_limit: int | None = None) -> int: + if new_limit is not None: + seen.append(new_limit) + return 10**9 + + monkeypatch.setattr(csv, "field_size_limit", record_limit) + + render_csv_preview(b"a\n1\n", max_rows=200, max_field_bytes=1024) + + assert seen == [] + + def test_render_csv_preview_names_an_empty_file() -> None: preview = render_csv_preview(b"", max_rows=200) diff --git a/tests/document_ingestion/test_local_runtime.py b/tests/document_ingestion/test_local_runtime.py index b318f0c45..7d741826e 100644 --- a/tests/document_ingestion/test_local_runtime.py +++ b/tests/document_ingestion/test_local_runtime.py @@ -302,6 +302,34 @@ async def test_writer_rebuilds_an_untouched_sidecar_that_the_indexer_annotated( assert rebuilt.frontmatter.source.checksum == "sha256:" + "b" * 64 +@pytest.mark.asyncio +async def test_unchanged_import_after_indexer_annotation_keeps_provenance_consistent( + file_service: FileService, +) -> None: + """The permalink the indexer adds must not leak into recorded checksums. + + Otherwise an unchanged re-import rewrites the run note with the annotated + checksum and a later source change is refused as an edit. + """ + knowledge = knowledge_api() + writer = LocalRawDocumentWriter(file_service, knowledge) + first = artifacts(checksum_char="a") + created = await writer.write(first) + sidecar = file_service.base_path / first.document_file_path + sidecar.write_bytes( + with_permalink(sidecar.read_text(encoding="utf-8"), "main/data/riders.csv").encode() + ) + + unchanged = await writer.write(artifacts(checksum_char="a")) + rebuilt = await writer.write(artifacts(checksum_char="b")) + + assert unchanged.document_created is False + assert unchanged.run_created is False + assert unchanged.document_db_checksum == created.document_db_checksum + assert rebuilt.document_created is True + assert knowledge.indexed.count(first.run_file_path) == 1 + + @pytest.mark.asyncio async def test_writer_refuses_to_replace_an_edited_raw_sidecar(file_service: FileService) -> None: writer = LocalRawDocumentWriter(file_service, knowledge_api()) From e2b702c1c2e9c4410e235aa7802c6f24354b5d4e Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 8 Sep 2026 11:25:40 -0500 Subject: [PATCH 05/10] fix(core): separate file and projection checksums for document writes Signed-off-by: phernandez --- .../document_ingestion/local_runtime.py | 37 ++++++++++++------- .../document_ingestion/test_local_runtime.py | 32 +++++++++++++--- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/basic_memory/document_ingestion/local_runtime.py b/src/basic_memory/document_ingestion/local_runtime.py index 4662cbebd..97b347a35 100644 --- a/src/basic_memory/document_ingestion/local_runtime.py +++ b/src/basic_memory/document_ingestion/local_runtime.py @@ -146,6 +146,15 @@ async def require_current(self, snapshot: DocumentSourceSnapshot) -> None: ) +@dataclass(frozen=True, slots=True) +class AcceptedDocumentNote: + """Keep physical file identity separate from normalized run provenance.""" + + file_checksum: str + projection_checksum: str + written: bool + + @dataclass(frozen=True, slots=True) class LocalRawDocumentWriter: """Write the sidecar and run notes into the project directory and index them.""" @@ -154,19 +163,17 @@ class LocalRawDocumentWriter: knowledge: DocumentKnowledgeApi async def write(self, artifacts: RawDocumentArtifacts) -> RawDocumentWriteResult: - document_checksum, document_created = await accept_document_note( - self.file_service, self.knowledge, artifacts - ) + document = await accept_document_note(self.file_service, self.knowledge, artifacts) run_created = await accept_run_note( - self.file_service, self.knowledge, artifacts, raw_checksum=document_checksum + self.file_service, self.knowledge, artifacts, raw_checksum=document.projection_checksum ) return RawDocumentWriteResult( document_external_id=artifacts.document_external_id, document_file_path=artifacts.document_file_path, - document_db_checksum=document_checksum, + document_db_checksum=document.file_checksum, run_id=artifacts.run_id, run_file_path=artifacts.run_file_path, - document_created=document_created, + document_created=document.written, run_created=run_created, ) @@ -175,10 +182,10 @@ async def accept_document_note( file_service: FileService, knowledge: DocumentKnowledgeApi, artifacts: RawDocumentArtifacts, -) -> tuple[str, bool]: +) -> AcceptedDocumentNote: """Write the sidecar note unless an identical raw projection already exists. - Returns the canonical checksum of the note on disk and whether it was written. + Return physical and normalized projection checksums with the write status. """ path = artifacts.document_file_path if await file_service.exists(path): @@ -199,7 +206,7 @@ async def accept_document_note( f"{path} has been enriched past the raw stage; refusing to overwrite it" ) if raw_document_matches(existing, artifacts): - return raw_projection_checksum(existing), False + return AcceptedDocumentNote(on_disk_checksum, raw_projection_checksum(existing), False) # Same source path, different bytes or engine: the raw projection is # rebuilt from the new run, but only while it is still the projection an # earlier run wrote. Note content is canonical, and a raw note a person @@ -210,16 +217,20 @@ async def accept_document_note( raise DocumentSidecarConflictError( f"{path} changed while it was being replaced; re-run the import" ) - await file_service.write_file(path, artifacts.document_markdown) + checksum = await file_service.write_file(path, artifacts.document_markdown) await knowledge.index_file(path) - return document_markdown_checksum(artifacts.document_markdown), True + return AcceptedDocumentNote( + canonical_db_checksum(checksum), + document_markdown_checksum(artifacts.document_markdown), + True, + ) def raw_projection_checksum(document: DocumentMarkdownV1) -> str: """Checksum of a raw sidecar as its run wrote it, before the indexer's permalink. - Every checksum the local runtime records or compares is this one, so a note - the indexer has annotated still matches the run note that produced it. + Run provenance uses LF serialization on creation, reuse and rebuild. Physical + file checksums retain native line endings for filesystem change detection. """ pristine_frontmatter = document.frontmatter.model_copy(update={"permalink": None}) pristine = assemble_document_markdown( diff --git a/tests/document_ingestion/test_local_runtime.py b/tests/document_ingestion/test_local_runtime.py index 7d741826e..cd0ccc160 100644 --- a/tests/document_ingestion/test_local_runtime.py +++ b/tests/document_ingestion/test_local_runtime.py @@ -9,6 +9,7 @@ import pytest +from basic_memory import file_utils from basic_memory.document_ingestion.csv_extractor import CSV_MEDIA_TYPE from basic_memory.document_ingestion.local_runtime import ( ApiDocumentSourceEntityResolver, @@ -35,6 +36,7 @@ DocumentIngestionStage, DocumentIngestionV1, assemble_document_markdown, + document_markdown_checksum, parse_document_ingestion_run_markdown, parse_document_markdown, ) @@ -217,9 +219,19 @@ def artifacts(*, checksum_char: str = "a") -> RawDocumentArtifacts: @pytest.mark.asyncio +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) async def test_writer_creates_the_sidecar_and_run_note_and_indexes_both( file_service: FileService, + monkeypatch: pytest.MonkeyPatch, + newline: str, ) -> None: + # Exercise Windows persistence on every host so checksum drift fails locally. + async def write_with_newlines(path: Path, content: str) -> None: + await file_utils.write_file_atomic_bytes( + path, content.replace("\r\n", "\n").replace("\n", newline).encode("utf-8") + ) + + monkeypatch.setattr(file_utils, "write_file_atomic", write_with_newlines) knowledge = knowledge_api() writer = LocalRawDocumentWriter(file_service, knowledge) built = artifacts() @@ -238,7 +250,15 @@ async def test_writer_creates_the_sidecar_and_run_note_and_indexes_both( ) assert run_note.frontmatter.output is not None assert run_note.frontmatter.output.raw is not None - assert run_note.frontmatter.output.raw.checksum == result.document_db_checksum + assert run_note.frontmatter.output.raw.checksum == document_markdown_checksum( + built.document_markdown + ) + + reused = await writer.write(built) + assert reused.document_db_checksum == result.document_db_checksum + assert reused.run_created is False + rebuilt = await writer.write(artifacts(checksum_char="b")) + assert rebuilt.document_created is True @pytest.mark.asyncio @@ -314,18 +334,18 @@ async def test_unchanged_import_after_indexer_annotation_keeps_provenance_consis knowledge = knowledge_api() writer = LocalRawDocumentWriter(file_service, knowledge) first = artifacts(checksum_char="a") - created = await writer.write(first) + await writer.write(first) sidecar = file_service.base_path / first.document_file_path sidecar.write_bytes( with_permalink(sidecar.read_text(encoding="utf-8"), "main/data/riders.csv").encode() ) unchanged = await writer.write(artifacts(checksum_char="a")) + assert unchanged.document_db_checksum == sha256(sidecar.read_bytes()) rebuilt = await writer.write(artifacts(checksum_char="b")) assert unchanged.document_created is False assert unchanged.run_created is False - assert unchanged.document_db_checksum == created.document_db_checksum assert rebuilt.document_created is True assert knowledge.indexed.count(first.run_file_path) == 1 @@ -412,7 +432,7 @@ async def test_writer_rewrites_a_run_note_that_names_other_sidecar_bytes( knowledge = knowledge_api() writer = LocalRawDocumentWriter(file_service, knowledge) built = artifacts() - first = await writer.write(built) + await writer.write(built) # Simulate an overlapping import that replaced the sidecar after this run # note was written: the note now names bytes that are no longer on disk. stale_note = build_raw_ingestion_run_markdown( @@ -429,7 +449,9 @@ async def test_writer_rewrites_a_run_note_that_names_other_sidecar_bytes( ) assert run_note.frontmatter.output is not None assert run_note.frontmatter.output.raw is not None - assert run_note.frontmatter.output.raw.checksum == first.document_db_checksum + assert run_note.frontmatter.output.raw.checksum == document_markdown_checksum( + built.document_markdown + ) assert knowledge.indexed.count(built.run_file_path) == 2 From 1a8ad7192f466179ce34b7f0dae52c5a855919f8 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 8 Sep 2026 11:28:21 -0500 Subject: [PATCH 06/10] fix(core): verify raw provenance before reusing document sidecars Signed-off-by: phernandez --- .../document_ingestion/local_runtime.py | 5 ++++- .../document_ingestion/test_local_runtime.py | 20 ++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/basic_memory/document_ingestion/local_runtime.py b/src/basic_memory/document_ingestion/local_runtime.py index 97b347a35..d1b8e2ff7 100644 --- a/src/basic_memory/document_ingestion/local_runtime.py +++ b/src/basic_memory/document_ingestion/local_runtime.py @@ -205,13 +205,16 @@ async def accept_document_note( raise DocumentSidecarConflictError( f"{path} has been enriched past the raw stage; refusing to overwrite it" ) + # Reuse must verify provenance too: otherwise an unchanged import could + # record a person's edits as generated bytes and authorize their deletion + # when the source changes later. Ambiguous run-note mismatches fail closed. + await require_untouched_raw_projection(file_service, existing, path=path) if raw_document_matches(existing, artifacts): return AcceptedDocumentNote(on_disk_checksum, raw_projection_checksum(existing), False) # Same source path, different bytes or engine: the raw projection is # rebuilt from the new run, but only while it is still the projection an # earlier run wrote. Note content is canonical, and a raw note a person # has annotated must not be replaced silently. - await require_untouched_raw_projection(file_service, existing, path=path) # Compare-and-swap: the note must still be the bytes judged untouched above. if canonical_db_checksum(await file_service.compute_checksum(path)) != on_disk_checksum: raise DocumentSidecarConflictError( diff --git a/tests/document_ingestion/test_local_runtime.py b/tests/document_ingestion/test_local_runtime.py index cd0ccc160..533a2feb0 100644 --- a/tests/document_ingestion/test_local_runtime.py +++ b/tests/document_ingestion/test_local_runtime.py @@ -351,7 +351,10 @@ async def test_unchanged_import_after_indexer_annotation_keeps_provenance_consis @pytest.mark.asyncio -async def test_writer_refuses_to_replace_an_edited_raw_sidecar(file_service: FileService) -> None: +@pytest.mark.parametrize("reimport_unchanged", [False, True]) +async def test_writer_refuses_to_replace_an_edited_raw_sidecar( + file_service: FileService, reimport_unchanged: bool +) -> None: writer = LocalRawDocumentWriter(file_service, knowledge_api()) first = artifacts(checksum_char="a") await writer.write(first) @@ -362,6 +365,8 @@ async def test_writer_refuses_to_replace_an_edited_raw_sidecar(file_service: Fil ) with pytest.raises(DocumentSidecarConflictError, match="edited since"): + if reimport_unchanged: + await writer.write(first) await writer.write(artifacts(checksum_char="b")) assert "My annotation about row three." in sidecar.read_text(encoding="utf-8") @@ -426,7 +431,7 @@ async def edit_between_checks(path: Path | str) -> str: @pytest.mark.asyncio -async def test_writer_rewrites_a_run_note_that_names_other_sidecar_bytes( +async def test_writer_refuses_a_run_note_that_names_other_sidecar_bytes( file_service: FileService, ) -> None: knowledge = knowledge_api() @@ -440,19 +445,16 @@ async def test_writer_rewrites_a_run_note_that_names_other_sidecar_bytes( ) await file_service.write_file(built.run_file_path, stale_note) - second = await writer.write(built) + with pytest.raises(DocumentSidecarConflictError, match="edited since"): + await writer.write(built) - assert second.document_created is False - assert second.run_created is True run_note = parse_document_ingestion_run_markdown( (file_service.base_path / built.run_file_path).read_text(encoding="utf-8") ) assert run_note.frontmatter.output is not None assert run_note.frontmatter.output.raw is not None - assert run_note.frontmatter.output.raw.checksum == document_markdown_checksum( - built.document_markdown - ) - assert knowledge.indexed.count(built.run_file_path) == 2 + assert run_note.frontmatter.output.raw.checksum == "sha256:" + "d" * 64 + assert knowledge.indexed.count(built.run_file_path) == 1 @pytest.mark.asyncio From a865e179cf56522ae2c71c5f075603f6b313f47c Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 8 Sep 2026 11:41:56 -0500 Subject: [PATCH 07/10] fix(core): honor persisted projections and ingestion byte bounds Signed-off-by: phernandez --- .../document_ingestion/csv_extractor.py | 8 ++++ .../document_ingestion/local_runtime.py | 33 +++++++++++++-- .../document_ingestion/test_csv_extractor.py | 12 ++++++ .../document_ingestion/test_local_runtime.py | 41 +++++++++++++++++++ 4 files changed, 91 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/document_ingestion/csv_extractor.py b/src/basic_memory/document_ingestion/csv_extractor.py index e3e71c7b5..733499c34 100644 --- a/src/basic_memory/document_ingestion/csv_extractor.py +++ b/src/basic_memory/document_ingestion/csv_extractor.py @@ -141,7 +141,9 @@ def render_csv_preview( header = next(reader, None) if header is None: return CsvPreview(markdown="_Empty CSV file._\n", row_count=0, shown_rows=0) + require_csv_field_bound(header, max_field_bytes) for row in reader: + require_csv_field_bound(row, max_field_bytes) row_count += 1 if len(shown) < max_rows: shown.append(row) @@ -162,6 +164,12 @@ def render_csv_preview( return CsvPreview(markdown="\n".join(lines) + "\n", row_count=row_count, shown_rows=len(shown)) +def require_csv_field_bound(cells: Sequence[str], max_field_bytes: int) -> None: + """Enforce this import's byte bound independently of the global character ceiling.""" + if any(len(cell.encode("utf-8")) > max_field_bytes for cell in cells): + raise CsvExtractionError("CSV field exceeds the configured field byte limit") + + def _table_row(cells: Sequence[str], width: int) -> str: # Ragged rows are common in hand-edited exports: pad short rows and drop # cells past the header width so every line stays a valid table row. diff --git a/src/basic_memory/document_ingestion/local_runtime.py b/src/basic_memory/document_ingestion/local_runtime.py index d1b8e2ff7..f219ef3b7 100644 --- a/src/basic_memory/document_ingestion/local_runtime.py +++ b/src/basic_memory/document_ingestion/local_runtime.py @@ -53,6 +53,10 @@ class DocumentSourceResolutionError(RuntimeError): """The requested path did not resolve to its own indexed file entity.""" +class DocumentSourceTooLargeError(RuntimeError): + """The local source exceeds the reader's allocation bound.""" + + class DocumentSidecarConflictError(RuntimeError): """The sidecar path holds content this runtime must not overwrite.""" @@ -115,6 +119,11 @@ class LocalDocumentSourceReader: """Read source bytes from the project directory; the checksum is the generation.""" project_home: Path + max_source_bytes: int = 25 * 1024 * 1024 + + def __post_init__(self) -> None: + if self.max_source_bytes < 1: + raise ValueError("max_source_bytes must be positive") async def read( self, @@ -122,7 +131,9 @@ async def read( *, observed_etag: str | None, ) -> DocumentSourceSnapshot: - content = await asyncio.to_thread((self.project_home / entity.file_path).read_bytes) + content = await asyncio.to_thread( + read_bounded_source, self.project_home / entity.file_path, self.max_source_bytes + ) checksum = sha256_checksum(content) if observed_etag is not None and observed_etag != checksum: raise DocumentSourceChangedError( @@ -138,7 +149,9 @@ async def read( async def require_current(self, snapshot: DocumentSourceSnapshot) -> None: current = await asyncio.to_thread( - (self.project_home / snapshot.entity.file_path).read_bytes + read_bounded_source, + self.project_home / snapshot.entity.file_path, + self.max_source_bytes, ) if sha256_checksum(current) != snapshot.checksum: raise DocumentSourceChangedError( @@ -146,6 +159,17 @@ async def require_current(self, snapshot: DocumentSourceSnapshot) -> None: ) +def read_bounded_source(path: Path, max_source_bytes: int) -> bytes: + """Bound parent-process allocation before the extractor applies its own limits.""" + with path.open("rb") as source: + content = source.read(max_source_bytes + 1) + if len(content) > max_source_bytes: + raise DocumentSourceTooLargeError( + f"{path.name} exceeds the local document source byte limit ({max_source_bytes})" + ) + return content + + @dataclass(frozen=True, slots=True) class AcceptedDocumentNote: """Keep physical file identity separate from normalized run provenance.""" @@ -221,10 +245,13 @@ async def accept_document_note( f"{path} changed while it was being replaced; re-run the import" ) checksum = await file_service.write_file(path, artifacts.document_markdown) + # Formatting is part of the accepted write; provenance must describe its + # persisted projection, not the extractor's pre-format Markdown. + persisted = parse_document_markdown(await file_service.read_file_content(path)) await knowledge.index_file(path) return AcceptedDocumentNote( canonical_db_checksum(checksum), - document_markdown_checksum(artifacts.document_markdown), + raw_projection_checksum(persisted), True, ) diff --git a/tests/document_ingestion/test_csv_extractor.py b/tests/document_ingestion/test_csv_extractor.py index b638a57b9..bc9b49b0a 100644 --- a/tests/document_ingestion/test_csv_extractor.py +++ b/tests/document_ingestion/test_csv_extractor.py @@ -86,6 +86,18 @@ def test_render_csv_preview_accepts_fields_larger_than_the_stdlib_default() -> N assert wide.decode() in preview.markdown +@pytest.mark.parametrize( + "content", [b"large\nx\n", b"h\nlarge\n", b"h\nx\nlarge\n", "h\néé\n".encode()] +) +def test_render_csv_preview_enforces_field_bytes_even_with_a_higher_parser_limit( + content: bytes, +) -> None: + render_csv_preview(b"header\nvalue\n", max_rows=1) + + with pytest.raises(CsvExtractionError, match="field byte limit"): + render_csv_preview(content, max_rows=1, max_field_bytes=3) + + def test_render_csv_preview_names_a_parser_failure(monkeypatch: pytest.MonkeyPatch) -> None: # The lenient reader only fails on a field above the process-wide size limit, # which a bounded source cannot produce under default limits; drive the diff --git a/tests/document_ingestion/test_local_runtime.py b/tests/document_ingestion/test_local_runtime.py index 533a2feb0..bcb8db462 100644 --- a/tests/document_ingestion/test_local_runtime.py +++ b/tests/document_ingestion/test_local_runtime.py @@ -10,11 +10,13 @@ import pytest from basic_memory import file_utils +from basic_memory.config import BasicMemoryConfig from basic_memory.document_ingestion.csv_extractor import CSV_MEDIA_TYPE from basic_memory.document_ingestion.local_runtime import ( ApiDocumentSourceEntityResolver, DocumentSidecarConflictError, DocumentSourceResolutionError, + DocumentSourceTooLargeError, LocalDocumentSourceReader, LocalRawDocumentWriter, default_document_extractors, @@ -160,6 +162,26 @@ async def test_reader_snapshots_bytes_with_the_checksum_as_the_generation(tmp_pa await reader.require_current(snapshot) +@pytest.mark.asyncio +async def test_reader_bounds_source_allocation_and_rechecks(tmp_path: Path) -> None: + source = write_source(tmp_path, b"a\n1\n") + reader = LocalDocumentSourceReader(tmp_path, max_source_bytes=4) + snapshot = await reader.read(csv_entity(), observed_etag=None) + # Exercise the real bounded read with a file much larger than the allowed allocation. + with source.open("wb") as file: + file.truncate(1024 * 1024) + + with pytest.raises(DocumentSourceTooLargeError, match="source byte limit"): + await reader.read(csv_entity(), observed_etag=None) + with pytest.raises(DocumentSourceTooLargeError, match="source byte limit"): + await reader.require_current(snapshot) + + +def test_reader_rejects_an_unbounded_read_size(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="must be positive"): + LocalDocumentSourceReader(tmp_path, max_source_bytes=-2) + + @pytest.mark.asyncio async def test_reader_rejects_a_stale_observed_etag(tmp_path: Path) -> None: write_source(tmp_path) @@ -261,6 +283,25 @@ async def write_with_newlines(path: Path, content: str) -> None: assert rebuilt.document_created is True +@pytest.mark.asyncio +async def test_writer_reuses_and_refreshes_a_formatted_sidecar( + file_service: FileService, + app_config: BasicMemoryConfig, +) -> None: + file_service.app_config = app_config.model_copy(update={"format_on_save": True}) + writer = LocalRawDocumentWriter(file_service, knowledge_api()) + built = artifacts() + await writer.write(built) + persisted = await file_service.read_file_content(built.document_file_path) + assert persisted != built.document_markdown + + reused = await writer.write(built) + assert reused.document_created is False + assert reused.run_created is False + refreshed = await writer.write(artifacts(checksum_char="b")) + assert refreshed.document_created is True + + @pytest.mark.asyncio async def test_writer_reuses_an_identical_raw_projection(file_service: FileService) -> None: knowledge = knowledge_api() From af3ffeabbd872cabb91020965de847d72d4ef104 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 8 Sep 2026 11:59:01 -0500 Subject: [PATCH 08/10] fix(core): preserve generated document identity during indexing Signed-off-by: phernandez --- src/basic_memory/markdown/utils.py | 13 ++++++- test-int/test_local_document_identity.py | 47 ++++++++++++++++++++++++ tests/markdown/test_markdown_utils.py | 26 +++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 test-int/test_local_document_identity.py diff --git a/src/basic_memory/markdown/utils.py b/src/basic_memory/markdown/utils.py index 9d7c55526..ef164e3cc 100644 --- a/src/basic_memory/markdown/utils.py +++ b/src/basic_memory/markdown/utils.py @@ -12,6 +12,7 @@ from basic_memory.markdown.entity_parser import normalize_frontmatter_metadata from basic_memory.models import Entity from basic_memory.models import Observation as ObservationModel +from basic_memory.schemas.document import derive_document_note_external_id def entity_model_from_markdown( @@ -46,7 +47,17 @@ def entity_model_from_markdown( # SQLAlchemy's Python-side default may not always evaluate, # so we explicitly set it here for reliability (fixes #512) if not model.external_id: - model.external_id = str(uuid.uuid4()) + # Generated document ledgers refer to an identity derived from the source. + # Validate its source identity before constructing the indexed representation; + # ordinary notes retain random IDs and existing entities keep their identity. + if markdown.frontmatter.metadata.get("schema") == "schema/document-extraction": + match markdown.frontmatter.metadata.get("source"): + case {"entity_external_id": str(source_id)}: + model.external_id = derive_document_note_external_id(uuid.UUID(source_id)) + case _: + raise ValueError("Generated document is missing its source entity identity") + else: + model.external_id = str(uuid.uuid4()) # Update basic fields model.title = markdown.frontmatter.title diff --git a/test-int/test_local_document_identity.py b/test-int/test_local_document_identity.py new file mode 100644 index 000000000..bd7646df1 --- /dev/null +++ b/test-int/test_local_document_identity.py @@ -0,0 +1,47 @@ +"""Local document run identities must resolve to the actual indexed sidecar.""" + +from pathlib import Path + +import pytest +from httpx import AsyncClient + +from basic_memory.document_ingestion.local_runtime import ( + ApiDocumentSourceEntityResolver, + LocalDocumentSourceReader, + LocalRawDocumentWriter, + default_document_extractors, +) +from basic_memory.document_ingestion.raw_document import RawDocumentRuntime +from basic_memory.markdown import EntityParser, MarkdownProcessor +from basic_memory.mcp.clients import KnowledgeClient, ProjectClient +from basic_memory.models import Project +from basic_memory.schemas.document import parse_document_ingestion_run_markdown +from basic_memory.services.file_service import FileService + + +@pytest.mark.asyncio +async def test_local_run_points_to_the_indexed_document( + client: AsyncClient, + test_project: Project, +) -> None: + home = Path(test_project.path) + (home / "riders.csv").write_bytes(b"team,name\nAST,Ana\n") + await ProjectClient(client).index( + test_project.external_id, force_full=False, run_in_background=False + ) + knowledge = KnowledgeClient(client, test_project.external_id) + files = FileService(home, MarkdownProcessor(EntityParser(home))) + runtime = RawDocumentRuntime( + source_resolver=ApiDocumentSourceEntityResolver(knowledge), + source_reader=LocalDocumentSourceReader(home), + extractors=default_document_extractors(), + writer=LocalRawDocumentWriter(files, knowledge), + ) + + result = await runtime.ingest(file_path="riders.csv", observed_etag=None) + indexed = await knowledge.get_entity(str(result.document_external_id)) + run = parse_document_ingestion_run_markdown(await files.read_file_content(result.run_file_path)) + + assert indexed.file_path == result.document_file_path + assert run.frontmatter.output is not None + assert str(run.frontmatter.output.document_entity_external_id) == indexed.external_id diff --git a/tests/markdown/test_markdown_utils.py b/tests/markdown/test_markdown_utils.py index 2d0a568a8..854928a71 100644 --- a/tests/markdown/test_markdown_utils.py +++ b/tests/markdown/test_markdown_utils.py @@ -8,6 +8,7 @@ from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Observation from basic_memory.markdown.utils import entity_model_from_markdown from basic_memory.models import Entity +from basic_memory.schemas.document import derive_document_note_external_id class TestEntityModelFromMarkdown: @@ -71,6 +72,31 @@ def test_existing_entity_preserves_external_id(self): # Should preserve the existing external_id assert entity.external_id == existing_external_id + @pytest.mark.parametrize("existing_id", [None, "12345678-1234-1234-1234-123456789012"]) + def test_generated_document_identity_comes_from_its_source( + self, existing_id: str | None + ) -> None: + source_id = "11111111-1111-1111-1111-111111111111" + markdown = self._create_markdown(note_type="document") + markdown.frontmatter.metadata.update( + {"schema": "schema/document-extraction", "source": {"entity_external_id": source_id}} + ) + existing = Entity(external_id=existing_id) if existing_id else None + + result = entity_model_from_markdown(Path("source.pdf.md"), markdown, entity=existing) + + assert result.external_id == (existing_id or derive_document_note_external_id(source_id)) + + @pytest.mark.parametrize("source", [None, {}, {"entity_external_id": "invalid-uuid"}]) + def test_generated_document_rejects_invalid_source_identity(self, source: object) -> None: + markdown = self._create_markdown(note_type="document") + markdown.frontmatter.metadata.update( + {"schema": "schema/document-extraction", "source": source} + ) + + with pytest.raises(ValueError): + entity_model_from_markdown(Path("source.pdf.md"), markdown) + def test_entity_with_empty_external_id_gets_new_one(self): """Test that an entity with empty string external_id gets a new UUID.""" markdown = self._create_markdown() From 7245d5c988bffbe17e6bfeffa504a3616e398901 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 8 Sep 2026 12:14:10 -0500 Subject: [PATCH 09/10] fix(core): refuse conflicting document source ownership Signed-off-by: phernandez --- .../document_ingestion/local_runtime.py | 26 +++++++++- test-int/test_local_document_identity.py | 49 +++++++++++++++++++ .../document_ingestion/test_local_runtime.py | 36 +++++++++++++- 3 files changed, 108 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/document_ingestion/local_runtime.py b/src/basic_memory/document_ingestion/local_runtime.py index f219ef3b7..df6695a2a 100644 --- a/src/basic_memory/document_ingestion/local_runtime.py +++ b/src/basic_memory/document_ingestion/local_runtime.py @@ -19,6 +19,9 @@ from typing import Protocol from uuid import UUID +from fastmcp.exceptions import ToolError +from httpx import HTTPStatusError + from basic_memory.document_ingestion.csv_extractor import CSV_MEDIA_TYPE, CsvExtractor from basic_memory.document_ingestion.markitdown_extractor import markitdown_extractors from basic_memory.document_ingestion.pdf_inspector import PdfInspector @@ -212,6 +215,20 @@ async def accept_document_note( Return physical and normalized projection checksums with the write status. """ path = artifacts.document_file_path + # A generated document owns one indexed identity. Moving only its source + # must not create a second canonical note with that same identity. + try: + indexed = await knowledge.get_entity(str(artifacts.document_external_id)) + except ToolError as error: + cause = error.__cause__ + if not isinstance(cause, HTTPStatusError) or cause.response.status_code != 404: + raise + else: + if indexed.file_path != path: + raise DocumentSidecarConflictError( + f"generated document is already indexed at {indexed.file_path}; " + "move its sidecar with the source before importing again" + ) if await file_service.exists(path): existing_markdown = await file_service.read_file_content(path) on_disk_checksum = canonical_db_checksum(await file_service.compute_checksum(path)) @@ -225,6 +242,13 @@ async def accept_document_note( f"{path} exists and is not a generated document note; " f"move it aside to ingest {artifacts.source.file_path}" ) from error + # A recreated source is a new entity, not a refresh of the old source. + # Preserve the old sidecar and its identity rather than publishing a ledger + # that points at an ID the indexer cannot assign to that existing note. + if existing.frontmatter.source.entity_external_id != artifacts.source.entity_external_id: + raise DocumentSidecarConflictError( + f"{path} belongs to a different source entity; move it aside before importing" + ) if existing.frontmatter.ingestion.stage is not DocumentIngestionStage.raw: raise DocumentSidecarConflictError( f"{path} has been enriched past the raw stage; refusing to overwrite it" @@ -239,7 +263,7 @@ async def accept_document_note( # rebuilt from the new run, but only while it is still the projection an # earlier run wrote. Note content is canonical, and a raw note a person # has annotated must not be replaced silently. - # Compare-and-swap: the note must still be the bytes judged untouched above. + # Preflight guard only: atomic acceptance across concurrent writers is tracked in #1530. if canonical_db_checksum(await file_service.compute_checksum(path)) != on_disk_checksum: raise DocumentSidecarConflictError( f"{path} changed while it was being replaced; re-run the import" diff --git a/test-int/test_local_document_identity.py b/test-int/test_local_document_identity.py index bd7646df1..ec88d39d1 100644 --- a/test-int/test_local_document_identity.py +++ b/test-int/test_local_document_identity.py @@ -45,3 +45,52 @@ async def test_local_run_points_to_the_indexed_document( assert indexed.file_path == result.document_file_path assert run.frontmatter.output is not None assert str(run.frontmatter.output.document_entity_external_id) == indexed.external_id + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["move", "recreate"]) +async def test_local_import_refuses_changed_source_ownership( + client: AsyncClient, + test_project: Project, + change: str, +) -> None: + from basic_memory.document_ingestion.local_runtime import DocumentSidecarConflictError + + home = Path(test_project.path) + source_path = "riders.csv" + (home / source_path).write_bytes(b"team,name\nAST,Ana\n") + projects = ProjectClient(client) + await projects.index(test_project.external_id, force_full=False, run_in_background=False) + knowledge = KnowledgeClient(client, test_project.external_id) + files = FileService(home, MarkdownProcessor(EntityParser(home))) + runtime = RawDocumentRuntime( + source_resolver=ApiDocumentSourceEntityResolver(knowledge), + source_reader=LocalDocumentSourceReader(home), + extractors=default_document_extractors(), + writer=LocalRawDocumentWriter(files, knowledge), + ) + first = await runtime.ingest(file_path=source_path, observed_etag=None) + sidecar_before = (home / first.document_file_path).read_bytes() + run_before = (home / first.run_file_path).read_bytes() + source = await knowledge.resolve_entity_response(source_path, strict=True) + + if change == "move": + source_path = "moved.csv" + (home / "riders.csv").rename(home / source_path) + await projects.index(test_project.external_id, force_full=False, run_in_background=False) + moved = await knowledge.resolve_entity_response(source_path, strict=True) + assert moved.external_id == source.external_id + else: + await knowledge.delete_entity(source.external_id) + (home / source_path).write_bytes(b"team,name\nAST,Bea\n") + await projects.index(test_project.external_id, force_full=False, run_in_background=False) + + with pytest.raises(DocumentSidecarConflictError): + await runtime.ingest(file_path=source_path, observed_etag=None) + + assert (home / first.document_file_path).read_bytes() == sidecar_before + assert (home / first.run_file_path).read_bytes() == run_before + indexed = await knowledge.get_entity(str(first.document_external_id)) + assert indexed.file_path == first.document_file_path + if change == "move": + assert not (home / "moved.csv.md").exists() diff --git a/tests/document_ingestion/test_local_runtime.py b/tests/document_ingestion/test_local_runtime.py index bcb8db462..ee86f0bf9 100644 --- a/tests/document_ingestion/test_local_runtime.py +++ b/tests/document_ingestion/test_local_runtime.py @@ -8,6 +8,8 @@ from uuid import UUID, uuid4 import pytest +from fastmcp.exceptions import ToolError +from httpx import HTTPStatusError, Request, Response from basic_memory import file_utils from basic_memory.config import BasicMemoryConfig @@ -64,7 +66,12 @@ async def resolve_entity_response( return self.resolved async def get_entity(self, entity_id: str) -> EntityResponseV2: - assert entity_id == self.resolved.external_id + if entity_id != self.resolved.external_id: + request = Request("GET", f"https://test/entities/{entity_id}") + response = Response(404, request=request) + raise ToolError("not found") from HTTPStatusError( + "not found", request=request, response=response + ) return self.entity async def index_file(self, file_path: str) -> None: @@ -453,7 +460,7 @@ async def test_writer_refuses_when_the_sidecar_changes_before_the_replacing_writ calls = 0 async def edit_between_checks(path: Path | str) -> str: - # The second checksum read is the compare-and-swap guard: mutate the file + # The second checksum read is a preflight guard: mutate the file # right before it so the guard sees bytes that differ from the decision. nonlocal calls calls += 1 @@ -548,3 +555,28 @@ def test_default_document_extractors_cover_pdf_office_and_csv() -> None: extractors = default_document_extractors(python_executable="python-x") assert set(extractors) == {PDF_MEDIA_TYPE, CSV_MEDIA_TYPE, DOCX_MEDIA_TYPE, PPTX_MEDIA_TYPE} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("http_failure", [False, True]) +async def test_writer_propagates_identity_lookup_failures( + file_service: FileService, + monkeypatch: pytest.MonkeyPatch, + http_failure: bool, +) -> None: + knowledge = knowledge_api() + + async def unavailable(entity_id: str) -> EntityResponseV2: + if http_failure: + request = Request("GET", f"https://test/entities/{entity_id}") + response = Response(503, request=request) + raise ToolError("unavailable") from HTTPStatusError( + "unavailable", request=request, response=response + ) + raise ToolError("unavailable") + + monkeypatch.setattr(knowledge, "get_entity", unavailable) + built = artifacts() + with pytest.raises(ToolError, match="unavailable"): + await LocalRawDocumentWriter(file_service, knowledge).write(built) + assert not await file_service.exists(built.document_file_path) From 134abc84c6d2927a6dd2567fde303ce5da01e8df Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 8 Sep 2026 12:26:52 -0500 Subject: [PATCH 10/10] fix(core): preserve post-index text in document provenance Signed-off-by: phernandez --- .../document_ingestion/local_runtime.py | 44 ++++++++----------- test-int/test_local_document_identity.py | 14 +++++- .../document_ingestion/test_local_runtime.py | 36 +++++++++------ 3 files changed, 54 insertions(+), 40 deletions(-) diff --git a/src/basic_memory/document_ingestion/local_runtime.py b/src/basic_memory/document_ingestion/local_runtime.py index df6695a2a..01f8e583f 100644 --- a/src/basic_memory/document_ingestion/local_runtime.py +++ b/src/basic_memory/document_ingestion/local_runtime.py @@ -42,7 +42,6 @@ from basic_memory.schemas.document import ( DocumentIngestionStage, DocumentMarkdownV1, - assemble_document_markdown, derive_document_ingestion_run_path, document_markdown_checksum, parse_document_ingestion_run_markdown, @@ -256,9 +255,13 @@ async def accept_document_note( # Reuse must verify provenance too: otherwise an unchanged import could # record a person's edits as generated bytes and authorize their deletion # when the source changes later. Ambiguous run-note mismatches fail closed. - await require_untouched_raw_projection(file_service, existing, path=path) + await require_untouched_raw_projection( + file_service, existing, path=path, markdown=existing_markdown + ) if raw_document_matches(existing, artifacts): - return AcceptedDocumentNote(on_disk_checksum, raw_projection_checksum(existing), False) + return AcceptedDocumentNote( + on_disk_checksum, document_markdown_checksum(existing_markdown), False + ) # Same source path, different bytes or engine: the raw projection is # rebuilt from the new run, but only while it is still the projection an # earlier run wrote. Note content is canonical, and a raw note a person @@ -268,42 +271,31 @@ async def accept_document_note( raise DocumentSidecarConflictError( f"{path} changed while it was being replaced; re-run the import" ) - checksum = await file_service.write_file(path, artifacts.document_markdown) - # Formatting is part of the accepted write; provenance must describe its - # persisted projection, not the extractor's pre-format Markdown. - persisted = parse_document_markdown(await file_service.read_file_content(path)) + await file_service.write_file(path, artifacts.document_markdown) await knowledge.index_file(path) + # Formatting and index-owned permalink insertion both precede acceptance. + # Hash the persisted text directly: parsing/reassembling YAML would erase + # authored comments and make later user edits invisible to the refresh guard. + persisted = await file_service.read_file_content(path) return AcceptedDocumentNote( - canonical_db_checksum(checksum), - raw_projection_checksum(persisted), + canonical_db_checksum(await file_service.compute_checksum(path)), + document_markdown_checksum(persisted), True, ) -def raw_projection_checksum(document: DocumentMarkdownV1) -> str: - """Checksum of a raw sidecar as its run wrote it, before the indexer's permalink. - - Run provenance uses LF serialization on creation, reuse and rebuild. Physical - file checksums retain native line endings for filesystem change detection. - """ - pristine_frontmatter = document.frontmatter.model_copy(update={"permalink": None}) - pristine = assemble_document_markdown( - document.model_copy(update={"frontmatter": pristine_frontmatter}) - ) - return document_markdown_checksum(pristine) - - async def require_untouched_raw_projection( file_service: FileService, existing: DocumentMarkdownV1, *, path: str, + markdown: str, ) -> None: """Fail unless the raw sidecar on disk is still what its own run note recorded. - The indexer adds a ``permalink`` to a freshly written note; that is the one - change a pristine projection is allowed to carry. Anything else means a person - edited the note, and the rebuild must stop instead of discarding their words. + The run records the complete post-index text, including its permalink. + Text reads normalize native line endings; YAML comments, formatting and body + text remain significant so a refresh cannot discard authored changes. """ run_id = existing.frontmatter.ingestion.run_id run_path = derive_document_ingestion_run_path(run_id) @@ -320,7 +312,7 @@ async def require_untouched_raw_projection( ) from error output = run.frontmatter.output recorded_checksum = output.raw.checksum if output and output.raw else None - if recorded_checksum != raw_projection_checksum(existing): + if recorded_checksum != document_markdown_checksum(markdown): raise DocumentSidecarConflictError( f"{path} has been edited since run {run_id} wrote it; " "move it aside or finish enriching it before importing the source again" diff --git a/test-int/test_local_document_identity.py b/test-int/test_local_document_identity.py index ec88d39d1..7e4ff8e59 100644 --- a/test-int/test_local_document_identity.py +++ b/test-int/test_local_document_identity.py @@ -11,7 +11,7 @@ LocalRawDocumentWriter, default_document_extractors, ) -from basic_memory.document_ingestion.raw_document import RawDocumentRuntime +from basic_memory.document_ingestion.raw_document import RawDocumentRuntime, canonical_db_checksum from basic_memory.markdown import EntityParser, MarkdownProcessor from basic_memory.mcp.clients import KnowledgeClient, ProjectClient from basic_memory.models import Project @@ -42,9 +42,21 @@ async def test_local_run_points_to_the_indexed_document( indexed = await knowledge.get_entity(str(result.document_external_id)) run = parse_document_ingestion_run_markdown(await files.read_file_content(result.run_file_path)) + assert result.document_db_checksum == canonical_db_checksum( + await files.compute_checksum(result.document_file_path) + ) assert indexed.file_path == result.document_file_path assert run.frontmatter.output is not None assert str(run.frontmatter.output.document_entity_external_id) == indexed.external_id + reused = await runtime.ingest(file_path="riders.csv", observed_etag=None) + assert reused.document_created is False + assert reused.run_created is False + (home / "riders.csv").write_bytes(b"team,name\nAST,Bea\n") + refreshed = await runtime.ingest(file_path="riders.csv", observed_etag=None) + assert refreshed.document_created is True + assert refreshed.document_db_checksum == canonical_db_checksum( + await files.compute_checksum(refreshed.document_file_path) + ) @pytest.mark.asyncio diff --git a/tests/document_ingestion/test_local_runtime.py b/tests/document_ingestion/test_local_runtime.py index ee86f0bf9..bd5de146d 100644 --- a/tests/document_ingestion/test_local_runtime.py +++ b/tests/document_ingestion/test_local_runtime.py @@ -58,6 +58,7 @@ def __init__(self, *, resolved: EntityResolveResponse, entity: EntityResponseV2) self.resolved = resolved self.entity = entity self.indexed: list[str] = [] + self.files: FileService | None = None async def resolve_entity_response( self, identifier: str, *, strict: bool = False @@ -76,6 +77,9 @@ async def get_entity(self, entity_id: str) -> EntityResponseV2: async def index_file(self, file_path: str) -> None: self.indexed.append(file_path) + if self.files is not None and file_path.endswith(".csv.md"): + content = await self.files.read_file_content(file_path) + await self.files.write_file(file_path, with_permalink(content, "main/data/riders.csv")) def resolved(file_path: str) -> EntityResolveResponse: @@ -354,14 +358,12 @@ def with_permalink(markdown: str, permalink: str) -> str: async def test_writer_rebuilds_an_untouched_sidecar_that_the_indexer_annotated( file_service: FileService, ) -> None: - writer = LocalRawDocumentWriter(file_service, knowledge_api()) + knowledge = knowledge_api() + knowledge.files = file_service + writer = LocalRawDocumentWriter(file_service, knowledge) first = artifacts(checksum_char="a") await writer.write(first) sidecar = file_service.base_path / first.document_file_path - sidecar.write_text( - with_permalink(sidecar.read_text(encoding="utf-8"), "main/data/riders.csv"), - encoding="utf-8", - ) second = await writer.write(artifacts(checksum_char="b")) @@ -374,19 +376,13 @@ async def test_writer_rebuilds_an_untouched_sidecar_that_the_indexer_annotated( async def test_unchanged_import_after_indexer_annotation_keeps_provenance_consistent( file_service: FileService, ) -> None: - """The permalink the indexer adds must not leak into recorded checksums. - - Otherwise an unchanged re-import rewrites the run note with the annotated - checksum and a later source change is refused as an edit. - """ + """Provenance records the accepted post-index text, including its permalink.""" knowledge = knowledge_api() + knowledge.files = file_service writer = LocalRawDocumentWriter(file_service, knowledge) first = artifacts(checksum_char="a") await writer.write(first) sidecar = file_service.base_path / first.document_file_path - sidecar.write_bytes( - with_permalink(sidecar.read_text(encoding="utf-8"), "main/data/riders.csv").encode() - ) unchanged = await writer.write(artifacts(checksum_char="a")) assert unchanged.document_db_checksum == sha256(sidecar.read_bytes()) @@ -580,3 +576,17 @@ async def unavailable(entity_id: str) -> EntityResponseV2: with pytest.raises(ToolError, match="unavailable"): await LocalRawDocumentWriter(file_service, knowledge).write(built) assert not await file_service.exists(built.document_file_path) + + +@pytest.mark.asyncio +async def test_writer_preserves_authored_frontmatter_comments(file_service: FileService) -> None: + writer = LocalRawDocumentWriter(file_service, knowledge_api()) + first = artifacts() + await writer.write(first) + path = file_service.base_path / first.document_file_path + edited = path.read_text(encoding="utf-8").replace("---\n", "---\n# My source annotation\n", 1) + path.write_text(edited, encoding="utf-8") + + with pytest.raises(DocumentSidecarConflictError, match="edited since"): + await writer.write(artifacts(checksum_char="b")) + assert path.read_text(encoding="utf-8") == edited