diff --git a/CHANGELOG.md b/CHANGELOG.md index e90d16b11..239f084dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ ### Features +- **#1512**: Word, PowerPoint, and CSV files get the same sidecar Markdown note a + PDF gets. `bm import document ` indexes the project, extracts the file, + and writes `..md` next to it plus a run note under + `document-ingestion-runs/`, all through the existing parser-neutral document + contract. Office formats run through Microsoft's `markitdown` converters + (`basic-memory[documents]`) inside the same killable, byte-capped worker + process as pdf-inspector; CSV renders a bounded stdlib preview with strict + UTF-8. Re-running is a no-op while the source is unchanged, and the command + refuses to overwrite a hand-written or enriched note at the sidecar path. + PDF keeps pdf-inspector and works through the same command. + - **#610**: The manual's SYNOPSIS blocks are now generated from the tool registry. `just man-regen` renders the MCP call on every section-3 page from the schema clients actually receive (required parameters first, then defaults, in schema diff --git a/pyproject.toml b/pyproject.toml index e97ed17ca..24936db0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,9 @@ redis = [ pdf = [ "pdf-inspector>=0.2.6,<2", ] +documents = [ + "markitdown[docx,pptx]>=0.1.7,<0.2", +] [project.urls] Homepage = "https://github.com/basicmachines-co/basic-memory" @@ -149,6 +152,7 @@ dev = [ "libcst>=1.8.6", "pytest-timeout>=2.4.0", "pytest-split>=0.11.0", + "markitdown[docx,pptx]>=0.1.7,<0.2", ] [tool.hatch.version] diff --git a/src/basic_memory/cli/commands/import_document.py b/src/basic_memory/cli/commands/import_document.py new file mode 100644 index 000000000..385baa4fe --- /dev/null +++ b/src/basic_memory/cli/commands/import_document.py @@ -0,0 +1,114 @@ +"""Import command: extract one document file in a project into a sidecar Markdown note.""" + +# PEP 563 lazy annotations keep the ingestion stack out of module import (#886). +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Annotated, Optional + +import typer +from loguru import logger +from rich.console import Console +from rich.panel import Panel + +from basic_memory.cli.app import import_app +from basic_memory.cli.commands.command_utils import run_with_cleanup +from basic_memory.cli.commands.routing import force_routing +from basic_memory.config import ConfigManager + +if TYPE_CHECKING: + from basic_memory.document_ingestion.raw_document import RawDocumentWriteResult + +console = Console() + + +async def import_document(path: Path, project: str | None) -> tuple[str, RawDocumentWriteResult]: + """Index the source, extract it, and write its sidecar note into the project.""" + # Deferred: the ingestion stack and API client load only when the command runs (#886). + from basic_memory.document_ingestion.local_runtime import ( + ApiDocumentSourceEntityResolver, + LocalDocumentSourceReader, + LocalRawDocumentWriter, + default_document_extractors, + ) + from basic_memory.document_ingestion.raw_document import RawDocumentRuntime + from basic_memory.markdown import EntityParser, MarkdownProcessor + from basic_memory.mcp.async_client import get_client + from basic_memory.mcp.clients import KnowledgeClient, ProjectClient + from basic_memory.mcp.project_context import get_active_project + from basic_memory.services.file_service import FileService + + config_manager = ConfigManager() + project_name = project or config_manager.default_project + async with get_client(project_name=project_name) as client: + project_item = await get_active_project(client, project_name, None) + project_home = Path(project_item.path).expanduser().resolve() + source = path.expanduser().resolve() + if not source.is_file(): + raise typer.BadParameter(f"File not found: {path}") + if not source.is_relative_to(project_home): + raise typer.BadParameter( + f"{path} is not inside project {project_item.name!r} ({project_home})" + ) + relative_path = source.relative_to(project_home).as_posix() + + # The source must be an indexed file entity before it can own a document + # note, and a fresh drop may not have been seen by the watcher yet. + await ProjectClient(client).index( + project_item.external_id, force_full=False, run_in_background=False + ) + + knowledge = KnowledgeClient(client, project_item.external_id) + app_config = config_manager.config + markdown_processor = MarkdownProcessor(EntityParser(project_home), app_config=app_config) + file_service = FileService(project_home, markdown_processor, app_config=app_config) + runtime = RawDocumentRuntime( + source_resolver=ApiDocumentSourceEntityResolver(knowledge), + source_reader=LocalDocumentSourceReader(project_home), + extractors=default_document_extractors(), + writer=LocalRawDocumentWriter(file_service, knowledge), + ) + result = await runtime.ingest(file_path=relative_path, observed_etag=None) + return project_item.name, result + + +@import_app.command( + name="document", + help="Extract a PDF, docx, pptx, or csv file in a project into a sidecar Markdown note.", +) +def document( + path: Annotated[Path, typer.Argument(help="Path to a file inside the project directory")], + project: Annotated[ + Optional[str], + typer.Option("--project", "-p", help="Project name (defaults to the default project)"), + ] = None, +) -> None: + """Write ``.md`` next to the source and a run note under document-ingestion-runs/.""" + try: + # Trigger: the project may be configured for cloud routing. + # Why: this runtime reads the source and writes the sidecar in the local + # project directory, so the API it indexes through must be the local + # one; a cloud client would be asked to index files it cannot see. + # Outcome: local ASGI routing for the whole command, whatever the project mode. + with force_routing(local=True): + project_name, result = run_with_cleanup(import_document(path, project)) + except typer.BadParameter as error: + typer.echo(f"Error: {error}", err=True) + raise typer.Exit(1) + except Exception as error: + logger.error("Document import failed") + typer.echo(f"Error during import: {error}", err=True) + raise typer.Exit(1) + + document_state = "created" if result.document_created else "already current" + run_state = "created" if result.run_created else "already recorded" + console.print( + Panel( + f"[green]Document import complete![/green]\n\n" + f"Project: {project_name}\n" + f"Document note: {result.document_file_path} ({document_state})\n" + f"Run note: {result.run_file_path} ({run_state})\n" + f"Run id: {result.run_id}", + expand=False, + ) + ) diff --git a/src/basic_memory/cli/main.py b/src/basic_memory/cli/main.py index d7043ba2c..afaa3d4b5 100644 --- a/src/basic_memory/cli/main.py +++ b/src/basic_memory/cli/main.py @@ -24,6 +24,7 @@ def _version_only_invocation(argv: list[str]) -> bool: import_chatgpt, import_claude_conversations, import_claude_projects, + import_document, import_memory_json, inspect, install, diff --git a/src/basic_memory/document_ingestion/bounded_process.py b/src/basic_memory/document_ingestion/bounded_process.py new file mode 100644 index 000000000..c92018092 --- /dev/null +++ b/src/basic_memory/document_ingestion/bounded_process.py @@ -0,0 +1,166 @@ +"""Run one extractor over untrusted bytes in a killable, byte-capped child process. + +Every native or third-party document parser Basic Memory runs is synchronous and +parses attacker-controlled input. The parent therefore never calls a parser +in-process: it feeds the bytes to ``python -m `` on stdin, drains both +pipes with a hard ceiling, and kills the child at the deadline. The worker +applies POSIX rlimits to itself before reading any input. + +This module owns only the process mechanics. Each extractor owns its argv, +its output contract, and the error types its callers see. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Sequence + + +class BoundedProcessError(RuntimeError): + """Base failure for one bounded child process run.""" + + +class BoundedProcessSpawnError(BoundedProcessError): + """The running event loop cannot spawn subprocesses.""" + + +class BoundedProcessTimeoutError(BoundedProcessError): + """The child was killed after exceeding its wall-clock deadline.""" + + +class BoundedProcessOutputLimitError(BoundedProcessError): + """A child pipe crossed the output byte ceiling while streaming.""" + + def __init__(self, stream_name: str) -> None: + super().__init__(f"child process exceeded the output byte limit on {stream_name}") + self.stream_name = stream_name + + +class BoundedProcessExitError(BoundedProcessError): + """The child exited with a non-zero status.""" + + def __init__(self, returncode: int, detail: str) -> None: + super().__init__(f"child process failed with exit code {returncode}: {detail}") + self.returncode = returncode + self.detail = detail + + +async def run_bounded_process( + argv: Sequence[str], + stdin_bytes: bytes, + *, + timeout_seconds: float, + max_output_bytes: int, +) -> bytes: + """Run ``argv`` with ``stdin_bytes`` on stdin and return its capped stdout. + + The caller bounds concurrency; this function bounds one run. It raises a + :class:`BoundedProcessError` subclass for every failure mode so the caller + can name the failure in its own domain terms. + """ + try: + process = await asyncio.create_subprocess_exec( + *argv, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except NotImplementedError as error: + # Trigger: the running loop cannot spawn subprocesses. Basic Memory + # installs the selector loop on Windows for aiosqlite (see db.py), + # and that loop has no subprocess transport. + # Outcome: a named error instead of a bare NotImplementedError. + raise BoundedProcessSpawnError( + "child process needs an event loop with subprocess support; " + "Basic Memory runs the selector loop on Windows" + ) from error + try: + async with asyncio.timeout(timeout_seconds): + stdout, stderr = await _communicate_capped(process, stdin_bytes, cap=max_output_bytes) + except asyncio.CancelledError: + await kill_and_wait(process) + raise + except TimeoutError as error: + await kill_and_wait(process) + raise BoundedProcessTimeoutError( + "child process exceeded the configured deadline" + ) from error + except BoundedProcessOutputLimitError: + # A pipe crossed its ceiling while the child may still be writing. + await kill_and_wait(process) + raise + + if process.returncode != 0: + detail = stderr.decode("utf-8", errors="replace")[-2000:].strip() + raise BoundedProcessExitError( + process.returncode if process.returncode is not None else -1, + detail or "no error detail", + ) + return stdout + + +async def kill_and_wait(process: asyncio.subprocess.Process) -> None: + """Terminate and reap one child before releasing worker capacity.""" + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + # The child can exit between the returncode check and the signal. + # Waiting still reaps it and preserves the caller's original outcome. + pass + await process.wait() + + +# Read pipes in modest chunks so the ceiling applies as bytes arrive rather than +# after an entire stream has already been buffered. +_PIPE_CHUNK_BYTES = 64 * 1024 + + +async def _communicate_capped( + process: asyncio.subprocess.Process, + stdin_bytes: bytes, + *, + cap: int, +) -> tuple[bytes, bytes]: + """Feed stdin and drain both pipes with a hard byte ceiling on each. + + ``Process.communicate()`` buffers stdout and stderr without limit before the + caller can look at their size, so a hostile document that makes the worker + emit a huge field, or a runaway error stream, would land in parent memory. + This variant raises as soon as either pipe crosses ``cap``; the caller kills + the child. + """ + stdin, stdout, stderr = process.stdin, process.stdout, process.stderr + if stdin is None or stdout is None or stderr is None: # pragma: no cover - PIPE at spawn + raise RuntimeError("child process was started without pipes") + try: + async with asyncio.TaskGroup() as group: + stdout_task = group.create_task(_read_capped(stdout, cap=cap, stream_name="stdout")) + stderr_task = group.create_task(_read_capped(stderr, cap=cap, stream_name="stderr")) + group.create_task(_feed_stdin(stdin, stdin_bytes)) + except* BoundedProcessOutputLimitError as overflow: + raise overflow.exceptions[0] from None + await process.wait() + return stdout_task.result(), stderr_task.result() + + +async def _read_capped(stream: asyncio.StreamReader, *, cap: int, stream_name: str) -> bytes: + buffer = bytearray() + while chunk := await stream.read(_PIPE_CHUNK_BYTES): + buffer.extend(chunk) + if len(buffer) > cap: + raise BoundedProcessOutputLimitError(stream_name) + return bytes(buffer) + + +async def _feed_stdin(stdin: asyncio.StreamWriter, stdin_bytes: bytes) -> None: + try: + stdin.write(stdin_bytes) + await stdin.drain() + except (BrokenPipeError, ConnectionResetError): + # The child exited before consuming its input. Its exit status and + # stderr carry the failure; ``Process.communicate()`` ignores the same + # pair for the same reason. + pass + finally: + stdin.close() diff --git a/src/basic_memory/document_ingestion/csv_extractor.py b/src/basic_memory/document_ingestion/csv_extractor.py new file mode 100644 index 000000000..733499c34 --- /dev/null +++ b/src/basic_memory/document_ingestion/csv_extractor.py @@ -0,0 +1,184 @@ +"""In-process CSV to Markdown preview extraction. + +CSV needs no third-party parser: the stdlib reader is pure Python over bytes the +runtime has already bounded, so it runs in a worker thread rather than a child +process. Decoding is strict UTF-8 (BOM tolerated). A guessed charset is how +markitdown produced mojibake on a valid UTF-8 export, so a non-UTF-8 file fails +the extraction instead of being silently mangled. + +The Markdown is a preview, not a dump: header, the first ``max_rows`` rows, and +the total row count. A 7,000-row export rendered in full is search noise, not +knowledge; the source file stays in the project for anyone who needs every row. +""" + +from __future__ import annotations + +import asyncio +import csv +import io +import re +import time +from collections.abc import Sequence +from dataclasses import dataclass, field +from datetime import UTC, datetime + +from pydantic import BaseModel, ConfigDict, Field + +from basic_memory import __version__ as basic_memory_version +from basic_memory.document_ingestion.raw_document import ( + ExtractedDocument, + extraction_options_checksum, +) +from basic_memory.schemas.document import DocumentExtractionStatus, DocumentExtractionV1 + +CSV_ENGINE = "basic-memory/csv" +CSV_MEDIA_TYPE = "text/csv" +CSV_RAW_PIPELINE_VERSION = "csv-raw-v1" +CSV_EXTRACTION_PROFILE = "csv-preview-v1" + + +class CsvLimits(BaseModel): + """Bounds on one CSV preview extraction.""" + + model_config = ConfigDict(frozen=True, strict=True) + + max_source_bytes: int = Field(default=25 * 1024 * 1024, gt=0) + max_rows: int = Field(default=200, gt=0) + max_output_bytes: int = Field(default=5 * 1024 * 1024, gt=0) + # The stdlib parser rejects any field over its own limit (128 KiB by default), + # which is far below the source ceiling; exported text or JSON columns hit it. + max_field_bytes: int = Field(default=25 * 1024 * 1024, gt=0) + + +_DEFAULT_LIMITS = CsvLimits() +_LINE_BREAKS = re.compile(r"\r\n|\r|\n") + + +class CsvExtractionError(RuntimeError): + """A CSV preview extraction failed.""" + + +class CsvSourceTooLargeError(CsvExtractionError): + """Raised before parsing when the source exceeds the configured byte limit.""" + + +class CsvDecodeError(CsvExtractionError): + """Raised when the source is not UTF-8.""" + + +@dataclass(frozen=True, slots=True) +class CsvPreview: + """Rendered preview plus the counts that describe what it left out.""" + + markdown: str + row_count: int + shown_rows: int + + +@dataclass(frozen=True, slots=True) +class CsvExtractor: + """Render a bounded Markdown preview of one CSV file.""" + + limits: CsvLimits = field(default_factory=CsvLimits) + + async def extract(self, content: bytes, *, file_name: str) -> ExtractedDocument: + if len(content) > self.limits.max_source_bytes: + raise CsvSourceTooLargeError("CSV source exceeds the configured extraction byte limit") + started = time.perf_counter() + preview = await asyncio.to_thread( + render_csv_preview, + content, + max_rows=self.limits.max_rows, + max_field_bytes=self.limits.max_field_bytes, + ) + if len(preview.markdown.encode("utf-8")) > self.limits.max_output_bytes: + raise CsvExtractionError("CSV preview exceeds the configured output byte limit") + extraction = DocumentExtractionV1( + engine=CSV_ENGINE, + engine_version=basic_memory_version, + profile=CSV_EXTRACTION_PROFILE, + options_hash=extraction_options_checksum(self.limits), + classification="csv", + status=DocumentExtractionStatus.complete, + extracted_at=datetime.now(tz=UTC), + duration_ms=int((time.perf_counter() - started) * 1000), + page_count=0, + extracted_page_count=0, + requires_ocr=False, + ocr_page_count=0, + has_tables=preview.shown_rows > 0, + ) + return ExtractedDocument( + extraction=extraction, + markdown=preview.markdown, + kind="csv", + pipeline_version=CSV_RAW_PIPELINE_VERSION, + ) + + +def render_csv_preview( + content: bytes, + *, + max_rows: int, + max_field_bytes: int = _DEFAULT_LIMITS.max_field_bytes, +) -> CsvPreview: + """Render the header, the first ``max_rows`` rows, and the total row count.""" + try: + text = content.decode("utf-8-sig") + except UnicodeDecodeError as error: + raise CsvDecodeError("CSV source is not UTF-8 encoded") from error + + # ``csv.field_size_limit`` is process-wide and has no per-reader form. Only + # ever raise it: a higher ceiling cannot break another caller, and a source + # already bounded by ``max_source_bytes`` must not fail on one long column. + if csv.field_size_limit() < max_field_bytes: + csv.field_size_limit(max_field_bytes) + + reader = csv.reader(io.StringIO(text, newline="")) + shown: list[Sequence[str]] = [] + row_count = 0 + try: + header = next(reader, None) + if header is None: + return CsvPreview(markdown="_Empty CSV file._\n", row_count=0, shown_rows=0) + require_csv_field_bound(header, max_field_bytes) + for row in reader: + require_csv_field_bound(row, max_field_bytes) + row_count += 1 + if len(shown) < max_rows: + shown.append(row) + except csv.Error as error: + raise CsvExtractionError(f"CSV source could not be parsed: {error}") from error + + width = len(header) + lines = [ + _table_row(header, width), + "| " + " | ".join("---" for _ in range(width)) + " |", + *(_table_row(row, width) for row in shown), + "", + ] + if row_count > len(shown): + lines.append(f"_Showing {len(shown)} of {row_count} rows._") + else: + lines.append(f"_{row_count} rows._") + return CsvPreview(markdown="\n".join(lines) + "\n", row_count=row_count, shown_rows=len(shown)) + + +def require_csv_field_bound(cells: Sequence[str], max_field_bytes: int) -> None: + """Enforce this import's byte bound independently of the global character ceiling.""" + if any(len(cell.encode("utf-8")) > max_field_bytes for cell in cells): + raise CsvExtractionError("CSV field exceeds the configured field byte limit") + + +def _table_row(cells: Sequence[str], width: int) -> str: + # Ragged rows are common in hand-edited exports: pad short rows and drop + # cells past the header width so every line stays a valid table row. + padded = [*cells[:width], *([""] * (width - len(cells)))] + # Every line-break form is flattened, CRLF included: canonical note assembly + # turns a stray carriage return into a newline, which would split the row. + # Backslashes are escaped before pipes: a literal `\` before a `|` would + # otherwise turn the pipe escape into an escaped backslash and a live separator. + escaped = ( + _LINE_BREAKS.sub(" ", cell).replace("\\", "\\\\").replace("|", "\\|") for cell in padded + ) + return "| " + " | ".join(escaped) + " |" diff --git a/src/basic_memory/document_ingestion/local_runtime.py b/src/basic_memory/document_ingestion/local_runtime.py new file mode 100644 index 000000000..01f8e583f --- /dev/null +++ b/src/basic_memory/document_ingestion/local_runtime.py @@ -0,0 +1,354 @@ +"""Local project-directory runtime for raw document ingestion. + +Cloud reads sources from object storage and accepts notes through a queue. The +local runtime reads the file under the project directory, writes the sidecar +note back into it, and asks the API to index the new Markdown so the knowledge +graph sees it without waiting for the watcher. The file bytes' SHA-256 stands in +for a storage ETag: a local file has no version id, so the checksum is the only +generation marker available. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import sys +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Protocol +from uuid import UUID + +from fastmcp.exceptions import ToolError +from httpx import HTTPStatusError + +from basic_memory.document_ingestion.csv_extractor import CSV_MEDIA_TYPE, CsvExtractor +from basic_memory.document_ingestion.markitdown_extractor import markitdown_extractors +from basic_memory.document_ingestion.pdf_inspector import PdfInspector +from basic_memory.document_ingestion.raw_document import ( + PDF_MEDIA_TYPE, + DocumentExtractor, + DocumentSourceChangedError, + DocumentSourceEntity, + DocumentSourceSnapshot, + PdfDocumentExtractor, + RawDocumentArtifacts, + RawDocumentWriteResult, + build_raw_ingestion_run_markdown, + canonical_db_checksum, + raw_document_matches, +) +from basic_memory.file_utils import ParseError +from basic_memory.schemas.document import ( + DocumentIngestionStage, + DocumentMarkdownV1, + derive_document_ingestion_run_path, + document_markdown_checksum, + parse_document_ingestion_run_markdown, + parse_document_markdown, +) +from basic_memory.schemas.v2.entity import EntityResolveResponse, EntityResponseV2 +from basic_memory.services.file_service import FileService + + +class DocumentSourceResolutionError(RuntimeError): + """The requested path did not resolve to its own indexed file entity.""" + + +class DocumentSourceTooLargeError(RuntimeError): + """The local source exceeds the reader's allocation bound.""" + + +class DocumentSidecarConflictError(RuntimeError): + """The sidecar path holds content this runtime must not overwrite.""" + + +class DocumentKnowledgeApi(Protocol): + """The three knowledge-API calls the local runtime needs (satisfied by KnowledgeClient).""" + + async def resolve_entity_response( + self, identifier: str, *, strict: bool = False + ) -> EntityResolveResponse: ... + + async def get_entity(self, entity_id: str) -> EntityResponseV2: ... + + async def index_file(self, file_path: str) -> object: + """Index one Markdown file; only the side effect matters here.""" + + +def default_document_extractors( + *, python_executable: str = sys.executable +) -> dict[str, DocumentExtractor]: + """Every extractor the local runtime can dispatch to, keyed by media type.""" + extractors: dict[str, DocumentExtractor] = { + PDF_MEDIA_TYPE: PdfDocumentExtractor(PdfInspector(python_executable=python_executable)), + CSV_MEDIA_TYPE: CsvExtractor(), + } + extractors.update(markitdown_extractors(python_executable=python_executable)) + return extractors + + +def sha256_checksum(content: bytes) -> str: + return f"sha256:{hashlib.sha256(content).hexdigest()}" + + +@dataclass(frozen=True, slots=True) +class ApiDocumentSourceEntityResolver: + """Resolve a project-relative path to its indexed file entity through the API.""" + + knowledge: DocumentKnowledgeApi + + async def resolve(self, file_path: str) -> DocumentSourceEntity: + resolved = await self.knowledge.resolve_entity_response(file_path, strict=True) + # Strict resolution still tries ``.md`` when the path itself is not + # indexed, so an existing sidecar would be mistaken for its own source. + if resolved.file_path != file_path: + raise DocumentSourceResolutionError( + f"{file_path!r} resolved to {resolved.file_path!r}; " + "the source file itself is not indexed" + ) + entity = await self.knowledge.get_entity(resolved.external_id) + return DocumentSourceEntity( + entity_id=entity.id, + external_id=UUID(entity.external_id), + file_path=entity.file_path, + media_type=entity.content_type, + ) + + +@dataclass(frozen=True, slots=True) +class LocalDocumentSourceReader: + """Read source bytes from the project directory; the checksum is the generation.""" + + project_home: Path + max_source_bytes: int = 25 * 1024 * 1024 + + def __post_init__(self) -> None: + if self.max_source_bytes < 1: + raise ValueError("max_source_bytes must be positive") + + async def read( + self, + entity: DocumentSourceEntity, + *, + observed_etag: str | None, + ) -> DocumentSourceSnapshot: + content = await asyncio.to_thread( + read_bounded_source, self.project_home / entity.file_path, self.max_source_bytes + ) + checksum = sha256_checksum(content) + if observed_etag is not None and observed_etag != checksum: + raise DocumentSourceChangedError( + f"{entity.file_path} changed since it was observed ({observed_etag} -> {checksum})" + ) + return DocumentSourceSnapshot( + entity=entity, + content=content, + checksum=checksum, + size_bytes=len(content), + storage_etag=checksum, + ) + + async def require_current(self, snapshot: DocumentSourceSnapshot) -> None: + current = await asyncio.to_thread( + read_bounded_source, + self.project_home / snapshot.entity.file_path, + self.max_source_bytes, + ) + if sha256_checksum(current) != snapshot.checksum: + raise DocumentSourceChangedError( + f"{snapshot.entity.file_path} changed on disk during extraction" + ) + + +def read_bounded_source(path: Path, max_source_bytes: int) -> bytes: + """Bound parent-process allocation before the extractor applies its own limits.""" + with path.open("rb") as source: + content = source.read(max_source_bytes + 1) + if len(content) > max_source_bytes: + raise DocumentSourceTooLargeError( + f"{path.name} exceeds the local document source byte limit ({max_source_bytes})" + ) + return content + + +@dataclass(frozen=True, slots=True) +class AcceptedDocumentNote: + """Keep physical file identity separate from normalized run provenance.""" + + file_checksum: str + projection_checksum: str + written: bool + + +@dataclass(frozen=True, slots=True) +class LocalRawDocumentWriter: + """Write the sidecar and run notes into the project directory and index them.""" + + file_service: FileService + knowledge: DocumentKnowledgeApi + + async def write(self, artifacts: RawDocumentArtifacts) -> RawDocumentWriteResult: + document = await accept_document_note(self.file_service, self.knowledge, artifacts) + run_created = await accept_run_note( + self.file_service, self.knowledge, artifacts, raw_checksum=document.projection_checksum + ) + return RawDocumentWriteResult( + document_external_id=artifacts.document_external_id, + document_file_path=artifacts.document_file_path, + document_db_checksum=document.file_checksum, + run_id=artifacts.run_id, + run_file_path=artifacts.run_file_path, + document_created=document.written, + run_created=run_created, + ) + + +async def accept_document_note( + file_service: FileService, + knowledge: DocumentKnowledgeApi, + artifacts: RawDocumentArtifacts, +) -> AcceptedDocumentNote: + """Write the sidecar note unless an identical raw projection already exists. + + Return physical and normalized projection checksums with the write status. + """ + path = artifacts.document_file_path + # A generated document owns one indexed identity. Moving only its source + # must not create a second canonical note with that same identity. + try: + indexed = await knowledge.get_entity(str(artifacts.document_external_id)) + except ToolError as error: + cause = error.__cause__ + if not isinstance(cause, HTTPStatusError) or cause.response.status_code != 404: + raise + else: + if indexed.file_path != path: + raise DocumentSidecarConflictError( + f"generated document is already indexed at {indexed.file_path}; " + "move its sidecar with the source before importing again" + ) + if await file_service.exists(path): + existing_markdown = await file_service.read_file_content(path) + on_disk_checksum = canonical_db_checksum(await file_service.compute_checksum(path)) + try: + existing = parse_document_markdown(existing_markdown) + except (ParseError, ValueError) as error: + # No frontmatter (ParseError) or frontmatter that fails the contract + # (pydantic's ValidationError is a ValueError): the file at the sidecar + # path is a hand-written note, not a generated projection. + raise DocumentSidecarConflictError( + f"{path} exists and is not a generated document note; " + f"move it aside to ingest {artifacts.source.file_path}" + ) from error + # A recreated source is a new entity, not a refresh of the old source. + # Preserve the old sidecar and its identity rather than publishing a ledger + # that points at an ID the indexer cannot assign to that existing note. + if existing.frontmatter.source.entity_external_id != artifacts.source.entity_external_id: + raise DocumentSidecarConflictError( + f"{path} belongs to a different source entity; move it aside before importing" + ) + if existing.frontmatter.ingestion.stage is not DocumentIngestionStage.raw: + raise DocumentSidecarConflictError( + f"{path} has been enriched past the raw stage; refusing to overwrite it" + ) + # Reuse must verify provenance too: otherwise an unchanged import could + # record a person's edits as generated bytes and authorize their deletion + # when the source changes later. Ambiguous run-note mismatches fail closed. + await require_untouched_raw_projection( + file_service, existing, path=path, markdown=existing_markdown + ) + if raw_document_matches(existing, artifacts): + return AcceptedDocumentNote( + on_disk_checksum, document_markdown_checksum(existing_markdown), False + ) + # Same source path, different bytes or engine: the raw projection is + # rebuilt from the new run, but only while it is still the projection an + # earlier run wrote. Note content is canonical, and a raw note a person + # has annotated must not be replaced silently. + # Preflight guard only: atomic acceptance across concurrent writers is tracked in #1530. + if canonical_db_checksum(await file_service.compute_checksum(path)) != on_disk_checksum: + raise DocumentSidecarConflictError( + f"{path} changed while it was being replaced; re-run the import" + ) + await file_service.write_file(path, artifacts.document_markdown) + await knowledge.index_file(path) + # Formatting and index-owned permalink insertion both precede acceptance. + # Hash the persisted text directly: parsing/reassembling YAML would erase + # authored comments and make later user edits invisible to the refresh guard. + persisted = await file_service.read_file_content(path) + return AcceptedDocumentNote( + canonical_db_checksum(await file_service.compute_checksum(path)), + document_markdown_checksum(persisted), + True, + ) + + +async def require_untouched_raw_projection( + file_service: FileService, + existing: DocumentMarkdownV1, + *, + path: str, + markdown: str, +) -> None: + """Fail unless the raw sidecar on disk is still what its own run note recorded. + + The run records the complete post-index text, including its permalink. + Text reads normalize native line endings; YAML comments, formatting and body + text remain significant so a refresh cannot discard authored changes. + """ + run_id = existing.frontmatter.ingestion.run_id + run_path = derive_document_ingestion_run_path(run_id) + if not await file_service.exists(run_path): + raise DocumentSidecarConflictError( + f"{path} was written by run {run_id} but that run note is missing; " + "move the note aside to rebuild it" + ) + try: + run = parse_document_ingestion_run_markdown(await file_service.read_file_content(run_path)) + except (ParseError, ValueError) as error: + raise DocumentSidecarConflictError( + f"{run_path} exists and is not a generated ingestion run note" + ) from error + output = run.frontmatter.output + recorded_checksum = output.raw.checksum if output and output.raw else None + if recorded_checksum != document_markdown_checksum(markdown): + raise DocumentSidecarConflictError( + f"{path} has been edited since run {run_id} wrote it; " + "move it aside or finish enriching it before importing the source again" + ) + + +async def accept_run_note( + file_service: FileService, + knowledge: DocumentKnowledgeApi, + artifacts: RawDocumentArtifacts, + *, + raw_checksum: str, +) -> bool: + """Write the run note for this run id unless it already names the accepted sidecar bytes.""" + path = artifacts.run_file_path + if await file_service.exists(path): + try: + existing = parse_document_ingestion_run_markdown( + await file_service.read_file_content(path) + ) + except (ParseError, ValueError) as error: + raise DocumentSidecarConflictError( + f"{path} exists and is not a generated ingestion run note" + ) from error + output = existing.frontmatter.output + recorded_checksum = output.raw.checksum if output and output.raw else None + if recorded_checksum == raw_checksum: + return False + # Trigger: the run note names different sidecar bytes than the note on disk. + # Why: two imports of the same unchanged source can interleave (same run id, + # different extracted_at), leaving one run's sidecar beside the other + # run's note; reusing the stale note would freeze that mismatch. + # Outcome: rewrite the run note so provenance converges on this pass. + markdown = build_raw_ingestion_run_markdown( + artifacts, raw_checksum=raw_checksum, raw_created_at=datetime.now(tz=UTC) + ) + await file_service.write_file(path, markdown) + await knowledge.index_file(path) + return True diff --git a/src/basic_memory/document_ingestion/markitdown_extractor.py b/src/basic_memory/document_ingestion/markitdown_extractor.py new file mode 100644 index 000000000..d968ada02 --- /dev/null +++ b/src/basic_memory/document_ingestion/markitdown_extractor.py @@ -0,0 +1,241 @@ +"""Bounded subprocess adapter for Microsoft's markitdown Office converters. + +markitdown is a synchronous, in-process library with no resource limits that +parses attacker-controlled Office archives (zip plus XML through lxml, images +through Pillow). Every conversion therefore runs in a killable child process +with explicit source, output, memory, CPU, wall-clock, and concurrency +ceilings, the same shape as the pdf-inspector adapter. + +The worker calls one specific converter chosen from the indexed media type. It +never uses markitdown's content-sniffing ``MarkItDown.convert()`` loop, so +magika, the URL converters, zip recursion, and the plain-text fallthrough are +never in play. The ``markitdown`` package is imported only by the worker +module, so ``basic-memory[documents]`` stays optional for callers that merely +reference the contracts. +""" + +from __future__ import annotations + +import asyncio +import importlib.util +import sys +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import UTC, datetime +from enum import StrEnum + +from pydantic import BaseModel, ConfigDict, Field + +from basic_memory.document_ingestion.bounded_process import ( + BoundedProcessExitError, + BoundedProcessOutputLimitError, + BoundedProcessSpawnError, + BoundedProcessTimeoutError, + run_bounded_process, +) +from basic_memory.document_ingestion.raw_document import ( + DocumentExtractor, + ExtractedDocument, + extraction_options_checksum, +) +from basic_memory.schemas.document import DocumentExtractionStatus, DocumentExtractionV1 + +MARKITDOWN_ENGINE = "microsoft/markitdown" +MARKITDOWN_WORKER_MODULE = "basic_memory.document_ingestion.markitdown_worker" +MARKITDOWN_RAW_PIPELINE_VERSION = "markitdown-raw-v1" + +DOCX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" +PPTX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.presentationml.presentation" + + +class MarkitdownFormat(StrEnum): + """Office formats routed to one specific markitdown converter.""" + + docx = "docx" + pptx = "pptx" + + +# Dispatch key is the exact media type the indexer records for the source entity. +MARKITDOWN_FORMATS_BY_MEDIA_TYPE: Mapping[str, MarkitdownFormat] = { + DOCX_MEDIA_TYPE: MarkitdownFormat.docx, + PPTX_MEDIA_TYPE: MarkitdownFormat.pptx, +} +MEDIA_TYPES_BY_MARKITDOWN_FORMAT: Mapping[MarkitdownFormat, str] = { + fmt: media_type for media_type, fmt in MARKITDOWN_FORMATS_BY_MEDIA_TYPE.items() +} + + +class MarkitdownLimits(BaseModel): + """Resource limits enforced around one markitdown conversion.""" + + model_config = ConfigDict(frozen=True, strict=True) + + max_source_bytes: int = Field(default=25 * 1024 * 1024, gt=0) + max_output_bytes: int = Field(default=5 * 1024 * 1024, gt=0) + # lxml, Pillow, and python-pptx need more address space than the Rust PDF + # parser; 512 MiB made valid decks fail before conversion began. + max_memory_bytes: int = Field(default=1024 * 1024 * 1024, gt=0) + timeout_seconds: float = Field(default=60.0, gt=0) + cpu_seconds: int = Field(default=50, gt=0) + max_concurrency: int = Field(default=1, gt=0) + + +class MarkitdownOutput(BaseModel): + """Validated, serialization-safe output from the markitdown worker process.""" + + model_config = ConfigDict(frozen=True, strict=True) + + engine: str = MARKITDOWN_ENGINE + engine_version: str + format: MarkitdownFormat + markdown: str + processing_time_ms: int = Field(ge=0) + # Slides are the pptx page unit; docx has no page model, so it reports None. + slide_count: int | None = Field(default=None, ge=0) + + +class MarkitdownExtractionError(RuntimeError): + """A bounded markitdown conversion failed.""" + + +class MarkitdownSourceTooLargeError(MarkitdownExtractionError): + """Raised before spawning when the source exceeds the configured byte limit.""" + + +class MarkitdownNotInstalledError(MarkitdownExtractionError): + """Raised before spawning when the optional ``documents`` extra is missing.""" + + +@dataclass(slots=True) +class MarkitdownExtractor: + """Run one markitdown converter in killable, concurrency-bounded subprocesses.""" + + format: MarkitdownFormat + limits: MarkitdownLimits = field(default_factory=MarkitdownLimits) + python_executable: str = sys.executable + _semaphore: asyncio.Semaphore = field(init=False, repr=False) + + def __post_init__(self) -> None: + self._semaphore = asyncio.Semaphore(self.limits.max_concurrency) + + async def extract(self, content: bytes, *, file_name: str) -> ExtractedDocument: + """Convert one Office document without blocking the caller's event loop.""" + if len(content) > self.limits.max_source_bytes: + raise MarkitdownSourceTooLargeError( + "Office document source exceeds the configured extraction byte limit" + ) + # Fail here with an install hint instead of a child-process ImportError + # traceback: the worker is the only module that imports markitdown. + if importlib.util.find_spec("markitdown") is None: + raise MarkitdownNotInstalledError( + "Office document extraction needs the markitdown package; " + "install basic-memory[documents]" + ) + + argv = ( + self.python_executable, + "-m", + MARKITDOWN_WORKER_MODULE, + "--format", + self.format.value, + "--file-name", + file_name, + "--max-output-bytes", + str(self.limits.max_output_bytes), + "--max-memory-bytes", + str(self.limits.max_memory_bytes), + "--cpu-seconds", + str(self.limits.cpu_seconds), + ) + async with self._semaphore: + try: + stdout = await run_bounded_process( + argv, + content, + timeout_seconds=self.limits.timeout_seconds, + max_output_bytes=self.limits.max_output_bytes, + ) + except BoundedProcessSpawnError as error: + raise MarkitdownExtractionError( + "Office document extraction needs an event loop with subprocess support; " + "Basic Memory runs the selector loop on Windows" + ) from error + except BoundedProcessTimeoutError as error: + raise MarkitdownExtractionError( + "Office document extraction exceeded the configured deadline" + ) from error + except BoundedProcessOutputLimitError as error: + raise MarkitdownExtractionError( + "Office document extraction exceeded the configured output byte limit " + f"on {error.stream_name}" + ) from error + except BoundedProcessExitError as error: + raise MarkitdownExtractionError( + f"Office document extraction failed with exit code {error.returncode}: " + f"{error.detail}" + ) from error + + try: + output = MarkitdownOutput.model_validate_json(stdout, strict=True) + except ValueError as error: + raise MarkitdownExtractionError( + "Office document extraction returned an invalid result" + ) from error + if output.format is not self.format: + raise MarkitdownExtractionError( + f"Office document extraction converted {output.format.value}, " + f"expected {self.format.value}" + ) + return markitdown_extracted_document( + output, limits=self.limits, extracted_at=datetime.now(tz=UTC) + ) + + +def markitdown_extracted_document( + output: MarkitdownOutput, + *, + limits: MarkitdownLimits, + extracted_at: datetime, +) -> ExtractedDocument: + """Map worker output into the parser-neutral extraction contract.""" + # markitdown emits one marker per slide, so slides fill the page diagnostics + # for pptx. docx reports zero pages, which the contract reads as "not + # paginated" for a complete extraction. + page_count = output.slide_count or 0 + extraction = DocumentExtractionV1( + engine=output.engine, + engine_version=output.engine_version, + profile=f"markitdown-{output.format.value}-v1", + options_hash=extraction_options_checksum(limits), + classification=output.format.value, + status=DocumentExtractionStatus.complete, + extracted_at=extracted_at, + duration_ms=output.processing_time_ms, + page_count=page_count, + extracted_page_count=page_count, + requires_ocr=False, + ocr_page_count=0, + pages_needing_ocr=(), + has_tables=any(line.startswith("|") for line in output.markdown.splitlines()), + ) + return ExtractedDocument( + extraction=extraction, + markdown=output.markdown, + kind=output.format.value, + pipeline_version=MARKITDOWN_RAW_PIPELINE_VERSION, + ) + + +def markitdown_extractors( + limits: MarkitdownLimits | None = None, + *, + python_executable: str = sys.executable, +) -> dict[str, DocumentExtractor]: + """Return one bounded extractor per supported Office media type.""" + shared_limits = limits or MarkitdownLimits() + return { + media_type: MarkitdownExtractor( + format=fmt, limits=shared_limits, python_executable=python_executable + ) + for media_type, fmt in MARKITDOWN_FORMATS_BY_MEDIA_TYPE.items() + } diff --git a/src/basic_memory/document_ingestion/markitdown_worker.py b/src/basic_memory/document_ingestion/markitdown_worker.py new file mode 100644 index 000000000..2daf1739a --- /dev/null +++ b/src/basic_memory/document_ingestion/markitdown_worker.py @@ -0,0 +1,134 @@ +"""One-shot markitdown conversion process used by the async adapter. + +Runs as ``python -m basic_memory.document_ingestion.markitdown_worker`` with the +Office file bytes on stdin and one validated JSON ``MarkitdownOutput`` on stdout. +Resource limits are applied before any source bytes are read so a malformed +archive cannot exhaust the parent process. + +The worker calls the converter for the requested format directly. That skips +markitdown's content sniffing and its try-every-converter loop, which is where +the plain-text fallthrough and the ASCII-decode crash live. +""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import io +import re +import sys +import time +from typing import assert_never + +from markitdown import DocumentConverter, StreamInfo +from markitdown.converters import DocxConverter, PptxConverter + +from basic_memory.document_ingestion.markitdown_extractor import ( + MEDIA_TYPES_BY_MARKITDOWN_FORMAT, + MarkitdownFormat, + MarkitdownOutput, +) +from basic_memory.document_ingestion.worker_limits import apply_cpu_limit, apply_memory_limit + +SLIDE_MARKER = "" in extracted.markdown + extraction = extracted.extraction + assert extraction.engine == MARKITDOWN_ENGINE + assert extraction.engine_version == importlib.metadata.version("markitdown") + assert extraction.profile == "markitdown-pptx-v1" + assert extraction.classification == "pptx" + assert extraction.status is DocumentExtractionStatus.complete + assert extraction.page_count == 2 + assert extraction.extracted_page_count == 2 + assert extraction.has_tables is True + + +@pytest.mark.asyncio +async def test_markitdown_extractor_passes_format_and_limits_to_the_worker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + process = FakeProcess(stdout=valid_output_json()) + install_fake_process(monkeypatch, process) + extractor = MarkitdownExtractor( + format=MarkitdownFormat.docx, + limits=MarkitdownLimits(max_output_bytes=1000, max_memory_bytes=2000, cpu_seconds=3), + python_executable="python-x", + ) + + extracted = await extractor.extract(b"docx-bytes", file_name="plan.docx") + + assert process.argv == ( + "python-x", + "-m", + MARKITDOWN_WORKER_MODULE, + "--format", + "docx", + "--file-name", + "plan.docx", + "--max-output-bytes", + "1000", + "--max-memory-bytes", + "2000", + "--cpu-seconds", + "3", + ) + assert process.stdin.written == b"docx-bytes" + assert extracted.kind == "docx" + assert extracted.markdown == "# Goals" + assert extracted.extraction.page_count == 0 + assert extracted.extraction.has_tables is False + + +@pytest.mark.asyncio +async def test_markitdown_extractor_rejects_an_oversized_source_before_spawning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def unexpected_subprocess(*args: object, **kwargs: object) -> None: + raise AssertionError("must not spawn for an oversized source") + + monkeypatch.setattr(asyncio, "create_subprocess_exec", unexpected_subprocess) + extractor = MarkitdownExtractor( + format=MarkitdownFormat.docx, limits=MarkitdownLimits(max_source_bytes=3) + ) + + with pytest.raises(MarkitdownSourceTooLargeError): + await extractor.extract(b"docx-bytes", file_name="plan.docx") + + +@pytest.mark.asyncio +async def test_markitdown_extractor_names_the_missing_extra( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(importlib.util, "find_spec", lambda name: None) + + with pytest.raises(MarkitdownNotInstalledError, match=r"basic-memory\[documents\]"): + await MarkitdownExtractor(format=MarkitdownFormat.docx).extract( + b"docx-bytes", file_name="plan.docx" + ) + + +@pytest.mark.asyncio +async def test_markitdown_extractor_names_a_loop_without_subprocess_support( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def selector_loop_spawn(*args: object, **kwargs: object) -> None: + raise NotImplementedError + + monkeypatch.setattr(asyncio, "create_subprocess_exec", selector_loop_spawn) + + with pytest.raises(MarkitdownExtractionError, match="subprocess support"): + await MarkitdownExtractor(format=MarkitdownFormat.docx).extract( + b"docx-bytes", file_name="plan.docx" + ) + + +@pytest.mark.asyncio +async def test_markitdown_extractor_kills_the_child_on_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + process = FakeProcess(returncode=None, hang=True) + install_fake_process(monkeypatch, process) + extractor = MarkitdownExtractor( + format=MarkitdownFormat.docx, limits=MarkitdownLimits(timeout_seconds=0.01) + ) + + with pytest.raises(MarkitdownExtractionError, match="deadline"): + await extractor.extract(b"docx-bytes", file_name="plan.docx") + + assert process.killed + assert process.waited + + +@pytest.mark.asyncio +async def test_markitdown_extractor_reports_a_failed_child( + monkeypatch: pytest.MonkeyPatch, +) -> None: + install_fake_process(monkeypatch, FakeProcess(returncode=3, stderr=b"boom\n")) + + with pytest.raises(MarkitdownExtractionError, match="exit code 3: boom"): + await MarkitdownExtractor(format=MarkitdownFormat.docx).extract( + b"docx-bytes", file_name="plan.docx" + ) + + +@pytest.mark.asyncio +async def test_markitdown_extractor_kills_a_child_that_overruns_a_pipe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + process = FakeProcess(returncode=None, stdout=b"x" * 64, hang=True) + install_fake_process(monkeypatch, process) + extractor = MarkitdownExtractor( + format=MarkitdownFormat.docx, limits=MarkitdownLimits(max_output_bytes=16) + ) + + with pytest.raises(MarkitdownExtractionError, match="output byte limit on stdout"): + await extractor.extract(b"docx-bytes", file_name="plan.docx") + + assert process.killed + + +@pytest.mark.asyncio +async def test_markitdown_extractor_rejects_an_invalid_child_result( + monkeypatch: pytest.MonkeyPatch, +) -> None: + install_fake_process(monkeypatch, FakeProcess(stdout=b'{"engine_version": 1}')) + + with pytest.raises(MarkitdownExtractionError, match="invalid result"): + await MarkitdownExtractor(format=MarkitdownFormat.docx).extract( + b"docx-bytes", file_name="plan.docx" + ) + + +@pytest.mark.asyncio +async def test_markitdown_extractor_rejects_a_worker_that_converted_another_format( + monkeypatch: pytest.MonkeyPatch, +) -> None: + install_fake_process( + monkeypatch, FakeProcess(stdout=valid_output_json(format=MarkitdownFormat.pptx)) + ) + + with pytest.raises(MarkitdownExtractionError, match="converted pptx, expected docx"): + await MarkitdownExtractor(format=MarkitdownFormat.docx).extract( + b"docx-bytes", file_name="plan.docx" + ) + + +def test_markitdown_extracted_document_uses_slides_as_pages() -> None: + output = MarkitdownOutput( + engine_version="0.1.7", + format=MarkitdownFormat.pptx, + markdown="\n# A\n\n\n# B\n\n", + processing_time_ms=3, + slide_count=3, + ) + + extracted = markitdown_extracted_document( + output, limits=MarkitdownLimits(), extracted_at=datetime(2026, 9, 7, tzinfo=UTC) + ) + + assert extracted.extraction.page_count == 3 + assert extracted.extraction.extracted_page_count == 3 + assert extracted.extraction.requires_ocr is False + + +def test_markitdown_extractors_cover_docx_and_pptx_with_shared_limits() -> None: + limits = MarkitdownLimits(timeout_seconds=5.0) + + extractors = markitdown_extractors(limits, python_executable="python-x") + + assert set(extractors) == {DOCX_MEDIA_TYPE, PPTX_MEDIA_TYPE} + docx = extractors[DOCX_MEDIA_TYPE] + pptx = extractors[PPTX_MEDIA_TYPE] + assert isinstance(docx, MarkitdownExtractor) and docx.format is MarkitdownFormat.docx + assert isinstance(pptx, MarkitdownExtractor) and pptx.format is MarkitdownFormat.pptx + assert docx.limits is limits and pptx.limits is limits + assert docx.python_executable == "python-x" + + +def test_markitdown_limits_are_strict() -> None: + with pytest.raises(ValidationError): + MarkitdownLimits.model_validate({"max_source_bytes": "1"}) diff --git a/tests/document_ingestion/test_markitdown_worker.py b/tests/document_ingestion/test_markitdown_worker.py new file mode 100644 index 000000000..5cadb2e4d --- /dev/null +++ b/tests/document_ingestion/test_markitdown_worker.py @@ -0,0 +1,161 @@ +"""Tests for the one-shot markitdown worker process.""" + +from __future__ import annotations + +import importlib.metadata +import io +import json +import zipfile +from types import SimpleNamespace + +import pytest + +from basic_memory.document_ingestion import markitdown_worker, worker_limits +from basic_memory.document_ingestion.markitdown_extractor import ( + MARKITDOWN_ENGINE, + MarkitdownFormat, + MarkitdownOutput, +) +from tests.document_ingestion.office_fixtures import ( + minimal_docx, + minimal_pptx, + pptx_with_picture, +) + + +def test_convert_office_bytes_renders_docx_headings() -> None: + output = markitdown_worker.convert_office_bytes( + minimal_docx(), + format=MarkitdownFormat.docx, + file_name="plan.docx", + max_output_bytes=1_000_000, + ) + + assert output.engine == MARKITDOWN_ENGINE + assert output.engine_version == importlib.metadata.version("markitdown") + assert output.format is MarkitdownFormat.docx + assert output.slide_count is None + assert output.markdown == "# Goals\n\nShip document ingestion for Office formats." + + +def test_convert_office_bytes_counts_pptx_slides_and_keeps_notes_and_tables() -> None: + output = markitdown_worker.convert_office_bytes( + minimal_pptx(), + format=MarkitdownFormat.pptx, + file_name="deck.pptx", + max_output_bytes=1_000_000, + ) + + assert output.slide_count == 2 + assert "" in output.markdown + assert "Speaker note: mention the sidecar pattern" in output.markdown + assert "| pdf | pdf-inspector |" in output.markdown + # markitdown's own convert loop collapses blank runs; the direct call must too. + assert "\n\n\n" not in output.markdown + + +def test_convert_office_bytes_drops_picture_references_with_filename_alt_text() -> None: + output = markitdown_worker.convert_office_bytes( + pptx_with_picture(), + format=MarkitdownFormat.pptx, + file_name="deck.pptx", + max_output_bytes=1_000_000, + ) + + assert output.markdown == "\n# Architecture" + + +@pytest.mark.parametrize( + ("markdown", "expected"), + [ + ("Before ![Revenue by region](Picture3.jpg) after", "Before Revenue by region after"), + ("![image.png](Picture2.jpg)", ""), + ("![](data:image/png;base64,AAAA)", ""), + ("![ Chart 1.PNG ](x)", ""), + ("No pictures here", "No pictures here"), + ], +) +def test_strip_image_references_keeps_only_descriptive_alt_text( + markdown: str, expected: str +) -> None: + assert markitdown_worker.strip_image_references(markdown) == expected + + +def test_convert_office_bytes_enforces_the_output_limit() -> None: + with pytest.raises(ValueError, match="output byte limit"): + markitdown_worker.convert_office_bytes( + minimal_docx(), + format=MarkitdownFormat.docx, + file_name="plan.docx", + max_output_bytes=10, + ) + + +def test_convert_office_bytes_fails_on_bytes_that_are_not_an_office_archive() -> None: + with pytest.raises(zipfile.BadZipFile): + markitdown_worker.convert_office_bytes( + b"not a zip archive", + format=MarkitdownFormat.docx, + file_name="plan.docx", + max_output_bytes=1_000_000, + ) + + +def run_main( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + *, + data: bytes, + max_output_bytes: int, +) -> tuple[str, list[tuple[str, int]]]: + applied: list[tuple[str, int]] = [] + monkeypatch.setattr( + markitdown_worker, "apply_memory_limit", lambda n: applied.append(("memory", n)) + ) + monkeypatch.setattr(markitdown_worker, "apply_cpu_limit", lambda n: applied.append(("cpu", n))) + monkeypatch.setattr( + markitdown_worker.sys, + "argv", + [ + "markitdown_worker", + "--format", + "docx", + "--file-name", + "plan.docx", + "--max-output-bytes", + str(max_output_bytes), + "--max-memory-bytes", + "1024", + "--cpu-seconds", + "5", + ], + ) + monkeypatch.setattr(markitdown_worker.sys, "stdin", SimpleNamespace(buffer=io.BytesIO(data))) + markitdown_worker.main() + return capsys.readouterr().out, applied + + +def test_main_applies_limits_then_writes_one_validated_json_result( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + payload, applied = run_main( + monkeypatch, capsys, data=minimal_docx(), max_output_bytes=1_000_000 + ) + + assert applied == [("memory", 1024), ("cpu", 5)] + output = MarkitdownOutput.model_validate_json(payload, strict=True) + assert output.format is MarkitdownFormat.docx + assert json.loads(payload)["markdown"].startswith("# Goals") + + +def test_main_bounds_the_whole_envelope( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + # The body alone fits under the cap; the JSON envelope around it does not. + with pytest.raises(ValueError, match="result exceeds"): + run_main(monkeypatch, capsys, data=minimal_docx(), max_output_bytes=120) + + +def test_worker_limits_module_is_the_single_owner_of_rlimits() -> None: + assert markitdown_worker.apply_cpu_limit is worker_limits.apply_cpu_limit + assert markitdown_worker.apply_memory_limit is worker_limits.apply_memory_limit diff --git a/tests/document_ingestion/test_pdf_inspector.py b/tests/document_ingestion/test_pdf_inspector.py index abdb6d99a..8aed2de8a 100644 --- a/tests/document_ingestion/test_pdf_inspector.py +++ b/tests/document_ingestion/test_pdf_inspector.py @@ -10,6 +10,8 @@ import pytest from pydantic import ValidationError +from basic_memory.document_ingestion.bounded_process import kill_and_wait +from tests.document_ingestion.process_fakes import FakeProcess, install_fake_process from basic_memory.document_ingestion.pdf_inspector import ( PDF_INSPECTOR_ENGINE, PdfInspector, @@ -19,7 +21,6 @@ PdfInspectorProcessError, PdfInspectorSourceTooLargeError, PdfInspectorTimeoutError, - _kill_and_wait, ) @@ -59,77 +60,6 @@ def minimal_text_pdf() -> bytes: return bytes(document) -class FakeStdin: - """Scripted stdin writer; flags when the adapter starts feeding the child.""" - - def __init__(self, *, started: asyncio.Event, broken: bool) -> None: - self._started = started - self._broken = broken - self.written = b"" - self.closed = False - - def write(self, data: bytes) -> None: - self._started.set() - self.written += data - - async def drain(self) -> None: - if self._broken: - raise BrokenPipeError - - def close(self) -> None: - self.closed = True - - -def pipe(data: bytes, *, eof: bool = True) -> asyncio.StreamReader: - reader = asyncio.StreamReader() - reader.feed_data(data) - if eof: - reader.feed_eof() - return reader - - -class FakeProcess: - """Stand-in for asyncio's subprocess handle with scripted pipes.""" - - def __init__( - self, - *, - returncode: int | None = 0, - stdout: bytes = b"", - stderr: bytes = b"", - hang: bool = False, - kill_raises: bool = False, - stdin_broken: bool = False, - ) -> None: - self.returncode = returncode - self._kill_raises = kill_raises - self.communicate_started = asyncio.Event() - self.stdin = FakeStdin(started=self.communicate_started, broken=stdin_broken) - # A hanging child never closes stdout, so the capped read waits forever. - self.stdout = pipe(stdout, eof=not hang) - self.stderr = pipe(stderr) - self.killed = False - self.waited = False - - def kill(self) -> None: - if self._kill_raises: - raise ProcessLookupError - self.killed = True - self.returncode = -9 - - async def wait(self) -> int: - self.waited = True - return self.returncode if self.returncode is not None else -9 - - -def install_fake_process(monkeypatch: pytest.MonkeyPatch, process: FakeProcess) -> None: - async def create_subprocess(*args: object, **kwargs: object) -> FakeProcess: - _ = (args, kwargs) - return process - - monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess) - - @pytest.mark.skipif( sys.platform == "win32", reason="Basic Memory pins the selector event loop on Windows, which cannot spawn subprocesses", @@ -309,11 +239,11 @@ async def test_pdf_inspector_rejects_invalid_child_result( @pytest.mark.asyncio async def test_kill_and_wait_tolerates_a_child_that_already_exited() -> None: raced = FakeProcess(returncode=None, kill_raises=True) - await _kill_and_wait(cast(asyncio.subprocess.Process, raced)) + await kill_and_wait(cast(asyncio.subprocess.Process, raced)) assert raced.waited is True finished = FakeProcess(returncode=0) - await _kill_and_wait(cast(asyncio.subprocess.Process, finished)) + await kill_and_wait(cast(asyncio.subprocess.Process, finished)) assert finished.killed is False assert finished.waited is True diff --git a/tests/document_ingestion/test_pdf_inspector_worker.py b/tests/document_ingestion/test_pdf_inspector_worker.py index f09eba5dc..ea8db094c 100644 --- a/tests/document_ingestion/test_pdf_inspector_worker.py +++ b/tests/document_ingestion/test_pdf_inspector_worker.py @@ -13,7 +13,7 @@ import pytest -from basic_memory.document_ingestion import pdf_inspector_worker +from basic_memory.document_ingestion import pdf_inspector_worker, worker_limits from basic_memory.document_ingestion.pdf_inspector import PDF_INSPECTOR_ENGINE, PdfInspectorOutput from basic_memory.document_ingestion.pdf_inspector import PdfInspectorLimits from basic_memory.document_ingestion.raw_document import ( @@ -184,40 +184,40 @@ def fake_posix_resource() -> SimpleNamespace: def test_worker_bounds_linux_address_space(monkeypatch: pytest.MonkeyPatch) -> None: resource = fake_posix_resource() - monkeypatch.setattr(pdf_inspector_worker.sys, "platform", "linux") - monkeypatch.setattr(pdf_inspector_worker, "resource", resource) + monkeypatch.setattr(worker_limits.sys, "platform", "linux") + monkeypatch.setattr(worker_limits, "resource", resource) - pdf_inspector_worker._apply_memory_limit(512 * 1024 * 1024) + worker_limits.apply_memory_limit(512 * 1024 * 1024) resource.setrlimit.assert_called_once_with("RLIMIT_AS", (512 * 1024 * 1024, 512 * 1024 * 1024)) def test_worker_skips_address_space_limit_off_linux(monkeypatch: pytest.MonkeyPatch) -> None: resource = fake_posix_resource() - monkeypatch.setattr(pdf_inspector_worker.sys, "platform", "darwin") - monkeypatch.setattr(pdf_inspector_worker, "resource", resource) + monkeypatch.setattr(worker_limits.sys, "platform", "darwin") + monkeypatch.setattr(worker_limits, "resource", resource) - pdf_inspector_worker._apply_memory_limit(512 * 1024 * 1024) + worker_limits.apply_memory_limit(512 * 1024 * 1024) resource.setrlimit.assert_not_called() def test_worker_bounds_cpu_time(monkeypatch: pytest.MonkeyPatch) -> None: resource = fake_posix_resource() - monkeypatch.setattr(pdf_inspector_worker, "resource", resource) + monkeypatch.setattr(worker_limits, "resource", resource) - pdf_inspector_worker._apply_cpu_limit(25) + worker_limits.apply_cpu_limit(25) resource.setrlimit.assert_called_once_with("RLIMIT_CPU", (25, 26)) def test_worker_skips_rlimits_without_posix_resource(monkeypatch: pytest.MonkeyPatch) -> None: """Windows has no rlimits; the parent's deadline is the only ceiling there.""" - monkeypatch.setattr(pdf_inspector_worker, "resource", None) - monkeypatch.setattr(pdf_inspector_worker.sys, "platform", "linux") + monkeypatch.setattr(worker_limits, "resource", None) + monkeypatch.setattr(worker_limits.sys, "platform", "linux") - pdf_inspector_worker._apply_cpu_limit(25) - pdf_inspector_worker._apply_memory_limit(1024) + worker_limits.apply_cpu_limit(25) + worker_limits.apply_memory_limit(1024) def run_worker_main( @@ -226,10 +226,10 @@ def run_worker_main( applied: list[tuple[str, int]] = [] monkeypatch.setattr(pdf_inspector_worker, "pdf_inspector", engine) monkeypatch.setattr( - pdf_inspector_worker, "_apply_memory_limit", lambda n: applied.append(("memory", n)) + pdf_inspector_worker, "apply_memory_limit", lambda n: applied.append(("memory", n)) ) monkeypatch.setattr( - pdf_inspector_worker, "_apply_cpu_limit", lambda n: applied.append(("cpu", n)) + pdf_inspector_worker, "apply_cpu_limit", lambda n: applied.append(("cpu", n)) ) monkeypatch.setattr(sys, "stdin", SimpleNamespace(buffer=io.BytesIO(b"%PDF-test"))) monkeypatch.setattr( diff --git a/tests/markdown/test_markdown_utils.py b/tests/markdown/test_markdown_utils.py index 2d0a568a8..854928a71 100644 --- a/tests/markdown/test_markdown_utils.py +++ b/tests/markdown/test_markdown_utils.py @@ -8,6 +8,7 @@ from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Observation from basic_memory.markdown.utils import entity_model_from_markdown from basic_memory.models import Entity +from basic_memory.schemas.document import derive_document_note_external_id class TestEntityModelFromMarkdown: @@ -71,6 +72,31 @@ def test_existing_entity_preserves_external_id(self): # Should preserve the existing external_id assert entity.external_id == existing_external_id + @pytest.mark.parametrize("existing_id", [None, "12345678-1234-1234-1234-123456789012"]) + def test_generated_document_identity_comes_from_its_source( + self, existing_id: str | None + ) -> None: + source_id = "11111111-1111-1111-1111-111111111111" + markdown = self._create_markdown(note_type="document") + markdown.frontmatter.metadata.update( + {"schema": "schema/document-extraction", "source": {"entity_external_id": source_id}} + ) + existing = Entity(external_id=existing_id) if existing_id else None + + result = entity_model_from_markdown(Path("source.pdf.md"), markdown, entity=existing) + + assert result.external_id == (existing_id or derive_document_note_external_id(source_id)) + + @pytest.mark.parametrize("source", [None, {}, {"entity_external_id": "invalid-uuid"}]) + def test_generated_document_rejects_invalid_source_identity(self, source: object) -> None: + markdown = self._create_markdown(note_type="document") + markdown.frontmatter.metadata.update( + {"schema": "schema/document-extraction", "source": source} + ) + + with pytest.raises(ValueError): + entity_model_from_markdown(Path("source.pdf.md"), markdown) + def test_entity_with_empty_external_id_gets_new_one(self): """Test that an entity with empty string external_id gets a new UUID.""" markdown = self._create_markdown() diff --git a/uv.lock b/uv.lock index e16b89711..e633fe32c 100644 --- a/uv.lock +++ b/uv.lock @@ -327,6 +327,9 @@ dependencies = [ ] [package.optional-dependencies] +documents = [ + { name = "markitdown", extra = ["docx", "pptx"] }, +] milvus = [ { name = "pymilvus" }, { name = "pymilvus", extra = ["milvus-lite"], marker = "sys_platform != 'win32'" }, @@ -346,6 +349,7 @@ dev = [ { name = "icecream" }, { name = "libcst" }, { name = "logfire" }, + { name = "markitdown", extra = ["docx", "pptx"] }, { name = "pdf-inspector" }, { name = "psycopg" }, { name = "pyright" }, @@ -381,6 +385,7 @@ requires-dist = [ { name = "logfire", specifier = ">=4.19.0" }, { name = "loguru", specifier = ">=0.7.3" }, { name = "markdown-it-py", specifier = ">=3.0.0" }, + { name = "markitdown", extras = ["docx", "pptx"], marker = "extra == 'documents'", specifier = ">=0.1.7,<0.2" }, { name = "mcp", specifier = ">=2,<3" }, { name = "mdformat", specifier = ">=0.7.22" }, { name = "mdformat-frontmatter", specifier = ">=2.0.8" }, @@ -412,7 +417,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32'", specifier = ">=0.21.0" }, { name = "watchfiles", specifier = ">=1.0.4" }, ] -provides-extras = ["milvus", "pdf", "redis"] +provides-extras = ["documents", "milvus", "pdf", "redis"] [package.metadata.requires-dev] dev = [ @@ -422,6 +427,7 @@ dev = [ { name = "icecream", specifier = ">=2.1.3" }, { name = "libcst", specifier = ">=1.8.6" }, { name = "logfire", specifier = ">=4.19.0" }, + { name = "markitdown", extras = ["docx", "pptx"], specifier = ">=0.1.7,<0.2" }, { name = "pdf-inspector", specifier = ">=0.2.6,<2" }, { name = "psycopg", specifier = ">=3.2.0" }, { name = "pyright", specifier = ">=1.1.408" }, @@ -448,6 +454,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl", hash = "sha256:d16c9bbc61ea14637596c5f6fbff2ee99cbe3573e46a716401734ef50c3060c2", size = 1333658, upload-time = "2025-12-13T06:50:28.266Z" }, ] +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + [[package]] name = "cachetools" version = "7.1.4" @@ -658,6 +677,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] +[[package]] +name = "cobble" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/7a/a507c709be2c96e1bb6102eb7b7f4026c5e5e223ef7d745a17d239e9d844/cobble-0.1.4.tar.gz", hash = "sha256:de38be1539992c8a06e569630717c485a5f91be2192c461ea2b220607dfa78aa", size = 3805, upload-time = "2024-06-01T18:11:09.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/e1/3714a2f371985215c219c2a70953d38e3eed81ef165aed061d21de0e998b/cobble-0.1.4-py3-none-any.whl", hash = "sha256:36c91b1655e599fd428e2b95fdd5f0da1ca2e9f1abb0bc871dec21a0e78a2b44", size = 3984, upload-time = "2024-06-01T18:11:07.911Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -829,6 +857,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/7c/2e5dcf53909deddd0bf38cbe277ad9806be038276b1c6c436561b4d9b2e2/dateparser-1.4.1-py3-none-any.whl", hash = "sha256:f25d4e051a84be27a35bd297e3e1dc59ff78373701b89be352ba80372d22d0d0", size = 300503, upload-time = "2026-06-15T08:45:45.951Z" }, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + [[package]] name = "detect-installer" version = "0.1.0" @@ -2032,6 +2069,134 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/f0/92f2d609d6642b5f30cb50a885d2bf1483301c69d5786286500d15651ef2/lsprotocol-2025.0.0-py3-none-any.whl", hash = "sha256:f9d78f25221f2a60eaa4a96d3b4ffae011b107537facee61d3da3313880995c7", size = 76250, upload-time = "2025-06-17T21:30:19.455Z" }, ] +[[package]] +name = "lxml" +version = "6.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/ad/28ecd7cb894d172f3c9c80a075eeeb2017ac62e3632cee05a5f9493547eb/lxml-6.1.3.tar.gz", hash = "sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21", size = 4211198, upload-time = "2026-09-02T14:48:02.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/1f/a180b57d9eeabaab77f9d5aa30356898ea749c4795596a8f66d1eb6bef2e/lxml-6.1.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c0710ac085a157b593c38fbcacd950f15c4afa8e2057527185875ab302752bc", size = 8602094, upload-time = "2026-09-02T14:47:26.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/25/070c92013a1c029a602b03560d68772313d918268667fa993da7961759c9/lxml-6.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:623c8799c17128753c65699f1c3aa32402657393a9ad6db09ed8b98ddf76611d", size = 4638308, upload-time = "2026-09-02T14:47:29.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1c/722e88883173097a1a375153e3c2447eba3060d0231522cf6596e99f4195/lxml-6.1.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f683dc6300317700025e41d89a43e0276692ded16113a3c43eab704d605c58e5", size = 4939696, upload-time = "2026-09-02T14:47:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/db/36/aa413bc214dc4f785ad2b2ddd8cc99aae7062d49ab155e91e6011af00daf/lxml-6.1.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11", size = 5105247, upload-time = "2026-09-02T14:47:36.734Z" }, + { url = "https://files.pythonhosted.org/packages/a3/a0/a1f7f1313795bfec67b77f01ef3b1128d49f2d7f66a8413fa55d47f4e25f/lxml-6.1.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b37772102d44bb6628186accca3a121b1fa3a6b3d97518a8c29a5229ca4c0d0a", size = 5011915, upload-time = "2026-09-02T14:47:39.846Z" }, + { url = "https://files.pythonhosted.org/packages/b9/78/840e7e3f1d0cc7a5cfac5d8505b97e25b6427fd774ac4bae672aaebfb4b5/lxml-6.1.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddcf547bea2aee967d6a77779376a45e77e610e8465147a1f3d7e20d539d6e32", size = 5638175, upload-time = "2026-09-02T14:47:43.644Z" }, + { url = "https://files.pythonhosted.org/packages/0a/20/e022dbc6b4753a9bc9fc5fb28a27163430c1731b9913997f6544c1b2518c/lxml-6.1.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c", size = 5244675, upload-time = "2026-09-02T14:47:47.635Z" }, + { url = "https://files.pythonhosted.org/packages/99/83/82cde81d2b5eb38d1539fdfdf318abdd014a7e604f4df01c9cd3deb18f2a/lxml-6.1.3-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:a5c18810318303ce9afb3f95e2ddb54834f96fa699a8600433fd5a93dcf44c56", size = 5358205, upload-time = "2026-09-02T14:47:50.306Z" }, + { url = "https://files.pythonhosted.org/packages/d2/a1/f3b057371c8cb29f2a9c9c44ea320592446e40b74a4b0af68c3d8e65bc73/lxml-6.1.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:3e42265103fb385d8642a78672edf376c6f7e1d3598a7a4f9cb1278f2f6b5f6f", size = 4704495, upload-time = "2026-09-02T14:47:53.251Z" }, + { url = "https://files.pythonhosted.org/packages/1a/a4/230eb28be5d412152ffc3c679b51fe1aeede5a53f3a8eb6e9748f2f4754f/lxml-6.1.3-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:21402998e4b78e7cce237d2788841aaa21ac9a4d1574d04dc2d12ee41ae807b5", size = 5255117, upload-time = "2026-09-02T14:47:55.963Z" }, + { url = "https://files.pythonhosted.org/packages/a3/18/1969f56763af24ce42ea156007b0b2d73fddea552e283b2010416394f0f4/lxml-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:38fc4e4e4e084e0bd491949482527d406788045c546d4f8789e93fc527b91385", size = 5054424, upload-time = "2026-09-02T14:47:58.131Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/2a90acc1f6fabaa3a8db9340437822bd8d041b205d626a4b3e8621aaa390/lxml-6.1.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5609efdb0d3c95499c00046bc53648b3482ec2175b5503d6e611b3f0555dc71d", size = 4785572, upload-time = "2026-09-02T14:48:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/a5/1e/b90e845b1dcd0f2f3f26b98283d857f25909223aacd265eee032c34ab8b1/lxml-6.1.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:97ce49699d87ebf8aad631b55d65b33219a4f1bfefbbf5bff19dc9af160aeaf9", size = 5656516, upload-time = "2026-09-02T14:48:03.419Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ab/0a1b802c57f3fba5c4efd77d5c6b78adaa8f7b681f0c90456b140fe8bf6c/lxml-6.1.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:48542c9acba9ff9450bd18d871d2c2c8787fdb283572b623d206f1b927cd7d9e", size = 5245982, upload-time = "2026-09-02T14:48:06.109Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/2c016fbceb3778137459292538d9dfa7e3ad9070fe409c15254ddd90d2cc/lxml-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c55e71a9b1db1f107efb60da49c093689b74c5c31a708e5379e2fd9439d4fbb5", size = 5267340, upload-time = "2026-09-02T14:48:08.374Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b1/736d18fd6f0835761923b7bac1f0c27d60c1200384e9093f05d8c5100525/lxml-6.1.3-cp312-cp312-win32.whl", hash = "sha256:b3ff39654f0ce6ebd4db154211136dbe7e8157bcc3bed2344c87f32c7c6ecb6c", size = 3602606, upload-time = "2026-09-02T14:48:10.384Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5b/6ed903e4e6278a020c8a6f0dbbe78030d041840a6b4a64ea441a1e414077/lxml-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:3e9a00d1c2c30936f7add097c41afc5da6556c580909104aafd382cac92a855c", size = 4005999, upload-time = "2026-09-02T14:48:12.51Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1b/7bcebb7b6332cb3ae85e9c13b139adb6f23f75c71d84041c56a5005d9a29/lxml-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:1aeca87830c4fe649dcf93fe2b059525b71c72587f21be4ae4af7103082a79fa", size = 3666631, upload-time = "2026-09-02T14:48:14.567Z" }, + { url = "https://files.pythonhosted.org/packages/52/05/3ef45db776baea068044c799bbba68f3ca00a440c0e930a17c572f3d9639/lxml-6.1.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:3a48093cdb058a93af842ede9703520e810b05dcd0fc6d7190a06376c3bfb6bd", size = 8590357, upload-time = "2026-09-02T14:48:17.413Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a5/eee2fc77eee5ea68e4a4334b1def1781a3beaeefd3d98e81b4a38dc447b7/lxml-6.1.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:887c021d9a977cff89cb273047c1352997b772a8908a25c21836861f69b92be1", size = 4632616, upload-time = "2026-09-02T14:48:20.745Z" }, + { url = "https://files.pythonhosted.org/packages/35/42/df27b56848acd29d8a720acc28977911aab36f2a09df4208d5502e887415/lxml-6.1.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:611a51e61c92f62345a50b0035df6fc0d678f9299f33728826d831598862f59d", size = 4936186, upload-time = "2026-09-02T14:48:22.94Z" }, + { url = "https://files.pythonhosted.org/packages/ab/8d/8a7b91df0b54d09d25f5f44885d6b3e0a6d6643a8c070191580318d20c42/lxml-6.1.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b477912f42c5c33405a10c759d22f80cf5af043ae02d95b9d8e5e5bc555739ed", size = 5093324, upload-time = "2026-09-02T14:48:25.132Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7e/8f340ddcd43790332fb0de8a26628d571a492da3300cd191821698407c96/lxml-6.1.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cffe18571ccc51d742cd08cbb3f8b756de9311d18c7ea98f5d92f37b8fb60c2", size = 4998850, upload-time = "2026-09-02T14:48:27.394Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c1/9c5bb572f1f09ec9e4322bd4a4e9f4ad48347fc56ef94cf4df58a5279dc8/lxml-6.1.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:75cc6569e86be5785b6188ef1642670c6adbc984e81ec35e224842ecd9eefcc8", size = 5626813, upload-time = "2026-09-02T14:48:29.61Z" }, + { url = "https://files.pythonhosted.org/packages/ac/7d/8bf1fd8bae8247743968bb76d027a1ac5bd2c4b44495fba6a71b30d10706/lxml-6.1.3-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d85dfab42dd672f87a7f76e9de7172962aee69fa12044f0d6e1a23cbd53fb80e", size = 5232385, upload-time = "2026-09-02T14:48:31.969Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/6cef69ed81cb7df0d03b0dd09d08e6e2cf5061a743ff6f42f0b741548e9b/lxml-6.1.3-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:42632b4024ab24a6b488f559ac851312509888b6b80ae2aa11cf29a646a0d245", size = 5347088, upload-time = "2026-09-02T14:48:34.13Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e1/8e5fd8ddc8c7d685badb0f2db149e3c9da84eefc2827c01c658df2c4e3cb/lxml-6.1.3-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:febd35ef45f603c2d74b74655efdbf45e14f55fc0aef4ac82b663ca829b283e0", size = 4707227, upload-time = "2026-09-02T14:48:36.62Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7e/00041382a11be40a88bf405ebff11c8efabd3de79f2691e1638b1c47a8a0/lxml-6.1.3-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a43b3bdf11e477dc7770609d3477316f974354dfc8425d596f64f471cc8daf6e", size = 5240208, upload-time = "2026-09-02T14:48:38.893Z" }, + { url = "https://files.pythonhosted.org/packages/fd/fe/316538b5cff0936fa63d45d421c655730fcbb5a28dcac728c175083002bc/lxml-6.1.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d582042c69857c364e8153de6e18e0da9b7b515a6a8113caf69a6ec8e0520f2", size = 5050271, upload-time = "2026-09-02T14:48:41.213Z" }, + { url = "https://files.pythonhosted.org/packages/c9/91/455bcccb3ac725373007344d351151810cd19762d1673b64b811f4359a42/lxml-6.1.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8e49a646acfab83c68974f4aa1d0a2acca9e88d7d627ae0fc13201b14b76d310", size = 4780433, upload-time = "2026-09-02T14:48:43.779Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f6/580440e2f52cf00bba5c5e1080bfa88cdfcde73be71a11d95170ddbb663f/lxml-6.1.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0dee106e9aa97fb00541b1ed7827070564d0549c3d3fba8920e6b20fd980f748", size = 5645928, upload-time = "2026-09-02T14:48:46.187Z" }, + { url = "https://files.pythonhosted.org/packages/f6/dc/d123c1f244306543d545f62443f794959e4f1ea709fe100f8740d514e74a/lxml-6.1.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:dd5e90f34cffcfed97f36cf066325773d2b6021c60c29942e53a18b028501b1d", size = 5231184, upload-time = "2026-09-02T14:48:48.691Z" }, + { url = "https://files.pythonhosted.org/packages/c3/3c/fe55b2bd5c6113c906511cd88f6a470195c5fbff1124f19970ab706c3477/lxml-6.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9b3e7d71bf6acff341233417abbdface29c647e3113892d9aaedc02eb4aa2bc", size = 5255814, upload-time = "2026-09-02T14:48:50.948Z" }, + { url = "https://files.pythonhosted.org/packages/e7/a7/485df55acf55dc35e4ca89d2f48f03889e5a3241826b18b85102b32ce9d8/lxml-6.1.3-cp313-cp313-win32.whl", hash = "sha256:160fcf381f76c3aeac28a756bec44f48942a8f7245a87aa28e3a523b4d90cd87", size = 3602214, upload-time = "2026-09-02T14:48:53.236Z" }, + { url = "https://files.pythonhosted.org/packages/c0/28/e46a7702bd95e9043291f7c3539b6184cba66f96cea9936f20939b284eeb/lxml-6.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:e477aca0bc0d19f3b4ae9e4f2a1cfd687c31bf772d78734910658186b40b2477", size = 4004091, upload-time = "2026-09-02T14:48:55.699Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1d/154c78e20479a43916e63f19cb720d83f44f024b03228be44c92d9a97b24/lxml-6.1.3-cp313-cp313-win_arm64.whl", hash = "sha256:b1cc980905221a5d8b3c476330730b3adb40ff80add71ffbdb6215ba055656f1", size = 3665468, upload-time = "2026-09-02T14:48:57.703Z" }, + { url = "https://files.pythonhosted.org/packages/0c/15/fc75a70b0af6021d0ea16811f1fc71cc42cd06ce90fe10f007a69b2eed84/lxml-6.1.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2bec13085dc8ef48a3fe62f7dfcacfeda2c785cdf19cc8eeda2bb9ed081da165", size = 8609725, upload-time = "2026-09-02T14:49:00.156Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/398fcf9018f881ec9aeaafae1ddd6586dfb13314a35d35e899de373dcae0/lxml-6.1.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4f4db7c7e954d289d71878938348b3d91b904a3e8210a11939359fb758a58e7d", size = 4639629, upload-time = "2026-09-02T14:49:02.81Z" }, + { url = "https://files.pythonhosted.org/packages/a7/2d/49b6a6ad7ce8f64b07b9fe852ff0c6d3fcbb26db61bee4f63d4120180a1c/lxml-6.1.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2cae5d5c90a62d9139c512a0cb1aad1d182b022b5740daea2617eb5bf7fc658e", size = 4965074, upload-time = "2026-09-02T14:49:05.133Z" }, + { url = "https://files.pythonhosted.org/packages/66/bc/6230cf80e4331c33383b0b6b73dc31a393dd76edd4cb73d761de5123034d/lxml-6.1.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c6c0c13128a32eb04a51357e56a094e13aa8e6d3d1884de2e9ae923f6915e1a8", size = 5099355, upload-time = "2026-09-02T14:49:07.343Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cf/d1143d9b7717e07a82f158a1fc9ce6e581fdad1226734950af869e3ffde4/lxml-6.1.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2221e88679d1351e9a40aaee54bc65679b9795bbd0160bc3d5e36b163344eb75", size = 5036795, upload-time = "2026-09-02T14:49:09.65Z" }, + { url = "https://files.pythonhosted.org/packages/31/6f/194bb00ffb89712c30f5a7e1b8e685590e140fad6c8261fec172c09a3dc0/lxml-6.1.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cfb398886a7eb4c719161c3efcff2a1248febc53a4d8e5072d2d8a87fed84ac9", size = 5658740, upload-time = "2026-09-02T14:49:11.9Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/27e3cee3dcdb3b7bc09727b642bdbfcd098490ea77df04611db9060d7722/lxml-6.1.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7eb78ba28b187e1e9203a55c60fcf70df2d22cb205fe6d51b9383d6097419f0", size = 5245991, upload-time = "2026-09-02T14:49:14.154Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/8312560579fc980bbd2233a8a673cc46f7d613d3633f2bf08a21e8f4ad13/lxml-6.1.3-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:ea6b1e9105b4b24a34c722432d9fb578f9ed83af21fa1abda639011e0f22bbb6", size = 5354136, upload-time = "2026-09-02T14:49:16.459Z" }, + { url = "https://files.pythonhosted.org/packages/74/d8/eda60f4f73a9c780b5d6e1175484f66e6c81a2c93346e2906a1fec9c7a02/lxml-6.1.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:e8b17e23df3e827a69d25af70990ca2420e92668aaffaeeb3cd2351d7916a023", size = 4704379, upload-time = "2026-09-02T14:49:19.032Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c8/c9cc60057be78ac34bd2b842e45e6e88edbfe5e532e82c3b82381b7aab49/lxml-6.1.3-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b7c37339d7e75cab9a123a04248e243cefefb302ad6db566ea0c77cbcde421e", size = 5258676, upload-time = "2026-09-02T14:49:21.306Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/66894008fee8d1785b8db129747ae963fd427b68f456918df7f2f24a8b98/lxml-6.1.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:83e3a51e7933db700a0da0db31849db3a24022d9970da9bb73001e1d0326fd92", size = 5090069, upload-time = "2026-09-02T14:49:23.562Z" }, + { url = "https://files.pythonhosted.org/packages/8b/31/c1b60404859f4c3cd1f41f29c65a24e25cea78fde822d9574a21f66810be/lxml-6.1.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9bde9ae026a55b9a192078dfa6e27dd0ca4a050171ab6272e92f97b757dfdf48", size = 4741958, upload-time = "2026-09-02T14:49:26.037Z" }, + { url = "https://files.pythonhosted.org/packages/23/b8/6285f0cf546f14da2554cabdeaf7c2c2ff3190c74807f0de2e8810a786f9/lxml-6.1.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1a635e837b50a1819bebfedaac5916498ea024120969da8790500148fb0a894d", size = 5683245, upload-time = "2026-09-02T14:49:28.438Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f6/2168cab44336dcb15fed0f0b78577225b83297cdf0dee349c95420c3dcb0/lxml-6.1.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d0c5c362bc94f1929dc7e96e715bbe7bd17037f802e6d8f0d1545df9133c0559", size = 5246087, upload-time = "2026-09-02T14:49:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/f5/89/32f5de69a0a31f30e6164981851f87b37ecb2c4ee838e504b88d49d4818e/lxml-6.1.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c59e4265608da6a041f54646ecc0c9ecdbb19aaf14c4c684bb6c2114998cc415", size = 5269352, upload-time = "2026-09-02T14:49:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a1/741d952ed3a7ef7a50055c6415aec3f067015e97f72f4389ce77b09657ba/lxml-6.1.3-cp314-cp314-win32.whl", hash = "sha256:2e62c569ec7531b679b184cbfe335c501c1d13c4b363560013019962eb630e6d", size = 3662783, upload-time = "2026-09-02T14:50:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bc/5811cc73cac05e324e05ba9b0924e1a163a317a167ede8a9c748b11db30a/lxml-6.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:66299564c046bc7e0cc5de5106601eae907e9fa5904cd68a323380a8502f7861", size = 4073951, upload-time = "2026-09-02T14:50:26.348Z" }, + { url = "https://files.pythonhosted.org/packages/92/18/3768c8b01ac3a9bed1914715e6011711b00e2a11628ffa6f7fa37f8e0269/lxml-6.1.3-cp314-cp314-win_arm64.whl", hash = "sha256:ebd054ad1737a68fb7c5c073d405cef2b88bb824e294de3b4a4e995b47f0e376", size = 3749279, upload-time = "2026-09-02T14:50:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/72/38/84684784738d9451db2b330de2483f496690c3a5c642071df24135739b37/lxml-6.1.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5a143e6207579de8baeded4eaac9134413200359f1969d636f0bfb98ee8c3c8f", size = 8860296, upload-time = "2026-09-02T14:49:36.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/b7/fc4c50bb1b38e864010ea396046cabe85129bf9e65b11edcfbc37d356241/lxml-6.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a1cec0f99b9b914d39176347a93b7610dc09324491aee1cbc57cd291a41a1d55", size = 4755190, upload-time = "2026-09-02T14:49:39.872Z" }, + { url = "https://files.pythonhosted.org/packages/94/e2/ee9aa6ed2b666b2db1f6f7fd48964ff9da39ebe827ef5eac0ab881f639d9/lxml-6.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f6b9d2aad499c769ee8287609ab0e6de99d8bcea99c6e6c2e64945259fd52fb2", size = 4979517, upload-time = "2026-09-02T14:49:42.153Z" }, + { url = "https://files.pythonhosted.org/packages/29/e3/e7763d1661b283ddd4fa36f91b9a497db6b8d2aff55028b16c7f642e0755/lxml-6.1.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a23fefdb345b2d4d0ff2860571b5ff9a89a28b6a120f720e8fb0324d346626", size = 5115270, upload-time = "2026-09-02T14:49:44.493Z" }, + { url = "https://files.pythonhosted.org/packages/2d/cd/22205d5b4d177e3f4156f780412426ee7c7f8107809f119f0dcc40fa51e3/lxml-6.1.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:545ccc14fb05485f48b4439ec35beb16d5b5280eb6c81c658bd4707a2a119414", size = 5032449, upload-time = "2026-09-02T14:49:46.841Z" }, + { url = "https://files.pythonhosted.org/packages/da/43/06a4626c3bb79ef8c501b674afab8100d64e798665bb2a97d1c960636a49/lxml-6.1.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:93476b6514b373fc6ca67d26c442784f7807c86f00635bfe79f935c3eab2af17", size = 5603325, upload-time = "2026-09-02T14:49:49.664Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/733682a0c2de9f5779ba207bbb3f3f6be8c6bda863fc01739b186b38783a/lxml-6.1.3-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8db38ff3fb7aee7d6a82ae4da2eef1178656fe1216841fbd24870062a9d60473", size = 5229023, upload-time = "2026-09-02T14:49:52.447Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8a/e69cdaca3fd33a647942925664f01b20908d41a6968c182305be9c38fb11/lxml-6.1.3-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:25f4118c438f96bb466e83108506d03d5c31b1bd2387e83e5b070bda6ded9c37", size = 5317811, upload-time = "2026-09-02T14:49:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b2/0c397588174403c2ab68fc464abf97e03e7324f9c6cb6a99023104707195/lxml-6.1.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:1beb0f9909b26cee938df9ba56b15252a84429b1fc30ce6fca161390b9789a70", size = 4646516, upload-time = "2026-09-02T14:49:57.761Z" }, + { url = "https://files.pythonhosted.org/packages/56/7e/cfea25afafbe49db8b225764f7f74bb37c2a7f5e717d917d3d4a5e098ed4/lxml-6.1.3-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3a27ac6c780c8b8a1cd231b58407634cafc1c4cc28cd6c7141362df0f36351e7", size = 5240626, upload-time = "2026-09-02T14:50:00.279Z" }, + { url = "https://files.pythonhosted.org/packages/a1/75/7a587771bb52ebb0e2c57b6dbe9fd96a70fbb54d72ddd97d54c5f8ec18d5/lxml-6.1.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1932d7ce78a561367512c594fe66eac2b2ec9b9264cfd9b5f950622f4a116e2", size = 5086619, upload-time = "2026-09-02T14:50:03.245Z" }, + { url = "https://files.pythonhosted.org/packages/1e/01/94c0ebe6d831861542d251e038052e52bf6d33f1d18f1cfffdc82851065a/lxml-6.1.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7d0f5976aa2701996f759b30172925829867547bb073af0ae67d1307a0f0262c", size = 4758828, upload-time = "2026-09-02T14:50:05.873Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/938d67bd0e5b1fdfa52be28aefdffbad57e1f6b8e921c2aab88542c75f40/lxml-6.1.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e7ce578aa8a80910a72a8ca0bbea3baae10100827249001999726a788456d8", size = 5627083, upload-time = "2026-09-02T14:50:08.555Z" }, + { url = "https://files.pythonhosted.org/packages/d8/65/4e51522f6c214650db0abb7b16ccd11b1238b8a05a8d59aa4ebed59c9f67/lxml-6.1.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d97c5227621af74b111882a290b10f371780a38eef9d9e730408fba2259b52fb", size = 5235170, upload-time = "2026-09-02T14:50:11.255Z" }, + { url = "https://files.pythonhosted.org/packages/92/c2/e73d19365665f6b16ef84df21199befc3b06e4c539046ad2d9595f6fb9ea/lxml-6.1.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da707f14ea3c35ee463d50acd596d6488e4b2b4ae7cf77a5bf93f55c023d63e8", size = 5252273, upload-time = "2026-09-02T14:50:13.782Z" }, + { url = "https://files.pythonhosted.org/packages/48/a9/7f386c84c9fe2854e1ca6e231c285e1c8f392971ac353c6865e6ec49faff/lxml-6.1.3-cp314-cp314t-win32.whl", hash = "sha256:9efe56a68179f3adc4de41861c9358931db03837c48dd5e1c78077b84dd07f3a", size = 3902712, upload-time = "2026-09-02T14:50:16.171Z" }, + { url = "https://files.pythonhosted.org/packages/82/a6/8a3eb793f7900ef01c7f99e6f5fcbcfbdff35251cfaef66b32a4c16352d6/lxml-6.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c9389b3784b56c58d933b5e0aecdf28f901b073ff385358d8a7d40907f6e14b2", size = 4400979, upload-time = "2026-09-02T14:50:18.621Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c4/3807bea283b4fe9e9d9f5dde46a73df91178472b335d2778e10b2a37aa22/lxml-6.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32a409be3190b088f960ac92bfedfbef2f86c49ff940765e1548177592d20026", size = 3823401, upload-time = "2026-09-02T14:50:21.119Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8e/4614fcd65496054cfb7172662f3576a59200278739506433b8c241ea422a/lxml-6.1.3-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:6ea2f13dce778ca072ccee598bca46a092ce192e8fd907b6c1f0e52c800529a0", size = 8609378, upload-time = "2026-09-02T14:50:31.772Z" }, + { url = "https://files.pythonhosted.org/packages/f2/51/2cdce3c65fa99a6195dd8fbd512d33407c1000ad99f63e0a285b63d7a8eb/lxml-6.1.3-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c581b1d68b3845fb86c6b2983e755b29bf001461c59fa411d2c26a911b6559a9", size = 4640022, upload-time = "2026-09-02T14:50:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/52/09/0b30084e9eb1c546a4be3d9c56df70058d116b1a320400a59b0f7da87bf0/lxml-6.1.3-cp315-cp315-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e01125896585139453cab8cb235893644d8815d7509520da95ae3ee8d1c1f79", size = 5037928, upload-time = "2026-09-02T14:50:37.007Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0e/5c37275a3e361f6138dc06db748ea565c1fe8a5f4ee5e2ddd80047c81a89/lxml-6.1.3-cp315-cp315-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:290f66b97ede0e552e1cb44a0fd8a74f9753ee635b50830a0b122fb72788d015", size = 5661932, upload-time = "2026-09-02T14:50:39.777Z" }, + { url = "https://files.pythonhosted.org/packages/70/c5/b71ffb289b15e2642e2a3cf6d468c44da39ea119061a99e5b05e3d10f217/lxml-6.1.3-cp315-cp315-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73fc05988ed20809450474ba760a87c8ad4e455fc09783c02195e56ec634b41a", size = 5249209, upload-time = "2026-09-02T14:50:42.141Z" }, + { url = "https://files.pythonhosted.org/packages/81/ea/9910da149a23932f9301652e57661cd9e42b0df18f12be21159b7255f92b/lxml-6.1.3-cp315-cp315-manylinux_2_31_armv7l.whl", hash = "sha256:dc3a44689eea43eab836e5c98a8ab015dc2419987d1ea6eafc7c590cdff86bed", size = 4704543, upload-time = "2026-09-02T14:50:44.634Z" }, + { url = "https://files.pythonhosted.org/packages/76/07/9290329cd188c62e22021f79df04ee0cc33d9a93b0d38bd65ccd452ad9d0/lxml-6.1.3-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:209c3ccbfe35a04ac6d24f0611f9d1cbf8025d49991b14acd935236234d6c156", size = 5261298, upload-time = "2026-09-02T14:50:47.301Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0c/aba78bd3401cd99b73a0aed8e2b9b43e14be94fab3603d4bbc8a62365f2a/lxml-6.1.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2f5b2a2b9811b853b39bfa41367c6d78747b8e3e80e07fc5a24aae295c1a4d7d", size = 5090453, upload-time = "2026-09-02T14:50:49.952Z" }, + { url = "https://files.pythonhosted.org/packages/8d/dc/fa4426c3355aa0216cbeb3911495b5f65a26e0df85859a89928fe28f0396/lxml-6.1.3-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:6a406d0b3cb207b0fa460ed4dc93e866f44f105da0169361cb18ff998a44c7f0", size = 4744709, upload-time = "2026-09-02T14:50:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/be/2b/224fe7918658ab7c532ac2412f3c1eb28f71e6364fb07566262d0cc6a7b6/lxml-6.1.3-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:53258656846f5c48996b882fb4b135885e088a3ad3d96b4bc0530f95124d1f69", size = 5685802, upload-time = "2026-09-02T14:50:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/21/44/7d480819b9adcae5f84dd8ac529132c6b7a578544398225cd20321adcd91/lxml-6.1.3-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:aa633613ff907ea91b9b0489a1f0da1b8725d8c6ccec6b77e8a1c9c235044bb0", size = 5249019, upload-time = "2026-09-02T14:50:57.985Z" }, + { url = "https://files.pythonhosted.org/packages/72/83/385a267ea1b6b283f2249dd827ef360a295e9db14e13ef4665a120c60d64/lxml-6.1.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:90f709b9accab6b2e4d14f5c8718203877a0486bcb3afd74d8b539ecd1e961d4", size = 5271886, upload-time = "2026-09-02T14:51:01.667Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0d/f967b0eb172ae876855a402d6d9b11fa86e3e0c89ca9bbfeadf7ffbfa719/lxml-6.1.3-cp315-cp315-win32.whl", hash = "sha256:b4fc6b03b9d9d90557274f571ab30e7fbbfc527955536935d96f98b6817a86e4", size = 3662894, upload-time = "2026-09-02T14:51:45.173Z" }, + { url = "https://files.pythonhosted.org/packages/f4/48/d8a8c4160a29e663109ad520bac2deb37fcd014756d024561e8bc3e611ec/lxml-6.1.3-cp315-cp315-win_amd64.whl", hash = "sha256:33cadd956b667997e4de1635fce9541f2e8ede2038fcde8cf55aa14d571d1bad", size = 4074626, upload-time = "2026-09-02T14:51:47.77Z" }, + { url = "https://files.pythonhosted.org/packages/25/20/3e1395d34d19f9254625d0b567b81cf70d37d3417be074f4d63b94a2be3c/lxml-6.1.3-cp315-cp315-win_arm64.whl", hash = "sha256:8a330c0ee5fa318c7b5cbbaad882baeca3f570357e7eb25ab34bf31008150758", size = 3749495, upload-time = "2026-09-02T14:51:50.663Z" }, + { url = "https://files.pythonhosted.org/packages/8f/c6/7465ffd9c43883526a382df6fa4846c9d8d419214f7effbf65270e795471/lxml-6.1.3-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:0bf5a3e397df2ec4258eb5eea4c1ac6cf013ca1abd04a176903bff20a70021fe", size = 8857677, upload-time = "2026-09-02T14:51:05.109Z" }, + { url = "https://files.pythonhosted.org/packages/ed/eb/1f3a917e299df43c8162c3e6f64fc2cea3bcf277910f35bff5b8e5d39901/lxml-6.1.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:13d22c0d57355366b393936acf6b98a5e0edeadddd3fccbc6a846c50a76b8741", size = 4754522, upload-time = "2026-09-02T14:51:08.137Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f9/f81b4bdb6efb7a596be29603d8758154d00a5f545db9f3cef9d9041c8f64/lxml-6.1.3-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cad7617727a96d189bd6f979d0fadf765198c7934e85f4edaba9bf3ad919a300", size = 5033744, upload-time = "2026-09-02T14:51:10.633Z" }, + { url = "https://files.pythonhosted.org/packages/c8/0f/26d9bfaacb319c86e0eca8a1a0bf1130d36a7afbd318883e23caea63763d/lxml-6.1.3-cp315-cp315t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cae82b5ca24b0c2beedb269f6e2a96f466acd926879ab00ae19f1a65cbf9ffb0", size = 5615269, upload-time = "2026-09-02T14:51:13.357Z" }, + { url = "https://files.pythonhosted.org/packages/5d/90/73675f3f4141350ed65d6fec533b107d4e802c5caa340cf111771edd86e0/lxml-6.1.3-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:69cafd61aea04ebb3502c93c2aaa568b12931ca0802231e0b5de76bf8b6e74bd", size = 5236280, upload-time = "2026-09-02T14:51:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/fd/be/ed260767e7977de463a0f91f3f4fffcab85c0a2a024a21ffe1fa442c2c79/lxml-6.1.3-cp315-cp315t-manylinux_2_31_armv7l.whl", hash = "sha256:dc205732d593118cf701d986f40e9de7801bb2e371cb189ddbda9b7348f4d97e", size = 4650718, upload-time = "2026-09-02T14:51:19.102Z" }, + { url = "https://files.pythonhosted.org/packages/d0/fd/e9839d03b1e767f2725cf7d7d81b80d5f3f9fdc10ad8827e2479311b046e/lxml-6.1.3-cp315-cp315t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88e719b9437f148f7e1465df845c758dd1598618cbea3a2fd1e61a715542f2b2", size = 5243376, upload-time = "2026-09-02T14:51:21.606Z" }, + { url = "https://files.pythonhosted.org/packages/34/a5/4606e347e2788c301f677004aa83e28d24da9fe663a24380122af57be6fc/lxml-6.1.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:40983eabefd13da003e68170928c7acc011f0d095eefce5871a3c71c9385fb9a", size = 5092340, upload-time = "2026-09-02T14:51:24.21Z" }, + { url = "https://files.pythonhosted.org/packages/ea/99/3314a8661cdf30f493c55a87db283961dfaae08451976a2ca418958e1804/lxml-6.1.3-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:fad67b12ffe0f71e02b4932b04883cbc76a9072bbd30731409d3523cf058b011", size = 4758768, upload-time = "2026-09-02T14:51:26.813Z" }, + { url = "https://files.pythonhosted.org/packages/30/58/3bdc577f78ea8b7d72d39a84506f7001d5b28728f43e5b84891e3b7d9a4a/lxml-6.1.3-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6cd11e7550d89e551a87dcec30f04b1fca32e86b68708aa01a4daa455d8605e5", size = 5649546, upload-time = "2026-09-02T14:51:29.453Z" }, + { url = "https://files.pythonhosted.org/packages/6a/e4/652633de1a2395949ebb7a8fc7d089aba12a2b45f0fefbc9d29e3e3ab3cf/lxml-6.1.3-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ca0ec532ad2f5ba1e5ec120ac157769c57f01855b3d8bf37213f5d88abd9ba0a", size = 5234874, upload-time = "2026-09-02T14:51:32.262Z" }, + { url = "https://files.pythonhosted.org/packages/65/a6/c4581d171de30449304b4859bbd3607e9b40da13c0f88b68e6097c8d785e/lxml-6.1.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e99e09ab7741f1281e2677f4c0058c7f5267d182530b09c87e4f6aa26adf3887", size = 5260043, upload-time = "2026-09-02T14:51:34.841Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/ed6ee6186a89e69ca4ea9658b2a278f46a5efe8b5d4db56c7197f18653fe/lxml-6.1.3-cp315-cp315t-win32.whl", hash = "sha256:ace1d2c83b2bd24db5940600541140e87a325e119cb32d5fa9ad720d7e76648e", size = 3901093, upload-time = "2026-09-02T14:51:37.234Z" }, + { url = "https://files.pythonhosted.org/packages/67/9d/11d10257a4a048d04195d638bb61f0246ce2448eb05f682bcbab25a257a8/lxml-6.1.3-cp315-cp315t-win_amd64.whl", hash = "sha256:b49638355ea3bebba70da783ccbc630fd72afa16bc46c54474bfa1f9a915bbc6", size = 4395446, upload-time = "2026-09-02T14:51:39.884Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b7/44edd7de434181c582892e68d1ffe6775ca403ce14aea07cb5a218a936cf/lxml-6.1.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5a721a98c649855963811b59b55755b30566e7f7fc40bdc9803d66dee9f811cf", size = 3822836, upload-time = "2026-09-02T14:51:42.471Z" }, +] + +[[package]] +name = "magika" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "numpy" }, + { name = "onnxruntime" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/b6/8fdd991142ad3e037179a494b153f463024e5a211ef3ad948b955c26b4de/magika-0.6.2.tar.gz", hash = "sha256:37eb6ae8020f6e68f231bc06052c0a0cbe8e6fa27492db345e8dc867dbceb067", size = 3036634, upload-time = "2025-05-02T14:54:18.88Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/07/4f7748f34279f2852068256992377474f9700b6fbad6735d6be58605178f/magika-0.6.2-py3-none-any.whl", hash = "sha256:5ef72fbc07723029b3684ef81454bc224ac5f60986aa0fc5a28f4456eebcb5b2", size = 2967609, upload-time = "2025-05-02T14:54:09.696Z" }, + { url = "https://files.pythonhosted.org/packages/64/6d/0783af677e601d8a42258f0fbc47663abf435f927e58a8d2928296743099/magika-0.6.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9109309328a1553886c8ff36c2ee9a5e9cfd36893ad81b65bf61a57debdd9d0e", size = 12404787, upload-time = "2025-05-02T14:54:16.963Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ad/42e39748ddc4bbe55c2dc1093ce29079c04d096ac0d844f8ae66178bc3ed/magika-0.6.2-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:57cd1d64897634d15de552bd6b3ae9c6ff6ead9c60d384dc46497c08288e4559", size = 15091089, upload-time = "2025-05-02T14:54:11.59Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1f/28e412d0ccedc068fbccdae6a6233faaa97ec3e5e2ffd242e49655b10064/magika-0.6.2-py3-none-win_amd64.whl", hash = "sha256:711f427a633e0182737dcc2074748004842f870643585813503ff2553b973b9f", size = 12385740, upload-time = "2025-05-02T14:54:14.096Z" }, +] + [[package]] name = "mako" version = "1.3.12" @@ -2044,6 +2209,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, ] +[[package]] +name = "mammoth" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cobble" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/3c/a58418d2af00f2da60d4a51e18cd0311307b72d48d2fffec36a97b4a5e44/mammoth-1.11.0.tar.gz", hash = "sha256:a0f59e442f34d5b6447f4b0999306cbf3e67aaabfa8cb516f878fb1456744637", size = 53142, upload-time = "2025-09-19T10:35:20.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/54/2e39566a131b13f6d8d193f974cb6a34e81bb7cc2fa6f7e03de067b36588/mammoth-1.11.0-py2.py3-none-any.whl", hash = "sha256:c077ab0d450bd7c0c6ecd529a23bf7e0fa8190c929e28998308ff4eada3f063b", size = 54752, upload-time = "2025-09-19T10:35:18.699Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -2056,6 +2233,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, ] +[[package]] +name = "markdownify" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/ab/d1297139c0e2ceb151ae564c8c4f57ac0155d8f1f8b4cbd5d6523c82ea36/markdownify-1.2.3.tar.gz", hash = "sha256:1a176f05522c8a2cb1dd3ab9d307dcdadbed5c26ae717855bfc42b3b6d38d937", size = 18852, upload-time = "2026-06-30T20:27:39.06Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/10/fa543d484e8b1199243fe20eedd02cc5af050edebce98a7293a5773df592/markdownify-1.2.3-py3-none-any.whl", hash = "sha256:a189a0bedfd14009030fde5f85bb6f77c56897cb839b5c25315dd7d4e3e290ba", size = 15732, upload-time = "2026-06-30T20:27:38.094Z" }, +] + +[[package]] +name = "markitdown" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "charset-normalizer" }, + { name = "defusedxml" }, + { name = "magika" }, + { name = "markdownify" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/93/e8a4af0c47551beb6383e226e840cbc811a577b8096eb385251b3fcc8f62/markitdown-0.1.7.tar.gz", hash = "sha256:4d1f3c69cd43b82288fdc3653686d759dcf355ee7c681aa6a855aed98a1e4f44", size = 51767, upload-time = "2026-07-29T18:20:31.496Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/16/51d269a754d690ec31d3faa0686c8c14ac955dbc0580c358f256ba3391ec/markitdown-0.1.7-py3-none-any.whl", hash = "sha256:4eca912c87c6aa6897284a7f4bf6769a23bccf8544530f5d8b175fbe3797c916", size = 71093, upload-time = "2026-07-29T18:20:30.226Z" }, +] + +[package.optional-dependencies] +docx = [ + { name = "lxml" }, + { name = "mammoth" }, +] +pptx = [ + { name = "python-pptx" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -3497,6 +3713,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] +[[package]] +name = "python-pptx" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "pillow" }, + { name = "typing-extensions" }, + { name = "xlsxwriter" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/a9/0c0db8d37b2b8a645666f7fd8accea4c6224e013c42b1d5c17c93590cd06/python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095", size = 10109297, upload-time = "2024-08-07T17:33:37.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788, upload-time = "2024-08-07T17:33:28.192Z" }, +] + [[package]] name = "pytz" version = "2026.2" @@ -4030,6 +4261,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "soupsieve" +version = "2.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/99/a6ca3beb3ccacb41fb3321d8a60e5566f9e6467601ef8eba6a17e1b89778/soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74", size = 122445, upload-time = "2026-08-07T00:57:24.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823", size = 37370, upload-time = "2026-08-07T00:57:23.524Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.51" @@ -4629,6 +4869,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, ] +[[package]] +name = "xlsxwriter" +version = "3.2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940, upload-time = "2025-09-16T00:16:21.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315, upload-time = "2025-09-16T00:16:20.108Z" }, +] + [[package]] name = "yarl" version = "1.24.2"