diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 28958ba4..e6dda412 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -38,6 +38,7 @@ jobs: - name: Run unit tests run: > uv run pytest + tests/test_atlascloud_model.py tests/test_batch_api.py tests/test_csv_scraper_multi_graph.py tests/test_depth_search_graph.py @@ -46,4 +47,5 @@ jobs: tests/test_scrape_do.py tests/test_search_graph.py tests/utils/convert_to_md_test.py + tests/utils/output_parser_test.py tests/utils/parse_state_keys_test.py diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..7a15a082 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,136 @@ +# AGENTS.md + +Instructions for AI coding agents (Claude Code, Codex, Cursor, Copilot agents, โ€ฆ) +working on **ScrapeGraphAI**. Human contributors should read +[CONTRIBUTING.md](CONTRIBUTING.md); everything here is in addition to it. + +--- + +## 1. Golden rule: everything goes to `pre/beta` + +**`main` is never written to directly. All work is based on and merged into `pre/beta`.** + +`pre/beta` is the prerelease branch: pushes to it publish a `beta` prerelease via +semantic-release (see `.releaserc.yml`). `main` only receives releases when a +maintainer promotes `pre/beta`. + +```bash +# 1. always start from an up-to-date pre/beta +git fetch origin +git checkout -b feat/my-change origin/pre/beta + +# 2. commit your work +git add +git commit -m "feat(nodes): add X" + +# 3. push and open the PR against pre/beta +git push -u origin feat/my-change +gh pr create --base pre/beta --title "feat(nodes): add X" --body "..." +``` + +Checklist before you commit: + +- [ ] The branch is based on `origin/pre/beta` (`git merge-base --is-ancestor origin/pre/beta HEAD`). +- [ ] The PR base is `pre/beta`, **not** `main`. +- [ ] No commits directly on `main` or `pre/beta`, no force-push to either. +- [ ] One logical change per branch/PR. + +If a task genuinely requires targeting `main` (e.g. a hotfix on a released +version), stop and ask a maintainer first. + +## 2. Environment setup + +Python `>=3.12`, dependencies managed with [uv](https://docs.astral.sh/uv/): + +```bash +uv sync # create the venv and install deps +uv run pre-commit install # install the git hooks +``` + +Never hand-edit `uv.lock`; regenerate it with `uv lock` / `uv sync` and commit +the result only when you actually changed dependencies in `pyproject.toml`. + +## 3. Checks to run before pushing + +```bash +make lint # ruff + black --check + isort --check-only +make type-check # mypy (strict) +make test # pytest with coverage +make pre-commit # run all hooks on all files +``` + +Run at least `make lint` and the tests covering what you touched. Report the +real result: if something fails or you skipped a step, say so in the PR +description instead of implying a clean run. + +Style: PEP 8 + Google Python docstrings, `black` formatting, line length 88. +Match the conventions of the surrounding file rather than introducing new ones. + +## 4. Commit messages + +Commits are parsed by semantic-release (Conventional Commits, `conventionalcommits` +preset), so the message decides the next version number. Use: + +``` +feat: โœจ new feature -> minor bump +fix: ๐Ÿ› bug fix -> patch bump +docs: ๐Ÿ“š documentation +style: ๐Ÿ’… formatting only +refactor: โ™ป๏ธ no behaviour change +perf: โšก performance +test: ๐Ÿงช tests +build: ๐Ÿ“ฆ build system / deps +ci: ๐Ÿค– CI configuration +chore: ๐Ÿงน everything else +``` + +Format: `type(optional-scope): imperative summary`, optional body, and +`BREAKING CHANGE:` in the footer for incompatible changes. Reference issues with +`Fixes #123`. + +## 5. Files agents must not touch + +- `CHANGELOG.md` and the `version` field in `pyproject.toml` โ€” owned by + semantic-release; editing them by hand breaks releases. +- Git tags and release notes on GitHub. +- `.github/workflows/*` โ€” only when the task is explicitly about CI. +- Anything under `htmlcov/`, `coverage.xml`, `.pytest_cache/`, `__pycache__/`: + build artifacts, never commit them. + +Also: never commit secrets. API keys go in a local `.env` (git-ignored) and are +read via `os.getenv`; examples and tests must use placeholders such as +`OPENAI_APIKEY` from the environment. + +## 6. Repository layout + +``` +scrapegraphai/ +โ”œโ”€โ”€ graphs/ # pipelines (SmartScraperGraph, SearchGraph, โ€ฆ) +โ”œโ”€โ”€ nodes/ # single graph steps (FetchNode, ParseNode, GenerateAnswerNode, โ€ฆ) +โ”œโ”€โ”€ models/ # LLM wrappers and token/model metadata +โ”œโ”€โ”€ docloaders/ # loaders (ChromiumLoader, โ€ฆ) +โ”œโ”€โ”€ prompts/ # prompt templates +โ”œโ”€โ”€ helpers/ # shared constants and schemas +โ”œโ”€โ”€ integrations/ # third-party / managed-API integrations +โ””โ”€โ”€ utils/ # utilities (html cleanup, tokenization, โ€ฆ) +examples/ # runnable usage examples, one folder per graph +tests/ # pytest suite, mirrors the package layout +docs/ # documentation sources +``` + +When adding a node or graph, register it in the corresponding `__init__.py` and +add a test under `tests/` next to the existing ones for that layer. New +user-facing features need an entry in `examples/` and, when they change public +behaviour, a docs update. + +## 7. Working style expected from agents + +- Prefer small, reviewable diffs; do not reformat or "clean up" untouched files. +- Do not add dependencies unless the task requires it โ€” say why in the PR. +- Write all commits, PR titles/bodies, issue comments, code comments and + docstrings **in English**. +- Do not delete or rewrite existing tests to make a change pass. +- If a test is already failing on `pre/beta`, mention it rather than silently + fixing unrelated things in the same PR. +- Never commit other people's in-progress work: check `git status` and stage + only the files belonging to your change. diff --git a/CHANGELOG.md b/CHANGELOG.md index 945ee181..6c08454a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,32 @@ +## [2.2.0-beta.6](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.0-beta.5...v2.2.0-beta.6) (2026-08-19) + + +### Bug Fixes + +* **graph:** expose when the 8192 token fallback was used ([470da9d](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/470da9d51d6220a8020ee8a1d91b618d3d622042)), closes [#1121](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1121) +* lazy imports to prevent torchcodec FFmpeg DLL crash on Windows ([#1089](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1089)) ([#1092](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1092)) ([e5c2a42](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/e5c2a4292174bae948731379c88aafb14197ec6c)) +* pop model_tokens so it is not forwarded to the model client ([#1100](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1100)) ([d2b970c](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/d2b970ca77da64b6b6b91fa2353f2e41b9cfe105)) +* **search:** restore SearchGraph by migrating DuckDuckGo backend to ddgs ([#1083](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1083)) ([2139e37](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/2139e37c0addfb9039e3b985bb33801f20e4e05b)), closes [#1082](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1082) [#1082](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1082) +* update MiniMax model metadata and endpoints ([#1103](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1103)) ([e5f8f2b](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/e5f8f2bf008cd7ce7f3d5980bf6f98e6153b2264)) + + +### Docs + +* add AGENTS.md with contribution rules for AI agents ([3b2f986](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3b2f986612bbc70879a0337197a1064cf5875245)) +* **readme:** add Open Source vs Managed API comparison ([#1091](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1091)) ([ef3523b](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/ef3523b1e1c62052b5c0e6b14cebe7933441a7e5)) +* swap Integrations infographic for new API banner; fix CTA links ([71ab440](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/71ab4406867ef02c22349c2dedc18c57d8bdfdec)) + + +### CI + +* **release:** 2.1.3 [skip ci] ([cfb815f](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/cfb815fb0bba3dfc8262a24a7ddab28e22e13893)), closes [#1082](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1082) [#1082](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1082) +* **release:** 2.1.4 [skip ci] ([7fc9c57](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/7fc9c57a0bb861d829f38ed672e35c5ba0e6c79c)), closes [#1089](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1089) [#1092](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1092) +* **release:** 2.1.5 [skip ci] ([c9e0bd0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/c9e0bd036a1f5def8ac92759cda90ba52e3b35e5)), closes [#1100](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1100) [#1091](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1091) [#1095](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1095) +* **release:** 2.1.6 [skip ci] ([27d9d28](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/27d9d289f964844a28006f6d7d1518e293fd49ff)), closes [#1103](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1103) +* **release:** 2.1.7 [skip ci] ([ca112db](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/ca112db4d8b1bd34961e9f1b7e3adf0c7d600c01)), closes [#1121](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1121) +* run only deterministic unit suites in Test Suite workflow ([#1095](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1095)) ([037a42e](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/037a42ed73354c8b3eabd9dcc28236152602ef9e)) +* run the two new deterministic unit suites ([23e3d06](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/23e3d064e3850de222bcfc13907695278ca3bba9)), closes [#1104](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1104) [#1085](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1085) + ## [2.1.7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.1.6...v2.1.7) (2026-08-19) @@ -68,6 +97,44 @@ ## [2.2.0-beta.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.0-beta.1...v2.2.0-beta.2) (2026-06-01) +### Features + +* upgrade MiniMax default model to M3 ([#1080](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1080)) ([1b16c26](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/1b16c268f4e9044c1386ccfaf67b38692b487e5a)) + +## [2.2.0-beta.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.0-beta.4...v2.2.0-beta.5) (2026-06-23) + + +### Bug Fixes + +* lazy imports to prevent torchcodec FFmpeg DLL crash on Windows ([#1089](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1089)) ([2f55377](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/2f55377010fb16c81655aa1c8d08b88daa880797)) + +## [2.2.0-beta.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.0-beta.3...v2.2.0-beta.4) (2026-06-11) + + +### Bug Fixes + +* **nodes:** tolerate doubled-brace JSON output from models like DeepSeek ([#1085](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1085)) ([aaa5d2c](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/aaa5d2cf6d2657f8e8a3b9020ab8b40cdb311c46)) + +## [2.2.0-beta.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.0-beta.2...v2.2.0-beta.3) (2026-06-01) + + +### Bug Fixes + +* **nodes:** update outdated ChatOllama import path to langchain_ollama ([#1076](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1076)) ([e6054cb](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/e6054cbf19a7fe940899ea70b30706f676f86fa7)) + + +### Docs + +* ๐Ÿ“š Standardize and fix links across translated READMEs ([#1074](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1074)) ([458d36a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/458d36a6b83f4a412206cdbe9935a059e9d47f57)) + + +### CI + +* **release:** 2.1.2 [skip ci] ([210c992](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/210c99280048863774fa27053412185a5c18150d)), closes [#1076](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1076) [#1074](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1074) + +## [2.2.0-beta.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v2.2.0-beta.1...v2.2.0-beta.2) (2026-06-01) + + ### Features * upgrade MiniMax default model to M3 ([#1080](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/1080)) ([1b16c26](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/1b16c268f4e9044c1386ccfaf67b38692b487e5a)) @@ -1608,7 +1675,6 @@ Co-Authored-By: Claude Opus 4.6 (1M context) * implement ScrapeGraph class for only web scraping automation ([612c644](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/612c644623fa6f4fe77a64a5f1a6a4d6cd5f4254)) * Implement SmartScraperMultiParseMergeFirstGraph class that scrapes a list of URLs and merge the content first and finally generates answers to a given prompt. ([3e3e1b2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3e3e1b2f3ae8ed803d03b3b44b199e139baa68d4)) -======= ## [1.26.7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.26.6...v1.26.7) (2024-10-19) @@ -3682,7 +3748,6 @@ Co-Authored-By: Claude Opus 4.6 (1M context) * **release:** 1.6.1 [skip ci] ([44fbd71](https://github.com/VinciGit00/Scrapegraph-ai/commit/44fbd71742a57a4b10f22ed33781bb67aa77e58d)) ## [1.6.1](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.6.0...v1.6.1) (2024-06-15) -======= ### Bug Fixes diff --git a/pyproject.toml b/pyproject.toml index 65282ba5..20e58930 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "scrapegraphai" -version = "2.1.7" +version = "2.2.0b6" description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." authors = [ diff --git a/scrapegraphai/nodes/generate_answer_node.py b/scrapegraphai/nodes/generate_answer_node.py index 3c5ca7ed..6f4d2a1f 100644 --- a/scrapegraphai/nodes/generate_answer_node.py +++ b/scrapegraphai/nodes/generate_answer_node.py @@ -9,7 +9,6 @@ from langchain_core.prompts import PromptTemplate from langchain_aws import ChatBedrock from langchain_ollama import ChatOllama -from langchain_core.output_parsers import JsonOutputParser from langchain_core.runnables import RunnableParallel from langchain_openai import ChatOpenAI from requests.exceptions import Timeout @@ -23,7 +22,10 @@ TEMPLATE_NO_CHUNKS, TEMPLATE_NO_CHUNKS_MD, ) -from ..utils.output_parser import get_pydantic_output_parser +from ..utils.output_parser import ( + TolerantJsonOutputParser, + get_pydantic_output_parser, +) from .base_node import BaseNode @@ -148,7 +150,7 @@ def execute(self, state: dict) -> dict: format_instructions = "" else: if not isinstance(self.llm_model, ChatBedrock): - output_parser = JsonOutputParser() + output_parser = TolerantJsonOutputParser() format_instructions = ( "You must respond with a JSON object. Your response should be formatted as a valid JSON " "with a 'content' field containing your analysis. For example:\n" diff --git a/scrapegraphai/utils/output_parser.py b/scrapegraphai/utils/output_parser.py index a9d9ba31..db216e37 100644 --- a/scrapegraphai/utils/output_parser.py +++ b/scrapegraphai/utils/output_parser.py @@ -2,13 +2,53 @@ Functions to retrieve the correct output parser and format instructions for the LLM model. """ -from typing import Any, Callable, Dict, Type, Union +from typing import Any, Callable, Dict, List, Type, Union +from langchain_core.exceptions import OutputParserException +from langchain_core.outputs import Generation from langchain_core.output_parsers import JsonOutputParser from pydantic import BaseModel as BaseModelV2 from pydantic.v1 import BaseModel as BaseModelV1 +def _strip_doubled_braces(text: str) -> str: + """Strip one layer of the doubled braces some models echo from the prompt. + + The default ``format_instructions`` show the expected shape using LangChain's + escaped braces, e.g. ``{{"content": "..."}}``. Strongly instruction-following + models (GPT-4o, etc.) emit single braces, but some models (notably DeepSeek) + copy the doubled braces verbatim, producing ``{{"content": "..."}}`` which is + not valid JSON. This normalizes that single case and is a no-op otherwise. + """ + stripped = text.strip() + if stripped.startswith("{{") and stripped.endswith("}}"): + return stripped[1:-1] + return text + + +class TolerantJsonOutputParser(JsonOutputParser): + """A :class:`JsonOutputParser` tolerant of doubled-brace output. + + Behaviour is unchanged on the happy path: valid JSON is parsed by the parent + parser exactly as before. Only when parsing fails AND the output is wrapped in + doubled braces (``{{ ... }}``) does it retry once with a single layer of braces + removed. This keeps providers like DeepSeek working without altering output for + any model that already returns clean JSON. + """ + + def parse_result(self, result: List[Generation], *, partial: bool = False) -> Any: + try: + return super().parse_result(result, partial=partial) + except OutputParserException: + text = result[0].text + normalized = _strip_doubled_braces(text) + if normalized != text: + return super().parse_result( + [Generation(text=normalized)], partial=partial + ) + raise + + def get_structured_output_parser( schema: Union[Dict[str, Any], Type[BaseModelV1 | BaseModelV2], Type], ) -> Callable: diff --git a/tests/utils/output_parser_test.py b/tests/utils/output_parser_test.py new file mode 100644 index 00000000..0c8186f0 --- /dev/null +++ b/tests/utils/output_parser_test.py @@ -0,0 +1,44 @@ +"""Tests for scrapegraphai.utils.output_parser.TolerantJsonOutputParser.""" + +import pytest + +from scrapegraphai.utils.output_parser import ( + TolerantJsonOutputParser, + _strip_doubled_braces, +) + + +def test_strip_doubled_braces_unwraps_single_layer(): + assert _strip_doubled_braces('{{"content": "hi"}}') == '{"content": "hi"}' + + +def test_strip_doubled_braces_is_noop_for_clean_json(): + text = '{"content": "hi"}' + assert _strip_doubled_braces(text) == text + + +def test_strip_doubled_braces_ignores_unbalanced(): + text = '{{"content": "hi"}' + assert _strip_doubled_braces(text) == text + + +def test_tolerant_parser_parses_clean_json_unchanged(): + parser = TolerantJsonOutputParser() + assert parser.parse('{"content": "hi"}') == {"content": "hi"} + + +def test_tolerant_parser_recovers_doubled_braces(): + """Models such as DeepSeek echo the prompt's escaped braces verbatim.""" + parser = TolerantJsonOutputParser() + assert parser.parse('{{"content": "hi"}}') == {"content": "hi"} + + +def test_tolerant_parser_recovers_doubled_braces_with_whitespace(): + parser = TolerantJsonOutputParser() + assert parser.parse(' {{"content": "hi"}} ') == {"content": "hi"} + + +def test_tolerant_parser_still_raises_on_irrecoverable_output(): + parser = TolerantJsonOutputParser() + with pytest.raises(Exception): + parser.parse("this is not json at all")