diff --git a/CHANGELOG.md b/CHANGELOG.md index d60123b..0f9652a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/) +## [Unreleased] + +### Changed + +- Expanded task and parameter documentation to describe port/value shapes and the Jinja + templating edge cases around missing input + +### Fixed + +- GraphQL query plugin no longer converts non-ASCII characters to unicode escape sequences in + the JSON written to the target dataset +- Fixed a related upload bug this uncovered: the target dataset write passed a text stream to + an API that expects a byte stream, which silently truncated the uploaded file whenever the + JSON contained multi-byte UTF-8 characters (masked previously only because escaped output + was always pure ASCII) +- Query variables parameter description no longer renders its example as an indented code + block due to leftover indentation in the source string + ## [6.0.0] 2026-09-02 ### Changed diff --git a/cmem_plugin_graphql/workflow/graphql.py b/cmem_plugin_graphql/workflow/graphql.py index 4210294..dd69469 100644 --- a/cmem_plugin_graphql/workflow/graphql.py +++ b/cmem_plugin_graphql/workflow/graphql.py @@ -33,14 +33,37 @@ @Plugin( label="GraphQL query", - description="Executes a custom GraphQL query to a GraphQL endpoint" - " and saves result to a JSON dataset.", - documentation="""This workflow task performs GraphQL operations by sending - queries, mutations, and variables over operations. Allows for customization - in the GraphQL query using, Jinja queries and Jinja variables, which can be - obtained from entities. The result of the query is saved as a JSON document - in a pre-created JSON dataset. - """, + description="Sends a GraphQL query or mutation to an endpoint and returns the result," + " or writes it to a JSON dataset.", + documentation="""This task sends a GraphQL query or mutation to an endpoint and +captures the response. + +An input port accepts entities, but it only changes anything when **Query** or +**Query variables** actually contains Jinja syntax: the query and variables are +then rendered once per input entity and the endpoint is called once per entity, +with a failed entity logged and skipped rather than failing the whole task. A +purely static query and variables text runs exactly once and ignores any +connected input entirely. + +When **Target JSON Dataset** is left empty, the collected response(s) become +entities returned on the output port, one per query execution. When it is set, +the output port disappears instead and the same responses are written there as +a single JSON array. + +The task typically starts a chain that begins at a GraphQL API and lands the +result either in a downstream transform, via the output port, or in a JSON +dataset for later use. + +A Jinja-templated **Query** or **Query variables** is never checked for GraphQL +syntax errors until it is actually rendered - a mistake in it only surfaces at +runtime, as a failed entity, rather than as a configuration error when the task +is set up. If **Query** contains Jinja syntax and no input is connected at all, +the unrendered `{{ ... }}` text is sent to the GraphQL library as literal +syntax and the task fails outright. If only **Query variables** contains Jinja +syntax while **Query** is static, and no input is connected, the task instead +sends nothing and completes as if zero entities were processed, without +warning that the variables were never rendered. +""", parameters=[ PluginParameter( name="graphql_url", @@ -61,6 +84,9 @@ GraphQL is a query language for APIs and a runtime for fulfilling those queries with your existing data. Learn more on GraphQL [here](https://graphql.org/). +May also contain Jinja syntax (e.g. `{{ id }}`), which is rendered against each +input entity before the query is sent. + Example Query: query allFruits { fruits { id @@ -81,15 +107,19 @@ label="Query variables", description="""Pass dynamic variables when making a query or mutation. - Example Variables: {"id" : 1} - """, +May also contain Jinja syntax (e.g. `{"id": {{ id }}}`), which is rendered +against each input entity before the query is sent. + +Example Variables: `{"id" : 1}` +""", default_value="{}", param_type=MultilineStringParameterType(), ), PluginParameter( name="graphql_dataset", label="Target JSON Dataset", - description="The Dataset where this task will save the JSON results.", + description="The JSON dataset the result is written to. When set, the output port" + " is removed and the result is only available in the dataset.", param_type=DatasetParameterType(dataset_type="json"), advanced=True, default_value="", @@ -211,7 +241,7 @@ def execute(self, inputs: Sequence[Entities], context: ExecutionContext) -> Enti if dataset_id: write_to_dataset( dataset_id, - io.StringIO(json.dumps(payload, indent=2)), + io.BytesIO(json.dumps(payload, indent=2, ensure_ascii=False).encode("utf-8")), context=context.user, ) diff --git a/cmem_plugin_graphql/workflow/utils.py b/cmem_plugin_graphql/workflow/utils.py index 0898b33..b750c81 100644 --- a/cmem_plugin_graphql/workflow/utils.py +++ b/cmem_plugin_graphql/workflow/utils.py @@ -1,17 +1,9 @@ """Utils module""" -import json -import uuid from collections.abc import Iterator -from typing import Any import jinja2 -from cmem_plugin_base.dataintegration.entity import ( - Entities, - Entity, - EntityPath, - EntitySchema, -) +from cmem_plugin_base.dataintegration.entity import Entities def get_dict(entities: Entities) -> Iterator[dict[str, str]]: @@ -30,39 +22,3 @@ def is_jinja_template(value: str) -> bool: template = environment.from_string(value) res = template.render() return res != value - - -def get_entities_from_list(data: list[dict[str, Any]]) -> Entities: - """Generate entities from list""" - paths: list[str] = [] - unique_paths: set[str] = set() - entities = [] - # first pass to extract paths - for dict_ in data: - unique_paths.update(set(dict_.keys())) - - paths = list(unique_paths) - for dict_ in data: - entity = create_entity(paths, dict_) - entities.append(entity) - - schema = EntitySchema( - type_uri="https://example.org/vocab/RandomValueRow", - paths=[EntityPath(path=path) for path in paths], - ) - return Entities(entities=entities, schema=schema) - - -def create_entity(paths: list[str], dict_: dict[str, Any]) -> Entity: - """Create entity from dict based on order from paths list""" - values: list[list[str | None]] = [] - for path in paths: - value = dict_.get(path) - if value is None: - values.append([]) - elif type(value) in (int, float, bool, str): - values.append([value]) - else: - values.append([json.dumps(value)]) - entity_uri = f"urn:uuid:{uuid.uuid4()!s}" - return Entity(uri=entity_uri, values=values) diff --git a/tests/test_graphql.py b/tests/test_graphql.py index 9592ba0..0da7d2f 100644 --- a/tests/test_graphql.py +++ b/tests/test_graphql.py @@ -110,12 +110,16 @@ def _get_client() -> Client: return Client.from_context(context=TestExecutionContext()) -def _read_resource(project_name: str, filename: str) -> Any: # noqa: ANN401 - """Read a JSON resource from a CMEM project.""" +def _read_raw_resource(project_name: str, filename: str) -> str: + """Read the raw (undecoded) text content of a resource from a CMEM project.""" client = _get_client() - content = client.files.read(f"{project_name}:{filename}") + content: bytes = client.files.read(f"{project_name}:{filename}") + return content.decode("utf-8") - return json.loads(content) + +def _read_resource(project_name: str, filename: str) -> Any: # noqa: ANN401 + """Read a JSON resource from a CMEM project.""" + return json.loads(_read_raw_resource(project_name, filename)) @pytest.fixture(scope="module") @@ -157,6 +161,24 @@ def test_execution(project: str) -> None: assert graphql_response == str(result[0]) +@needs_cmem +def test_execution_preserves_unicode_characters(project: str) -> None: + """Test that non-ASCII characters from the GraphQL response are not escaped in the dataset""" + _ = project + query = "query{fruit(id:4){id,scientific_name,fruit_name,family}}" + + plugin = GraphQLPlugin( + graphql_url=GRAPHQL_URL, graphql_query=query, graphql_dataset=DATASET_NAME + ) + plugin.execute([], TestExecutionContext(project_id=PROJECT_NAME)) + raw_content = _read_raw_resource(PROJECT_NAME, RESOURCE_NAME) + assert "\\u00f3" not in raw_content + assert "\\u00e1" not in raw_content + fruit = json.loads(raw_content)[0]["fruit"] + assert fruit["fruit_name"] == "Limón" + assert fruit["family"] == "Rutáceae" + + @needs_cmem def test_execution_with_variables(project: str) -> None: """Test plugin execution"""