diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index 01cc317..df70731 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 import argparse +import contextlib +import dataclasses import os import re from functools import cache, total_ordering @@ -13,6 +15,7 @@ from rapids_metadata.remote import fetch_latest from .lint import Linter, LintMain +from .utils.yaml import Anchor, is_reference_anchor from .utils.dependencies_yaml import Handler, traverse_dependencies_yaml ALPHA_SPECIFIER: str = ">=0.0.0a0" @@ -48,16 +51,37 @@ def strip_cuda_suffix(args: argparse.Namespace, name: str) -> str: class AlphaSpecHandler(Handler): + @dataclasses.dataclass + class PackagesContext: + packages_is_reference_anchor: bool + def __init__(self, linter: Linter, args: argparse.Namespace): self.linter = linter self.args = args + def handle_packages( + self, + common_or_matrices_item_context: "Any", # noqa: ARG002 + anchor: "Optional[Anchor]", # noqa: ARG002 + key: "yaml.Node", # noqa: ARG002 + value: "yaml.Node", # noqa: ARG002 + ) -> "contextlib.nullcontext[AlphaSpecHandler.PackagesContext]": + return contextlib.nullcontext( + AlphaSpecHandler.PackagesContext(is_reference_anchor(anchor)) + ) + def handle_package( self, - packages_context: "Any", # noqa: ARG002 - anchor: "Optional[str]", + packages_context: "AlphaSpecHandler.PackagesContext", # noqa: ARG002 + anchor: "Optional[Anchor]", node: "yaml.Node", ) -> None: + if ( + packages_context.packages_is_reference_anchor + or is_reference_anchor(anchor) + ): + return + @total_ordering class SpecPriority: def __init__(self, spec: str): @@ -102,7 +126,7 @@ def create_specifier_string(specifiers: set[str]) -> str: ).add_replacement( (node.start_mark.index, node.end_mark.index), str( - (f"&{anchor} " if anchor else "") + (f"&{anchor.anchor_name} " if anchor else "") + req.name + create_specifier_string( {str(s) for s in req.specifier} | {ALPHA_SPECIFIER}, @@ -116,7 +140,7 @@ def create_specifier_string(specifiers: set[str]) -> str: ).add_replacement( (node.start_mark.index, node.end_mark.index), str( - (f"&{anchor} " if anchor else "") + (f"&{anchor.anchor_name} " if anchor else "") + req.name + create_specifier_string( {str(s) for s in req.specifier} - {ALPHA_SPECIFIER}, diff --git a/src/rapids_pre_commit_hooks/dependencies/cuda_suffixed.py b/src/rapids_pre_commit_hooks/dependencies/cuda_suffixed.py index 924f233..06732de 100644 --- a/src/rapids_pre_commit_hooks/dependencies/cuda_suffixed.py +++ b/src/rapids_pre_commit_hooks/dependencies/cuda_suffixed.py @@ -10,9 +10,10 @@ from packaging.requirements import InvalidRequirement, Requirement -from rapids_pre_commit_hooks.utils.dependencies_yaml import ( +from ..utils.dependencies_yaml import ( Handler, ) +from ..utils.yaml import Anchor, is_reference_anchor from rapids_metadata.remote import fetch_latest if TYPE_CHECKING: @@ -21,7 +22,7 @@ import yaml - from rapids_pre_commit_hooks.lint import Linter + from ..lint import Linter from rapids_metadata.metadata import RAPIDSMetadata, RAPIDSVersion @@ -82,6 +83,14 @@ class MatricesItemContext: default_factory=list ) + @dataclass + class PackagesContext: + parent_context: ( + "CUDASuffixedHandler.CommonItemContext | " + "CUDASuffixedHandler.MatricesItemContext" + ) + packages_is_reference_anchor: bool + def __init__(self, linter: "Linter", args: "argparse.Namespace") -> None: self.linter = linter self.args = args @@ -94,7 +103,7 @@ def handle_output_type( ), item: "yaml.Node", ) -> None: - if item.value in {"requirements", "pyproject"}: + if item.value in {"requirements", "constraints", "pyproject"}: output_types_context.has_python_output_type = True @contextlib.contextmanager @@ -321,15 +330,34 @@ def handle_matrix_item( matrix_context.cuda_node = value matrix_context.cuda_major = int(match.group("major")) - def handle_package( + def handle_packages( self, - packages_context: ( + common_or_matrices_item_context: ( "CUDASuffixedHandler.CommonItemContext | " "CUDASuffixedHandler.MatricesItemContext" ), - anchor: "Optional[str]", + anchor: "Optional[Anchor]", + key: "yaml.Node", # noqa: ARG002 + value: "yaml.Node", # noqa: ARG002 + ) -> "contextlib.nullcontext[CUDASuffixedHandler.PackagesContext]": + return contextlib.nullcontext( + CUDASuffixedHandler.PackagesContext( + common_or_matrices_item_context, is_reference_anchor(anchor) + ) + ) + + def handle_package( + self, + packages_context: "CUDASuffixedHandler.PackagesContext", + anchor: "Optional[Anchor]", item: "yaml.Node", ) -> None: + if ( + packages_context.packages_is_reference_anchor + or is_reference_anchor(anchor) + ): + return + try: req = Requirement(item.value) except InvalidRequirement: @@ -341,14 +369,19 @@ def handle_package( ) if req.name in cuda_suffixed_packages: - packages_context.suspicious_unsuffixed_packages.append( - (req.name, anchor, item) + packages_context.parent_context.suspicious_unsuffixed_packages.append( + (req.name, anchor.anchor_name if anchor else None, item) ) elif ( match := re.search( r"^(?P.*)(?P-cu[0-9]+)$", req.name ) ) and match.group("package") in cuda_suffixed_packages: - packages_context.suspicious_suffixed_packages.append( - (match.group("package"), match.group("suffix"), anchor, item) + packages_context.parent_context.suspicious_suffixed_packages.append( + ( + match.group("package"), + match.group("suffix"), + anchor.anchor_name if anchor else None, + item, + ) ) diff --git a/src/rapids_pre_commit_hooks/dependencies/use_cuda_wheels.py b/src/rapids_pre_commit_hooks/dependencies/use_cuda_wheels.py index ff88059..26821ee 100644 --- a/src/rapids_pre_commit_hooks/dependencies/use_cuda_wheels.py +++ b/src/rapids_pre_commit_hooks/dependencies/use_cuda_wheels.py @@ -8,9 +8,10 @@ from packaging.requirements import InvalidRequirement, Requirement -from rapids_pre_commit_hooks.utils.dependencies_yaml import ( +from ..utils.dependencies_yaml import ( Handler, ) +from ..utils.yaml import Anchor, is_reference_anchor if TYPE_CHECKING: import argparse @@ -18,7 +19,7 @@ import yaml - from rapids_pre_commit_hooks.lint import Linter + from ..lint import Linter def is_nvidia_library_package(req: "Requirement") -> bool: @@ -72,13 +73,18 @@ def is_cupy_ctk_package(req: "Requirement") -> bool: class UseCUDAWheelsHandler(Handler): @dataclass - class Context: + class CommonOrMatricesItemContext: has_use_cuda_wheels: bool = False use_cuda_wheels_node: "Optional[yaml.Node]" = None suspicious_packages: "list[tuple[yaml.Node, str]]" = field( default_factory=list ) + @dataclass + class PackagesContext: + parent_context: "UseCUDAWheelsHandler.CommonOrMatricesItemContext" + packages_is_reference_anchor: bool + def __init__(self, linter: "Linter", args: "argparse.Namespace"): self.linter = linter self.args = args @@ -89,8 +95,8 @@ def handle_common( dependency_set_context: "Any", # noqa: ARG002 key: "yaml.Node", value: "yaml.Node", # noqa: ARG002 - ) -> "Generator[UseCUDAWheelsHandler.Context]": - context = UseCUDAWheelsHandler.Context() + ) -> "Generator[UseCUDAWheelsHandler.CommonOrMatricesItemContext]": + context = UseCUDAWheelsHandler.CommonOrMatricesItemContext() yield context for node, name in context.suspicious_packages: @@ -109,8 +115,8 @@ def handle_matrices_item( self, matrices_context: "Any", # noqa: ARG002 item: "yaml.Node", # noqa: ARG002 - ) -> "Generator[UseCUDAWheelsHandler.Context]": - context = UseCUDAWheelsHandler.Context() + ) -> "Generator[UseCUDAWheelsHandler.CommonOrMatricesItemContext]": + context = UseCUDAWheelsHandler.CommonOrMatricesItemContext() yield context if not context.has_use_cuda_wheels: @@ -130,19 +136,18 @@ def handle_matrices_item( 'use_cuda_wheels: "true" instead', ) - @contextlib.contextmanager def handle_matrix( self, - matrices_item_context: "UseCUDAWheelsHandler.Context", + matrices_item_context: "UseCUDAWheelsHandler.CommonOrMatricesItemContext", # noqa: E501 key: "yaml.Node", value: "yaml.Node", # noqa: ARG002 - ) -> "Generator[UseCUDAWheelsHandler.Context]": + ) -> "contextlib.nullcontext[UseCUDAWheelsHandler.CommonOrMatricesItemContext]": # noqa: E501 matrices_item_context.use_cuda_wheels_node = key - yield matrices_item_context + return contextlib.nullcontext(matrices_item_context) def handle_matrix_item( self, - matrix_context: "UseCUDAWheelsHandler.Context", + matrix_context: "UseCUDAWheelsHandler.CommonOrMatricesItemContext", key: "yaml.Node", value: "yaml.Node", ) -> None: @@ -151,30 +156,41 @@ def handle_matrix_item( if value.value == "true": matrix_context.has_use_cuda_wheels = True - @contextlib.contextmanager def handle_packages( self, - common_or_matrices_item_context: "UseCUDAWheelsHandler.Context", + common_or_matrices_item_context: "UseCUDAWheelsHandler.CommonOrMatricesItemContext", # noqa: E501 + anchor: "Optional[Anchor]", key: "yaml.Node", value: "yaml.Node", # noqa: ARG002 - ) -> "Generator[UseCUDAWheelsHandler.Context]": + ) -> "contextlib.nullcontext[UseCUDAWheelsHandler.PackagesContext]": if common_or_matrices_item_context.use_cuda_wheels_node is None: common_or_matrices_item_context.use_cuda_wheels_node = key - yield common_or_matrices_item_context + context = UseCUDAWheelsHandler.PackagesContext( + common_or_matrices_item_context, is_reference_anchor(anchor) + ) + return contextlib.nullcontext(context) def handle_package( self, - packages_context: "UseCUDAWheelsHandler.Context", - anchor: "Optional[str]", # noqa: ARG002 + packages_context: "UseCUDAWheelsHandler.PackagesContext", + anchor: "Optional[Anchor]", item: "yaml.Node", ) -> None: + if ( + packages_context.packages_is_reference_anchor + or is_reference_anchor(anchor) + ): + return + try: req = Requirement(item.value) except InvalidRequirement: return if is_nvidia_library_package(req): - packages_context.suspicious_packages.append((item, req.name)) + packages_context.parent_context.suspicious_packages.append( + (item, req.name) + ) elif is_cupy_ctk_package(req): - packages_context.suspicious_packages.append( + packages_context.parent_context.suspicious_packages.append( (item, f"{req.name}[ctk]") ) diff --git a/src/rapids_pre_commit_hooks/utils/dependencies_yaml.py b/src/rapids_pre_commit_hooks/utils/dependencies_yaml.py index fd3b514..794ac00 100644 --- a/src/rapids_pre_commit_hooks/utils/dependencies_yaml.py +++ b/src/rapids_pre_commit_hooks/utils/dependencies_yaml.py @@ -6,7 +6,12 @@ import yaml -from .yaml import AnchorPreservingLoader, check_and_mark_anchor, node_has_type +from .yaml import ( + Anchor, + AnchorPreservingLoader, + check_and_mark_anchor, + node_has_type, +) if TYPE_CHECKING: from collections.abc import Generator, Iterable @@ -114,6 +119,7 @@ def handle_matrix_item( def handle_packages( self, common_or_matrices_item_context: "Any", + anchor: "Optional[Anchor]", # noqa: ARG002 key: "yaml.Node", # noqa: ARG002 value: "yaml.Node", # noqa: ARG002 ) -> "contextlib.AbstractContextManager[Any]": @@ -122,7 +128,7 @@ def handle_packages( def handle_package( self, packages_context: "Any", # noqa: ARG002 - anchor: "Optional[str]", # noqa: ARG002 + anchor: "Optional[Anchor]", # noqa: ARG002 item: "yaml.Node", # noqa: ARG002 ) -> None: pass @@ -299,9 +305,8 @@ def traverse_package( node: "yaml.Node", ) -> None: if node_has_type(node, "str"): - descend, anchor = check_and_mark_anchor(anchors, used_anchors, node) - if descend: - handler.handle_package(packages_context, anchor, node) + anchor = check_and_mark_anchor(anchors, used_anchors, node) + handler.handle_package(packages_context, anchor, node) def traverse_packages( @@ -313,19 +318,18 @@ def traverse_packages( node: "yaml.Node", ) -> None: if node_has_type(node, "seq"): - descend, _ = check_and_mark_anchor(anchors, used_anchors, node) - if descend: - with handler.handle_packages( - common_or_matrices_item_context, key_node, node - ) as packages_context: - for package in node.value: - traverse_package( - handler, - packages_context, - anchors, - used_anchors, - package, - ) + anchor = check_and_mark_anchor(anchors, used_anchors, node) + with handler.handle_packages( + common_or_matrices_item_context, anchor, key_node, node + ) as packages_context: + for package in node.value: + traverse_package( + handler, + packages_context, + anchors, + used_anchors, + package, + ) def traverse_output_type( diff --git a/src/rapids_pre_commit_hooks/utils/yaml.py b/src/rapids_pre_commit_hooks/utils/yaml.py index bf91eb2..137ce84 100644 --- a/src/rapids_pre_commit_hooks/utils/yaml.py +++ b/src/rapids_pre_commit_hooks/utils/yaml.py @@ -1,6 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import dataclasses +from enum import Enum +from typing import Optional + import yaml @@ -30,13 +34,24 @@ def compose_document(self) -> "yaml.Node": return node +class AnchorType(Enum): + DEFINITION = 0 + REFERENCE = 1 + + +@dataclasses.dataclass +class Anchor: + anchor_type: AnchorType + anchor_name: str + + def node_has_type(node: "yaml.Node", tag_type: str) -> bool: return node.tag == f"tag:yaml.org,2002:{tag_type}" def check_and_mark_anchor( - anchors: dict[str, "yaml.Node"], used_anchors: set[str], node: "yaml.Node" -) -> tuple[bool, str | None]: + anchors: "dict[str, yaml.Node]", used_anchors: set[str], node: "yaml.Node" +) -> "Optional[Anchor]": for key, value in anchors.items(): if value == node: anchor = key @@ -44,7 +59,12 @@ def check_and_mark_anchor( else: anchor = None if anchor in used_anchors: - return False, anchor + return Anchor(AnchorType.REFERENCE, anchor) if anchor is not None: used_anchors.add(anchor) - return True, anchor + return Anchor(AnchorType.DEFINITION, anchor) + return None + + +def is_reference_anchor(anchor: "Optional[Anchor]") -> bool: + return anchor is not None and anchor.anchor_type == AnchorType.REFERENCE diff --git a/tests/rapids_pre_commit_hooks/dependencies/test_cuda_suffixed.py b/tests/rapids_pre_commit_hooks/dependencies/test_cuda_suffixed.py index 6b702c2..645108b 100644 --- a/tests/rapids_pre_commit_hooks/dependencies/test_cuda_suffixed.py +++ b/tests/rapids_pre_commit_hooks/dependencies/test_cuda_suffixed.py @@ -11,6 +11,7 @@ CUDASuffixedHandler, ) from rapids_pre_commit_hooks.utils import dependencies_yaml +from rapids_pre_commit_hooks.utils.yaml import Anchor, AnchorType from rapids_pre_commit_hooks_test_utils import ( find_yaml_node_for_span, parse_named_spans, @@ -31,6 +32,7 @@ class TestCUDASuffixedHandler: ["output_type", "expected"], [ pytest.param("requirements", True, id="requirements"), + pytest.param("constraints", True, id="constraints"), pytest.param("pyproject", True, id="pyproject"), pytest.param("conda", False, id="conda"), ], @@ -768,60 +770,132 @@ def test_handle_matrix_item( @pytest.mark.parametrize( [ "requirement", + "anchor", + "packages_is_reference_anchor", "suffixed_names", "unsuffixed_names", ], [ pytest.param( "package", + None, + False, [], ["package"], id="unsuffixed", ), pytest.param( "package[extra]>=1.0", + None, + False, [], ["package"], id="unsuffixed-with-extras-and-version", ), pytest.param( "package-cu12", + None, + False, [("package", "-cu12")], [], id="suffixed", ), pytest.param( "package-cu123==1.0", + None, + False, [("package", "-cu123")], [], id="multi-digit-suffix", ), pytest.param( "package-cu12x", + None, + False, [], [], id="invalid-cuda-suffix", ), pytest.param( "other-cu12", + None, + False, [], [], id="unknown-package", ), pytest.param( "not a requirement", + None, + False, [], [], id="invalid-requirement", ), + pytest.param( + "package", + Anchor(AnchorType.DEFINITION, "package"), + False, + [], + ["package"], + id="unsuffixed-anchor-definition", + ), + pytest.param( + "package", + Anchor(AnchorType.REFERENCE, "package"), + False, + [], + [], + id="unsuffixed-anchor-reference", + ), + pytest.param( + "package", + None, + True, + [], + [], + id="unsuffixed-packages-is-reference-anchor", + ), + pytest.param( + "package-cu12", + Anchor(AnchorType.DEFINITION, "package"), + False, + [("package", "-cu12")], + [], + id="suffixed-anchor-definition", + ), + pytest.param( + "package-cu12", + Anchor(AnchorType.REFERENCE, "package"), + False, + [], + [], + id="suffixed-anchor-reference", + ), + pytest.param( + "package-cu12", + None, + True, + [], + [], + id="suffixed-packages-is-reference-anchor", + ), ], ) def test_handle_package( - self, requirement, suffixed_names, unsuffixed_names + self, + requirement, + anchor, + packages_is_reference_anchor, + suffixed_names, + unsuffixed_names, ): package_node = _compose(requirement) rapids_version = SimpleNamespace(cuda_suffixed_packages={"package"}) - context = CUDASuffixedHandler.MatricesItemContext() + context = CUDASuffixedHandler.PackagesContext( + CUDASuffixedHandler.MatricesItemContext(), + packages_is_reference_anchor, + ) handler = CUDASuffixedHandler(Mock(), Mock()) with patch( @@ -829,14 +903,20 @@ def test_handle_package( "get_rapids_version", return_value=rapids_version, ): - handler.handle_package(context, None, package_node) + handler.handle_package(context, anchor, package_node) - assert context.suspicious_suffixed_packages == [ - (name, suffix, None, package_node) + assert context.parent_context.suspicious_suffixed_packages == [ + ( + name, + suffix, + anchor.anchor_name if anchor else None, + package_node, + ) for (name, suffix) in suffixed_names ] - assert context.suspicious_unsuffixed_packages == [ - (name, None, package_node) for name in unsuffixed_names + assert context.parent_context.suspicious_unsuffixed_packages == [ + (name, anchor.anchor_name if anchor else None, package_node) + for name in unsuffixed_names ] diff --git a/tests/rapids_pre_commit_hooks/dependencies/test_use_cuda_wheels.py b/tests/rapids_pre_commit_hooks/dependencies/test_use_cuda_wheels.py index eae4f61..d679d4b 100644 --- a/tests/rapids_pre_commit_hooks/dependencies/test_use_cuda_wheels.py +++ b/tests/rapids_pre_commit_hooks/dependencies/test_use_cuda_wheels.py @@ -13,6 +13,7 @@ is_nvidia_library_package, ) from rapids_pre_commit_hooks.utils import dependencies_yaml +from rapids_pre_commit_hooks.utils.yaml import Anchor, AnchorType from rapids_pre_commit_hooks_test_utils import ( find_yaml_node_for_span, parse_named_spans, @@ -432,7 +433,7 @@ def test_handle_matrix_item(self, content, expected_has_use_cuda_wheels): ) @pytest.mark.parametrize( - ["content"], + ["content", "anchor", "packages_is_reference_anchor"], [ pytest.param( """\ @@ -443,6 +444,8 @@ def test_handle_matrix_item(self, content, expected_has_use_cuda_wheels): : ~~~~~~~~packages_key : ~~packages """, + None, + False, id="matrix-node", ), pytest.param( @@ -452,11 +455,41 @@ def test_handle_matrix_item(self, content, expected_has_use_cuda_wheels): : ~~~~~~~~packages_key : ~~packages """, + None, + False, id="no-matrix-node", ), + pytest.param( + """\ + + matrix: {} + : ~~~~~~original_node + : ~~~~~~node + + packages: [] + : ~~~~~~~~packages_key + : ~~packages + """, + Anchor(AnchorType.DEFINITION, "packages"), + False, + id="anchor-definition", + ), + pytest.param( + """\ + + matrix: {} + : ~~~~~~original_node + : ~~~~~~node + + packages: [] + : ~~~~~~~~packages_key + : ~~packages + """, + Anchor(AnchorType.REFERENCE, "packages"), + True, + id="anchor-reference", + ), ], ) - def test_handle_packages(self, content): + def test_handle_packages( + self, content, anchor, packages_is_reference_anchor + ): content, spans = parse_named_spans(content) args = Mock() @@ -480,58 +513,108 @@ def test_handle_packages(self, content): handler = UseCUDAWheelsHandler(linter, args) with handler.handle_packages( - Mock(use_cuda_wheels_node=original_node), packages_key, packages + Mock(use_cuda_wheels_node=original_node), + anchor, + packages_key, + packages, ) as packages_context: - assert packages_context.use_cuda_wheels_node == node + assert packages_context.parent_context.use_cuda_wheels_node == node + assert ( + packages_context.packages_is_reference_anchor + == packages_is_reference_anchor + ) @pytest.mark.parametrize( - ["content", "expected_node", "expected_name"], + [ + "content", + "anchor", + "packages_is_reference_anchor", + "expected_node", + "expected_name", + ], [ pytest.param( "cuda-toolkit==13.0", + None, + False, True, "cuda-toolkit", id="cuda-toolkit", ), pytest.param( "cuda-toolkit[cufile]==13.0", + None, + False, True, "cuda-toolkit", id="cuda-toolkit-extras", ), pytest.param( "cupy-cuda12x[ctk]", + None, + False, True, "cupy-cuda12x[ctk]", id="cupy-ctk", ), pytest.param( "cupy-cuda13x[ctk,other]", + None, + False, True, "cupy-cuda13x[ctk]", id="cupy-ctk-and-other", ), pytest.param( "cupy-cuda13x[other]", + None, + False, False, None, id="cupy-others", ), pytest.param( "cupy-cuda13x", + None, + False, False, None, id="cupy-no-extras", ), pytest.param( "other-package", + None, + False, False, None, id="other-package", ), + pytest.param( + "cuda-toolkit==13.0", + Anchor(AnchorType.DEFINITION, "cuda_toolkit"), + False, + True, + "cuda-toolkit", + id="anchor-definition", + ), + pytest.param( + "cuda-toolkit==13.0", + Anchor(AnchorType.REFERENCE, "cuda_toolkit"), + False, + False, + None, + id="anchor-reference", + ), ], ) - def test_handle_package(self, content, expected_node, expected_name): + def test_handle_package( + self, + content, + anchor, + packages_is_reference_anchor, + expected_node, + expected_name, + ): args = Mock() linter = lint.Linter( "dependencies.yaml", content, "verify-dependencies" @@ -543,9 +626,12 @@ def test_handle_package(self, content, expected_node, expected_name): loader.dispose() handler = UseCUDAWheelsHandler(linter, args) - packages_context = Mock(suspicious_packages=[]) - handler.handle_package(packages_context, None, package_node) - assert packages_context.suspicious_packages == ( + packages_context = Mock( + parent_context=Mock(suspicious_packages=[]), + packages_is_reference_anchor=packages_is_reference_anchor, + ) + handler.handle_package(packages_context, anchor, package_node) + assert packages_context.parent_context.suspicious_packages == ( [(package_node, expected_name)] if expected_node else [] ) diff --git a/tests/rapids_pre_commit_hooks/test_alpha_spec.py b/tests/rapids_pre_commit_hooks/test_alpha_spec.py index f6c154e..a19fed8 100644 --- a/tests/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/tests/rapids_pre_commit_hooks/test_alpha_spec.py @@ -15,7 +15,11 @@ ) from rapids_pre_commit_hooks import alpha_spec, lint -from rapids_pre_commit_hooks.utils.yaml import AnchorPreservingLoader +from rapids_pre_commit_hooks.utils.yaml import ( + Anchor, + AnchorPreservingLoader, + AnchorType, +) from rapids_pre_commit_hooks_test_utils import parse_named_spans latest_version, latest_metadata = max( @@ -129,131 +133,315 @@ def test_strip_cuda_suffix(name, stripped_name): assert alpha_spec.strip_cuda_suffix(Mock(), name) == stripped_name -@pytest.mark.parametrize( - ["package", "anchor", "content", "mode", "replacement"], - [ - *chain( - *( - [ - (p, None, p, "development", f"{p}>=0.0.0a0"), - (p, None, p, "release", None), - (p, None, f"{p}>=0.0.0a0", "development", None), - (p, None, f"{p}>=0.0.0a0", "release", p), - ] - for p in latest_metadata.prerelease_packages - ) - ), - *chain( - *( - [ - ( - f"{p}-cu12", - None, - f"{p}-cu12", - "development", - f"{p}-cu12>=0.0.0a0", - ), - (f"{p}-cu11", None, f"{p}-cu11", "release", None), - ( - f"{p}-cu12", - None, - f"{p}-cu12>=0.0.0a0", - "development", - None, - ), - ( - f"{p}-cu11", - None, - f"{p}-cu11>=0.0.0a0", - "release", - f"{p}-cu11", - ), - ] - for p in latest_metadata.prerelease_packages - & latest_metadata.cuda_suffixed_packages +class TestAlphaSpecHandler: + @pytest.mark.parametrize( + ["anchor", "packages_is_reference_anchor"], + [ + pytest.param( + None, + False, + id="no-anchor", + ), + pytest.param( + Anchor(AnchorType.DEFINITION, "anchor"), + False, + id="anchor-definition", + ), + pytest.param( + Anchor(AnchorType.REFERENCE, "anchor"), + True, + id="anchor-reference", + ), + ], + ) + def test_handle_packages(self, anchor, packages_is_reference_anchor): + handler = alpha_spec.AlphaSpecHandler(Mock(), Mock()) + + with handler.handle_packages( + None, anchor, Mock(), Mock() + ) as packages_context: + assert ( + packages_context.packages_is_reference_anchor + == packages_is_reference_anchor ) - ), - *chain( - *( - [ - (f"{p}-cu12", None, f"{p}-cu12", "development", None), - (f"{p}-cu12", None, f"{p}-cu12>=0.0.0a0", "release", None), - ] - for p in latest_metadata.prerelease_packages - & ( - latest_metadata.all_packages - - latest_metadata.cuda_suffixed_packages + + @pytest.mark.parametrize( + [ + "package", + "anchor", + "content", + "mode", + "packages_is_reference_anchor", + "replacement", + ], + [ + *chain( + *( + [ + pytest.param( + p, + None, + p, + "development", + False, + f"{p}>=0.0.0a0", + id=f"{p}-development-no-suffix", + ), + pytest.param( + p, + None, + p, + "release", + False, + None, + id=f"{p}-release-no-suffix", + ), + pytest.param( + p, + None, + f"{p}>=0.0.0a0", + "development", + False, + None, + id=f"{p}-development-suffix", + ), + pytest.param( + p, + None, + f"{p}>=0.0.0a0", + "release", + False, + p, + id=f"{p}-release-suffix", + ), + ] + for p in latest_metadata.prerelease_packages ) - ) - ), - ( - "cuml", - None, - "cuml>=24.04,<24.06", - "development", - "cuml>=24.04,<24.06,>=0.0.0a0", - ), - ( - "cuml", - None, - "cuml>=24.04,<24.06,>=0.0.0a0", - "release", - "cuml>=24.04,<24.06", - ), - ( - "cuml", - "cuml", - "&cuml cuml>=24.04,<24.06,>=0.0.0a0", - "release", - "&cuml cuml>=24.04,<24.06", - ), - ("packaging", None, "packaging", "development", None), - ( - None, - None, - "--extra-index-url=https://pypi.nvidia.com", - "development", - None, - ), - ( - None, - None, - "--extra-index-url=https://pypi.nvidia.com", - "release", - None, - ), - (None, None, "gcc_linux-64=11.*", "development", None), - (None, None, "gcc_linux-64=11.*", "release", None), - ], -) -@patch( - "rapids_pre_commit_hooks.alpha_spec.get_rapids_version", - Mock(return_value=latest_metadata), -) -def test_check_package_spec(package, anchor, content, mode, replacement): - args = Mock(mode=mode) - linter = lint.Linter("dependencies.yaml", content, "verify-alpha-spec") - loader = AnchorPreservingLoader(content) - try: - composed = loader.get_single_node() - finally: - loader.dispose() - handler = alpha_spec.AlphaSpecHandler(linter, args) - handler.handle_package(Mock(), anchor, composed) - if replacement is None: - assert linter.warnings == [] - else: - expected_linter = lint.Linter( - "dependencies.yaml", content, "verify-alpha-spec" - ) - expected_linter.add_warning( - (composed.start_mark.index, composed.end_mark.index), - f"{'add' if mode == 'development' else 'remove'} " - f"alpha spec for RAPIDS package {package}", - ).add_replacement( - (composed.start_mark.index, composed.end_mark.index), replacement + ), + *chain( + *( + [ + pytest.param( + f"{p}-cu12", + None, + f"{p}-cu12", + "development", + False, + f"{p}-cu12>=0.0.0a0", + id=f"{p}-cu12-development-no-suffix", + ), + pytest.param( + f"{p}-cu11", + None, + f"{p}-cu11", + "release", + False, + None, + id=f"{p}-cu11-release-no-suffix", + ), + pytest.param( + f"{p}-cu12", + None, + f"{p}-cu12>=0.0.0a0", + "development", + False, + None, + id=f"{p}-cu12-development-suffix", + ), + pytest.param( + f"{p}-cu11", + None, + f"{p}-cu11>=0.0.0a0", + "release", + False, + f"{p}-cu11", + id=f"{p}-cu11-release-suffix", + ), + ] + for p in latest_metadata.prerelease_packages + & latest_metadata.cuda_suffixed_packages + ) + ), + *chain( + *( + [ + pytest.param( + f"{p}-cu12", + None, + f"{p}-cu12", + "development", + False, + None, + id=f"{p}-cu12-development-no-suffix", + ), + pytest.param( + f"{p}-cu12", + None, + f"{p}-cu12>=0.0.0a0", + "release", + False, + None, + id=f"{p}-cu12-release-suffix", + ), + ] + for p in latest_metadata.prerelease_packages + & ( + latest_metadata.all_packages + - latest_metadata.cuda_suffixed_packages + ) + ) + ), + pytest.param( + "cuml", + None, + "cuml>=24.04,<24.06", + "development", + False, + "cuml>=24.04,<24.06,>=0.0.0a0", + id="version-range-development-no-suffix", + ), + pytest.param( + "cuml", + None, + "cuml>=24.04,<24.06,>=0.0.0a0", + "release", + False, + "cuml>=24.04,<24.06", + id="version-range-release-suffix", + ), + pytest.param( + "cuml", + Anchor(AnchorType.DEFINITION, "cuml"), + "&cuml cuml>=24.04,<24.06", + "development", + False, + "&cuml cuml>=24.04,<24.06,>=0.0.0a0", + id="anchor-definition-development-no-suffix", + ), + pytest.param( + "cuml", + Anchor(AnchorType.REFERENCE, "cuml"), + "&cuml cuml>=24.04,<24.06", + "development", + False, + None, + id="anchor-reference-development-no-suffix", + ), + pytest.param( + "cuml", + Anchor(AnchorType.DEFINITION, "cuml"), + "&cuml cuml>=24.04,<24.06,>=0.0.0a0", + "release", + False, + "&cuml cuml>=24.04,<24.06", + id="anchor-definition-release-suffix", + ), + pytest.param( + "cuml", + Anchor(AnchorType.REFERENCE, "cuml"), + "&cuml cuml>=24.04,<24.06,>=0.0.0a0", + "release", + False, + None, + id="anchor-reference-release-suffix", + ), + pytest.param( + "cuml", + None, + "cuml>=24.04,<24.06", + "development", + True, + None, + id="packages-is-anchor-reference", + ), + pytest.param( + "packaging", + None, + "packaging", + "development", + False, + None, + id="non-rapids-package", + ), + pytest.param( + None, + None, + "--extra-index-url=https://pypi.nvidia.com", + "development", + False, + None, + id="extra-index-url-development", + ), + pytest.param( + None, + None, + "--extra-index-url=https://pypi.nvidia.com", + "release", + False, + None, + id="extra-index-url-release", + ), + pytest.param( + None, + None, + "gcc_linux-64=11.*", + "development", + False, + None, + id="conda-package-development", + ), + pytest.param( + None, + None, + "gcc_linux-64=11.*", + "release", + False, + None, + id="conda-package-release", + ), + ], + ) + @patch( + "rapids_pre_commit_hooks.alpha_spec.get_rapids_version", + Mock(return_value=latest_metadata), + ) + def test_handle_package( + self, + package, + anchor, + content, + mode, + packages_is_reference_anchor, + replacement, + ): + args = Mock(mode=mode) + linter = lint.Linter("dependencies.yaml", content, "verify-alpha-spec") + loader = AnchorPreservingLoader(content) + try: + composed = loader.get_single_node() + finally: + loader.dispose() + handler = alpha_spec.AlphaSpecHandler(linter, args) + handler.handle_package( + Mock(packages_is_reference_anchor=packages_is_reference_anchor), + anchor, + composed, ) - assert linter.warnings == expected_linter.warnings + if replacement is None: + assert linter.warnings == [] + else: + expected_linter = lint.Linter( + "dependencies.yaml", content, "verify-alpha-spec" + ) + expected_linter.add_warning( + (composed.start_mark.index, composed.end_mark.index), + f"{'add' if mode == 'development' else 'remove'} " + f"alpha spec for RAPIDS package {package}", + ).add_replacement( + (composed.start_mark.index, composed.end_mark.index), + replacement, + ) + assert linter.warnings == expected_linter.warnings def test_check_alpha_spec(): @@ -283,9 +471,12 @@ def test_check_alpha_spec_integration(tmp_path): + test: + common: + - output_types: pyproject - + packages: - + - cudf>=24.04,<24.06 - : ~~~~~~~~~~~~~~~~~~package + + packages: &packages + + - &cudf cudf>=24.04,<24.06 + : ~~~~~~~~~~~~~~~~~~~~~~~~package + + - *cudf + + - output_types: requirements + + packages: *packages """ ) @@ -303,5 +494,5 @@ def test_check_alpha_spec_integration(tmp_path): ) expected_linter.add_warning( spans["package"], "add alpha spec for RAPIDS package cudf" - ).add_replacement(spans["package"], "cudf>=24.04,<24.06,>=0.0.0a0") + ).add_replacement(spans["package"], "&cudf cudf>=24.04,<24.06,>=0.0.0a0") assert linter.warnings == expected_linter.warnings diff --git a/tests/rapids_pre_commit_hooks/utils/test_dependencies_yaml.py b/tests/rapids_pre_commit_hooks/utils/test_dependencies_yaml.py index 342b0ed..84f971c 100644 --- a/tests/rapids_pre_commit_hooks/utils/test_dependencies_yaml.py +++ b/tests/rapids_pre_commit_hooks/utils/test_dependencies_yaml.py @@ -6,6 +6,7 @@ import pytest import yaml +from rapids_pre_commit_hooks.utils.yaml import Anchor, AnchorType from rapids_pre_commit_hooks.utils import dependencies_yaml from rapids_pre_commit_hooks_test_utils import ( find_yaml_node_for_span, @@ -86,7 +87,7 @@ class TestChainedHandler: pytest.param( "handle_packages", True, - (Mock(), Mock()), + (Mock(), Mock(), Mock()), id="handle_packages", ), ], @@ -147,7 +148,7 @@ def test_context(self, hook_name, use_context, hook_args): ), pytest.param( "handle_package", - ("anchor", Mock()), + (Mock(), Mock()), id="handle_package", ), pytest.param( @@ -177,136 +178,147 @@ def test_no_context(self, hook_name, hook_args): assert manager.mock_calls == expected_calls -def test_traverse_package(): - packages = yaml.SafeLoader("""\ - - lib1 - """).get_single_node() - package = packages.value[0] - packages_context = Mock() - manager = MagicMock() - - expected_calls = [ - call.handler.handle_package(packages_context, None, package), - ] - manager.reset_mock() - - dependencies_yaml.traverse_package( - manager.handler, packages_context, {}, set(), package - ) - - assert manager.mock_calls == expected_calls - - -def test_traverse_package_anchor(): - packages = yaml.SafeLoader("""\ - - &lib1 lib1 - - *lib1 - """).get_single_node() - package = packages.value[0] +@pytest.mark.parametrize( + ["content", "used_anchors", "anchor"], + [ + pytest.param( + """\ + + - lib1 + : ~~~~node + """, + set(), + None, + id="no-anchor", + ), + pytest.param( + """\ + + - &lib1 lib1 + : ~~~~~~~~~~node + : ~~~~~~~~~~anchors.lib1 + + - *lib1 + """, + set(), + Anchor(AnchorType.DEFINITION, "lib1"), + id="anchor-definition", + ), + pytest.param( + """\ + + - &lib1 lib1 + : ~~~~~~~~~~node + : ~~~~~~~~~~anchors.lib1 + + - *lib1 + """, + {"lib1"}, + Anchor(AnchorType.REFERENCE, "lib1"), + id="anchor-reference", + ), + ], +) +def test_traverse_package(content, used_anchors, anchor): + content, spans = parse_named_spans(content) + composed = yaml.SafeLoader(content).get_single_node() + package = find_yaml_node_for_span(composed, spans["node"]) packages_context = Mock() manager = MagicMock() expected_calls = [ - call.handler.handle_package(packages_context, "lib1", package), + call.handler.handle_package(packages_context, anchor, package), ] manager.reset_mock() + anchors = { + name: find_yaml_node_for_span(composed, span) + for name, span in spans.get("anchors", {}).items() + } dependencies_yaml.traverse_package( - manager.handler, packages_context, {"lib1": package}, set(), package - ) - - assert manager.mock_calls == expected_calls - - -def test_traverse_package_used_anchor(): - packages = yaml.SafeLoader("""\ - - &lib1 lib1 - - *lib1 - """).get_single_node() - package = packages.value[1] - packages_context = Mock() - manager = MagicMock() - - expected_calls = [] - manager.reset_mock() - - dependencies_yaml.traverse_package( - manager.handler, packages_context, {"lib1": package}, {"lib1"}, package + manager.handler, packages_context, anchors, used_anchors, package ) assert manager.mock_calls == expected_calls -def test_traverse_packages(): - item = yaml.SafeLoader("""\ - packages: - - lib1 - - lib2 - """).get_single_node() - packages_key, packages = item.value[0] - item_context = Mock() - manager = MagicMock() - - expected_calls = [ - call.handler.handle_packages(item_context, packages_key, packages), - call.handler.handle_packages().__enter__(), - call.traverse_package( - manager.handler, - manager.handler.handle_packages().__enter__(), - {}, +@pytest.mark.parametrize( + ["content", "used_anchors", "used_anchors_after", "anchor"], + [ + pytest.param( + """\ + + packages: + : ~~~~~~~~packages_key + + - lib1 + : >packages + + - lib2 + : !packages + """, set(), - packages.value[0], + set(), + None, + id="no-anchor", ), - call.traverse_package( - manager.handler, - manager.handler.handle_packages().__enter__(), - {}, + pytest.param( + """\ + + - packages: &packages + : ~~~~~~~~packages_key + : >packages + : >anchors.packages + + - lib1 + + - lib2 + : !packages + : !anchors.packages + + - packages: *packages + """, set(), - packages.value[1], + {"packages"}, + Anchor(AnchorType.DEFINITION, "packages"), + id="anchor-definition", ), - call.handler.handle_packages().__exit__(None, None, None), - ] - manager.reset_mock() - - with ( - patch( - "rapids_pre_commit_hooks.utils.dependencies_yaml.traverse_package", - manager.traverse_package, + pytest.param( + """\ + + - packages: &packages + : >packages + : >anchors.packages + + - lib1 + + - lib2 + : !packages + : !anchors.packages + + - packages: *packages + : ~~~~~~~~packages_key + """, + {"packages"}, + {"packages"}, + Anchor(AnchorType.REFERENCE, "packages"), + id="anchor-reference", ), - ): - dependencies_yaml.traverse_packages( - manager.handler, item_context, {}, set(), packages_key, packages - ) - - assert manager.mock_calls == expected_calls - - -def test_traverse_packages_anchor(): - items = yaml.SafeLoader("""\ - - packages: &packages - - lib1 - - lib2 - - packages: *packages - """).get_single_node() - packages_key, packages = items.value[0].value[0] + ], +) +def test_traverse_packages(content, used_anchors, used_anchors_after, anchor): + content, spans = parse_named_spans(content) + composed = yaml.SafeLoader(content).get_single_node() + packages_key = find_yaml_node_for_span(composed, spans["packages_key"]) + packages = find_yaml_node_for_span(composed, spans["packages"]) item_context = Mock() manager = MagicMock() + anchors = { + name: find_yaml_node_for_span(composed, span) + for name, span in spans.get("anchors", {}).items() + } expected_calls = [ - call.handler.handle_packages(item_context, packages_key, packages), + call.handler.handle_packages( + item_context, anchor, packages_key, packages + ), call.handler.handle_packages().__enter__(), call.traverse_package( manager.handler, manager.handler.handle_packages().__enter__(), - {"packages": items.value[0].value[0][1]}, - {"packages"}, + anchors, + used_anchors_after, packages.value[0], ), call.traverse_package( manager.handler, manager.handler.handle_packages().__enter__(), - {"packages": items.value[0].value[0][1]}, - {"packages"}, + anchors, + used_anchors_after, packages.value[1], ), call.handler.handle_packages().__exit__(None, None, None), @@ -322,40 +334,8 @@ def test_traverse_packages_anchor(): dependencies_yaml.traverse_packages( manager.handler, item_context, - {"packages": items.value[0].value[0][1]}, - set(), - packages_key, - packages, - ) - - assert manager.mock_calls == expected_calls - - -def test_traverse_packages_used_anchor(): - items = yaml.SafeLoader("""\ - - packages: &packages - - lib1 - - lib2 - - packages: *packages - """).get_single_node() - packages_key, packages = items.value[1].value[0] - item_context = Mock() - manager = MagicMock() - - expected_calls = [] - manager.reset_mock() - - with ( - patch( - "rapids_pre_commit_hooks.utils.dependencies_yaml.traverse_package", - manager.traverse_package, - ), - ): - dependencies_yaml.traverse_packages( - manager.handler, - item_context, - {"packages": items.value[0].value[0][1]}, - {"packages"}, + anchors, + used_anchors, packages_key, packages, ) diff --git a/tests/rapids_pre_commit_hooks/utils/test_yaml.py b/tests/rapids_pre_commit_hooks/utils/test_yaml.py index d379ada..3f6885b 100644 --- a/tests/rapids_pre_commit_hooks/utils/test_yaml.py +++ b/tests/rapids_pre_commit_hooks/utils/test_yaml.py @@ -6,8 +6,11 @@ import pytest from rapids_pre_commit_hooks.utils.yaml import ( + Anchor, AnchorPreservingLoader, + AnchorType, check_and_mark_anchor, + is_reference_anchor, ) @@ -24,7 +27,6 @@ def test_anchor_preserving_loader(): [ "used_anchors_before", "node_index", - "descend", "anchor", "used_anchors_after", ], @@ -32,36 +34,31 @@ def test_anchor_preserving_loader(): ( set(), 0, - True, - "anchor1", + Anchor(AnchorType.DEFINITION, "anchor1"), {"anchor1"}, ), ( {"anchor1"}, 1, - True, - "anchor2", + Anchor(AnchorType.DEFINITION, "anchor2"), {"anchor1", "anchor2"}, ), ( set(), 2, - True, None, set(), ), ( {"anchor1", "anchor2"}, 0, - False, - "anchor1", + Anchor(AnchorType.REFERENCE, "anchor1"), {"anchor1", "anchor2"}, ), ( {"anchor1", "anchor2"}, 1, - False, - "anchor2", + Anchor(AnchorType.REFERENCE, "anchor2"), {"anchor1", "anchor2"}, ), ], @@ -69,7 +66,6 @@ def test_anchor_preserving_loader(): def test_check_and_mark_anchor( used_anchors_before, node_index, - descend, anchor, used_anchors_after, ): @@ -79,9 +75,32 @@ def test_check_and_mark_anchor( "anchor2": NODES[1], } used_anchors = set(used_anchors_before) - actual_descend, actual_anchor = check_and_mark_anchor( + actual_anchor = check_and_mark_anchor( ANCHORS, used_anchors, NODES[node_index] ) - assert actual_descend == descend assert actual_anchor == anchor assert used_anchors == used_anchors_after + + +@pytest.mark.parametrize( + ["anchor", "is_ref"], + [ + pytest.param( + None, + False, + id="none", + ), + pytest.param( + Anchor(AnchorType.DEFINITION, "anchor"), + False, + id="definition", + ), + pytest.param( + Anchor(AnchorType.REFERENCE, "anchor"), + True, + id="reference", + ), + ], +) +def test_is_reference_anchor(anchor, is_ref): + assert is_reference_anchor(anchor) == is_ref