diff --git a/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/__init__.py b/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/__init__.py new file mode 100644 index 000000000000..d540fd20468c --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/__init__.py @@ -0,0 +1,3 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- diff --git a/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/conftest.py b/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/conftest.py new file mode 100644 index 000000000000..a15b3f43b1c1 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/conftest.py @@ -0,0 +1,40 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Shared fixtures for deployment template E2E tests.""" + +from typing import Callable +import pytest +from azure.ai.ml import MLClient +from azure.ai.ml.entities import Environment + + +@pytest.fixture(scope="session") +def test_environment(registry_client: MLClient) -> str: + """Create or get a test environment in the registry for deployment templates. + + Returns: + The environment ID string to use in deployment templates. + """ + # Use a simple environment ID that references a basic docker image + # This doesn't require the environment to exist in the registry + return "azureml://registries/testFeed/environments/test-sklearn-env/versions/1" + + +@pytest.fixture +def deployment_template_name(randstr: Callable[[str], str]) -> str: + """Generate a unique deployment template name for testing.""" + return randstr("dt-e2e-") + + +@pytest.fixture +def basic_deployment_template_yaml() -> str: + """Path to basic deployment template YAML.""" + return "./tests/test_configs/deployment_template/basic_deployment_template.yaml" + + +@pytest.fixture +def minimal_deployment_template_yaml() -> str: + """Path to minimal deployment template YAML.""" + return "./tests/test_configs/deployment_template/minimal_deployment_template.yaml" diff --git a/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/test_deployment_template.py b/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/test_deployment_template.py new file mode 100644 index 000000000000..60b51e7dc005 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/test_deployment_template.py @@ -0,0 +1,541 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""E2E tests for Deployment Templates. + +These tests cover complete workflows for Deployment Template operations including: +- Create and retrieve deployment templates +- Update deployment templates +- List deployment templates +- Archive and restore deployment templates +- Delete deployment templates +- Load from YAML files +- Error handling scenarios +""" + +import logging +import pytest +from pathlib import Path +from typing import Callable + +from devtools_testutils import AzureRecordedTestCase +from test_utilities.utils import sleep_if_live + +from azure.ai.ml import MLClient, load_deployment_template +from azure.ai.ml.entities import DeploymentTemplate +from azure.core.exceptions import ResourceNotFoundError +from azure.core.paging import ItemPaged + + +logger = logging.getLogger(__name__) + + +def cleanup_template(client: MLClient, name: str, version: str) -> None: + """Helper function to clean up existing deployment template. + + Args: + client: ML registry client + name: Template name + version: Template version + """ + try: + client.deployment_templates.delete(name=name, version=version) + logger.info(f"Deleted existing template: {name} version {version}") + sleep_if_live(3) + except Exception: + pass # Template doesn't exist, continue + + +@pytest.mark.e2etest +@pytest.mark.usefixtures("recorded_test", "mock_asset_name") +class TestDeploymentTemplateE2E(AzureRecordedTestCase): + """E2E test class for Deployment Template operations.""" + + def test_deployment_template_create_and_get( + self, + registry_client: MLClient, + deployment_template_name: str, + basic_deployment_template_yaml: str, + ) -> None: + """Test creating and retrieving a deployment template. + + Args: + registry_client: ML registry client fixture + deployment_template_name: Generated unique template name + basic_deployment_template_yaml: Path to YAML config file + """ + template = None + try: + # Clean up any existing template from previous runs + try: + registry_client.deployment_templates.delete( + name=deployment_template_name, + version="1" + ) + logger.info(f"Deleted existing template: {deployment_template_name}") + sleep_if_live(5) + except Exception: + pass # Template doesn't exist, continue + + # Load deployment template from YAML + template = load_deployment_template(source=basic_deployment_template_yaml) + template.name = deployment_template_name + template.version = "1" + + # Create the deployment template + logger.info(f"Creating deployment template: {deployment_template_name}") + created_template = registry_client.deployment_templates.create_or_update(template) + + # Verify creation + assert created_template is not None + assert created_template.name == deployment_template_name + assert created_template.version == "1" + assert created_template.deployment_template_type == "Managed" + assert created_template.description == "Test deployment template for E2E testing" + assert "env" in created_template.tags + assert created_template.tags["env"] == "test" + + # Wait for propagation + sleep_if_live(5) + + # Get the deployment template + logger.info(f"Getting deployment template: {deployment_template_name}") + retrieved_template = registry_client.deployment_templates.get( + name=deployment_template_name, + version="1" + ) + + # Verify retrieved template + assert retrieved_template is not None + assert retrieved_template.name == deployment_template_name + assert retrieved_template.version == "1" + assert retrieved_template.deployment_template_type == "Managed" + assert retrieved_template.description == template.description + assert retrieved_template.instance_count == 2 + assert retrieved_template.default_instance_type == "Standard_DS3_v2" + + # Verify environment variables + assert retrieved_template.environment_variables is not None + assert "MODEL_PATH" in retrieved_template.environment_variables + assert "LOG_LEVEL" in retrieved_template.environment_variables + + # Verify request settings + if retrieved_template.request_settings: + assert retrieved_template.request_settings.request_timeout_ms == 5000 + assert retrieved_template.request_settings.max_concurrent_requests_per_instance == 1 + + finally: + # Cleanup + if template: + try: + logger.info(f"Cleaning up deployment template: {deployment_template_name}") + registry_client.deployment_templates.delete( + name=deployment_template_name, + version="1" + ) + except Exception as e: + logger.warning(f"Failed to cleanup template: {e}") + + def test_deployment_template_create_with_entity( + self, + registry_client: MLClient, + deployment_template_name: str, + ) -> None: + """Test creating a deployment template using DeploymentTemplate entity. + + Args: + registry_client: ML registry client fixture + deployment_template_name: Generated unique template name + """ + cleanup_template(registry_client, deployment_template_name, "1") + + try: + # Create deployment template entity programmatically + template = DeploymentTemplate( + name=deployment_template_name, + version="1", + description="E2E test template created from entity", + deployment_template_type="Managed", + environment="azureml://registries/testFeed/environments/test-sklearn-env/versions/1", + tags={"created_from": "entity", "test": "e2e"}, + environment_variables={ + "TEST_VAR": "test_value", + "ENV": "e2e" + }, + instance_count=1, + allowed_instance_types="Standard_DS2_v2", + default_instance_type="Standard_DS2_v2", + ) + + # Create the template + logger.info(f"Creating deployment template from entity: {deployment_template_name}") + created_template = registry_client.deployment_templates.create_or_update(template) + + # Verify creation + assert created_template is not None + assert created_template.name == deployment_template_name + assert created_template.version == "1" + assert created_template.description == "E2E test template created from entity" + assert created_template.tags["created_from"] == "entity" + assert created_template.environment_variables["TEST_VAR"] == "test_value" + + finally: + # Cleanup + try: + registry_client.deployment_templates.delete( + name=deployment_template_name, + version="1" + ) + except Exception as e: + logger.warning(f"Failed to cleanup template: {e}") + + def test_deployment_template_update( + self, + registry_client: MLClient, + deployment_template_name: str, + minimal_deployment_template_yaml: str, + ) -> None: + """Test updating a deployment template. + + Args: + client: ML client fixture + deployment_template_name: Generated unique template name + minimal_deployment_template_yaml: Path to YAML config file + """ + cleanup_template(registry_client, deployment_template_name, "1") + + try: + # Create initial template + template = load_deployment_template(source=minimal_deployment_template_yaml) + template.name = deployment_template_name + template.version = "1" + + logger.info(f"Creating initial deployment template: {deployment_template_name}") + created_template = registry_client.deployment_templates.create_or_update(template) + assert created_template.description == "Minimal deployment template for E2E testing" + + sleep_if_live(5) + + # Update the template + created_template.description = "Updated description for E2E test" + created_template.tags = {"updated": "true", "test": "e2e"} + + logger.info(f"Updating deployment template: {deployment_template_name}") + updated_template = registry_client.deployment_templates.create_or_update(created_template) + + # Verify update + assert updated_template.description == "Updated description for E2E test" + assert updated_template.tags["updated"] == "true" + + sleep_if_live(5) + + # Verify update persisted + retrieved_template = registry_client.deployment_templates.get( + name=deployment_template_name, + version="1" + ) + assert retrieved_template.description == "Updated description for E2E test" + + finally: + # Cleanup + try: + registry_client.deployment_templates.delete( + name=deployment_template_name, + version="1" + ) + except Exception as e: + logger.warning(f"Failed to cleanup template: {e}") + + def test_deployment_template_list( + self, + registry_client: MLClient, + deployment_template_name: str, + minimal_deployment_template_yaml: str, + ) -> None: + """Test listing deployment templates. + + Args: + client: ML client fixture + deployment_template_name: Generated unique template name + minimal_deployment_template_yaml: Path to YAML config file + """ + # Clean up all versions from previous runs + for version in ["1", "2", "3"]: + cleanup_template(registry_client, deployment_template_name, version) + + created_templates = [] + try: + # Create multiple versions of the template + for version in ["1", "2", "3"]: + template = load_deployment_template(source=minimal_deployment_template_yaml) + template.name = deployment_template_name + template.version = version + template.description = f"Version {version} for list test" + + logger.info(f"Creating deployment template version: {version}") + created = registry_client.deployment_templates.create_or_update(template) + created_templates.append(created) + sleep_if_live(2) + + sleep_if_live(5) + + # List all versions of the template + logger.info(f"Listing deployment template versions: {deployment_template_name}") + template_list = registry_client.deployment_templates.list(name=deployment_template_name) + + # Verify list results + assert template_list is not None + assert isinstance(template_list, ItemPaged) + + templates = list(template_list) + assert len(templates) >= 3 # At least our 3 versions + + # Verify our templates are in the list + assert len(templates) >= 3, f"Expected at least 3 templates but got {len(templates)}" + + finally: + # Cleanup all versions + for template in created_templates: + try: + registry_client.deployment_templates.delete( + name=template.name, + version=template.version + ) + except Exception as e: + logger.warning(f"Failed to cleanup template {template.version}: {e}") + + def test_deployment_template_archive_restore( + self, + registry_client: MLClient, + deployment_template_name: str, + minimal_deployment_template_yaml: str, + ) -> None: + """Test archiving and restoring a deployment template. + + Args: + client: ML client fixture + deployment_template_name: Generated unique template name + minimal_deployment_template_yaml: Path to YAML config file + """ + cleanup_template(registry_client, deployment_template_name, "1") + + try: + # Create template + template = load_deployment_template(source=minimal_deployment_template_yaml) + template.name = deployment_template_name + template.version = "1" + + logger.info(f"Creating deployment template: {deployment_template_name}") + created_template = registry_client.deployment_templates.create_or_update(template) + assert created_template.stage != "Archived" + + sleep_if_live(5) + + # Archive the template + logger.info(f"Archiving deployment template: {deployment_template_name}") + archived_template = registry_client.deployment_templates.archive( + name=deployment_template_name, + version="1" + ) + + # Verify archive + assert archived_template is not None + assert archived_template.stage == "Archived" + + sleep_if_live(5) + + # Verify archived state persisted + retrieved_template = registry_client.deployment_templates.get( + name=deployment_template_name, + version="1" + ) + assert retrieved_template.stage == "Archived" + + # Restore the template + logger.info(f"Restoring deployment template: {deployment_template_name}") + restored_template = registry_client.deployment_templates.restore( + name=deployment_template_name, + version="1" + ) + + # Verify restore + assert restored_template is not None + assert restored_template.stage != "Archived" + + sleep_if_live(5) + + # Verify restored state persisted + retrieved_template = registry_client.deployment_templates.get( + name=deployment_template_name, + version="1" + ) + assert retrieved_template.stage != "Archived" + + finally: + # Cleanup + try: + registry_client.deployment_templates.delete( + name=deployment_template_name, + version="1" + ) + except Exception as e: + logger.warning(f"Failed to cleanup template: {e}") + + def test_deployment_template_error_handling( + self, + registry_client: MLClient, + deployment_template_name: str, + ) -> None: + """Test error handling for invalid operations. + + Args: + client: ML client fixture + deployment_template_name: Generated unique template name + """ + # Test getting non-existent template + with pytest.raises(ResourceNotFoundError): + registry_client.deployment_templates.get( + name=deployment_template_name, + version="999" + ) + + # Test deleting non-existent template (note: delete may not be fully supported) + try: + registry_client.deployment_templates.delete( + name=deployment_template_name, + version="999" + ) + except (ResourceNotFoundError, AttributeError): + # Expected - either resource not found or delete not supported + pass + + # Test archiving non-existent template + with pytest.raises(ResourceNotFoundError): + registry_client.deployment_templates.archive( + name=deployment_template_name, + version="999" + ) + + def test_deployment_template_yaml_roundtrip( + self, + registry_client: MLClient, + deployment_template_name: str, + basic_deployment_template_yaml: str, + tmp_path: Path, + ) -> None: + """Test loading from YAML, creating, retrieving, and dumping back to YAML. + + Args: + client: ML client fixture + deployment_template_name: Generated unique template name + basic_deployment_template_yaml: Path to YAML config file + tmp_path: Pytest temporary directory fixture + """ + cleanup_template(registry_client, deployment_template_name, "1") + + try: + # Load from YAML + template = load_deployment_template(source=basic_deployment_template_yaml) + template.name = deployment_template_name + template.version = "1" + + # Create template + logger.info(f"Creating deployment template: {deployment_template_name}") + created_template = registry_client.deployment_templates.create_or_update(template) + + sleep_if_live(5) + + # Retrieve template + retrieved_template = registry_client.deployment_templates.get( + name=deployment_template_name, + version="1" + ) + + # Dump to YAML (if supported) + output_path = tmp_path / "roundtrip_template.yaml" + logger.info(f"Dumping template to YAML: {output_path}") + + try: + retrieved_template.dump(dest=str(output_path)) + # Verify file was created + assert output_path.exists() + + # Load dumped YAML and verify + reloaded_template = load_deployment_template(source=str(output_path)) + assert reloaded_template.name == deployment_template_name + assert reloaded_template.version == "1" + assert reloaded_template.deployment_template_type == template.deployment_template_type + except (AttributeError, NotImplementedError, AssertionError) as e: + logger.warning(f"YAML dump/roundtrip not fully supported: {e}") + + finally: + # Cleanup + try: + registry_client.deployment_templates.delete( + name=deployment_template_name, + version="1" + ) + except Exception as e: + logger.warning(f"Failed to cleanup template: {e}") + + def test_deployment_template_multiple_operations( + self, + registry_client: MLClient, + deployment_template_name: str, + minimal_deployment_template_yaml: str, + ) -> None: + """Test multiple operations in sequence on the same template. + + Args: + client: ML client fixture + deployment_template_name: Generated unique template name + minimal_deployment_template_yaml: Path to YAML config file + """ + template_name = deployment_template_name + cleanup_template(registry_client, template_name, "1") + + try: + # Create + template = load_deployment_template(source=minimal_deployment_template_yaml) + template.name = template_name + template.version = "1" + + logger.info(f"Creating deployment template: {template_name}") + created = registry_client.deployment_templates.create_or_update(template) + assert created.name == template_name + sleep_if_live(3) + + # Get + logger.info(f"Getting deployment template: {template_name}") + retrieved = registry_client.deployment_templates.get(name=template_name, version="1") + assert retrieved.name == template_name + sleep_if_live(2) + + # Update + logger.info(f"Updating deployment template: {template_name}") + retrieved.description = "Multi-operation test" + updated = registry_client.deployment_templates.create_or_update(retrieved) + assert updated.description == "Multi-operation test" + sleep_if_live(3) + + # Archive + logger.info(f"Archiving deployment template: {template_name}") + archived = registry_client.deployment_templates.archive(name=template_name, version="1") + assert archived.stage == "Archived" + sleep_if_live(3) + + # Restore + logger.info(f"Restoring deployment template: {template_name}") + restored = registry_client.deployment_templates.restore(name=template_name, version="1") + assert restored.stage != "Archived" + + finally: + # Cleanup + try: + registry_client.deployment_templates.delete( + name=template_name, + version="1" + ) + except Exception as e: + logger.warning(f"Failed to cleanup template: {e}") diff --git a/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/test_deployment_template_advanced.py b/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/test_deployment_template_advanced.py new file mode 100644 index 000000000000..fcff9bca548d --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/deployment_template/e2etests/test_deployment_template_advanced.py @@ -0,0 +1,534 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Advanced E2E tests for Deployment Templates. + +These tests cover advanced scenarios and integration testing: +- Complex deployment template configurations +- Integration with other Azure ML resources +- Performance and concurrency tests +- Edge cases and boundary conditions +""" + +import logging +import pytest + +from devtools_testutils import AzureRecordedTestCase +from test_utilities.utils import sleep_if_live + +from azure.ai.ml import MLClient, load_deployment_template +from azure.ai.ml.entities import DeploymentTemplate +from azure.ai.ml.entities._deployment.deployment_template_settings import ( + OnlineRequestSettings, + ProbeSettings +) + + + + +logger = logging.getLogger(__name__) + + +def cleanup_template(client: MLClient, name: str, version: str) -> None: + """Helper function to clean up existing deployment template. + + Args: + client: ML registry client + name: Template name + version: Template version + """ + try: + client.deployment_templates.delete(name=name, version=version) + logger.info(f"Deleted existing template: {name} version {version}") + sleep_if_live(3) + except Exception: + pass # Template doesn't exist, continue + + +@pytest.mark.e2etest +@pytest.mark.usefixtures("recorded_test", "mock_asset_name") +class TestDeploymentTemplateAdvancedE2E(AzureRecordedTestCase): + """Advanced E2E test class for Deployment Template operations.""" + + def test_deployment_template_with_custom_probes( + self, + registry_client: MLClient, + deployment_template_name: str, + ) -> None: + """Test creating a deployment template with custom liveness and readiness probes. + + Args: + registry_client: ML registry client fixture + deployment_template_name: Generated unique template name + """ + template_name = deployment_template_name + cleanup_template(registry_client, template_name, "1") + + try: + # Create liveness probe + liveness_probe = ProbeSettings( + initial_delay=60, + period=5, + timeout=1, + failure_threshold=3, + success_threshold=1, + scheme="HTTP", + method="GET", + path="/health/liveness", + port=8080 + ) + + # Create readiness probe + readiness_probe = ProbeSettings( + initial_delay=30, + period=5, + timeout=1, + failure_threshold=3, + success_threshold=1, + scheme="HTTP", + method="GET", + path="/health/readiness", + port=8080 + ) + + # Create deployment template with probes + template = DeploymentTemplate( + name=template_name, + version="1", + description="Template with custom probes", + deployment_template_type="Managed", + environment="azureml://registries/testFeed/environments/test-sklearn-env/versions/1", + liveness_probe=liveness_probe, + readiness_probe=readiness_probe, + instance_count=1, + allowed_instance_types="Standard_DS2_v2", + default_instance_type="Standard_DS2_v2", + ) + + logger.info(f"Creating deployment template with custom probes: {template_name}") + created_template = registry_client.deployment_templates.create_or_update(template) + + # Verify creation + assert created_template is not None + assert created_template.liveness_probe is not None + assert created_template.readiness_probe is not None + assert created_template.liveness_probe.initial_delay == 60 + assert created_template.readiness_probe.initial_delay == 30 + + sleep_if_live(5) + + # Get and verify probes + retrieved_template = registry_client.deployment_templates.get( + name=template_name, + version="1" + ) + assert retrieved_template.liveness_probe.path == "/health/liveness" + assert retrieved_template.readiness_probe.path == "/health/readiness" + + finally: + # Cleanup + try: + registry_client.deployment_templates.delete(name=template_name, version="1") + except Exception as e: + logger.warning(f"Failed to cleanup template: {e}") + + def test_deployment_template_with_request_settings( + self, + registry_client: MLClient, + deployment_template_name: str, + ) -> None: + """Test creating a deployment template with custom request settings. + + Args:`n registry_client: ML registry client fixture + deployment_template_name: Generated unique template name + """ + template_name = deployment_template_name + cleanup_template(registry_client, template_name, "1") + + try: + # Create request settings + request_settings = OnlineRequestSettings( + request_timeout_ms=10000, + max_concurrent_requests_per_instance=5 + ) + + # Create deployment template with request settings + template = DeploymentTemplate( + name=template_name, + version="1", + description="Template with request settings", + deployment_template_type="Managed", + environment="azureml://registries/testFeed/environments/test-sklearn-env/versions/1", + request_settings=request_settings, + instance_count=2, + allowed_instance_types="Standard_DS3_v2", + default_instance_type="Standard_DS3_v2", + ) + + logger.info(f"Creating deployment template with request settings: {template_name}") + created_template = registry_client.deployment_templates.create_or_update(template) + + # Verify creation + assert created_template is not None + assert created_template.request_settings is not None + assert created_template.request_settings.request_timeout_ms == 10000 + assert created_template.request_settings.max_concurrent_requests_per_instance == 5 + + sleep_if_live(5) + + # Get and verify request settings + retrieved_template = registry_client.deployment_templates.get( + name=template_name, + version="1" + ) + assert retrieved_template.request_settings.request_timeout_ms == 10000 + + finally: + # Cleanup + try: + registry_client.deployment_templates.delete(name=template_name, version="1") + except Exception as e: + logger.warning(f"Failed to cleanup template: {e}") + + def test_deployment_template_with_complex_environment_variables( + self, + registry_client: MLClient, + deployment_template_name: str, + ) -> None: + """Test creating a deployment template with complex environment variables. + + Args:`n registry_client: ML registry client fixture + deployment_template_name: Generated unique template name + """ + template_name = deployment_template_name + cleanup_template(registry_client, template_name, "1") + + try: + # Create complex environment variables + env_vars = { + "MODEL_PATH": "/var/azureml-app/models/primary", + "BACKUP_MODEL_PATH": "/var/azureml-app/models/backup", + "LOG_LEVEL": "DEBUG", + "ENABLE_METRICS": "true", + "BATCH_SIZE": "32", + "TIMEOUT_SECONDS": "300", + "API_VERSION": "v2", + "CORS_ORIGINS": "http://localhost:3000,https://example.com", + "MAX_RETRIES": "3", + "CACHE_ENABLED": "true" + } + + # Create deployment template + template = DeploymentTemplate( + name=template_name, + version="1", + description="Template with complex environment variables", + deployment_template_type="Managed", + environment="azureml://registries/testFeed/environments/test-sklearn-env/versions/1", + environment_variables=env_vars, + instance_count=1, + allowed_instance_types="Standard_DS2_v2", + default_instance_type="Standard_DS2_v2", + ) + + logger.info(f"Creating deployment template with env vars: {template_name}") + created_template = registry_client.deployment_templates.create_or_update(template) + + # Verify creation + assert created_template is not None + assert created_template.environment_variables is not None + assert len(created_template.environment_variables) == 10 + assert created_template.environment_variables["MODEL_PATH"] == "/var/azureml-app/models/primary" + assert created_template.environment_variables["ENABLE_METRICS"] == "true" + + sleep_if_live(5) + + # Get and verify all environment variables + retrieved_template = registry_client.deployment_templates.get( + name=template_name, + version="1" + ) + assert "CORS_ORIGINS" in retrieved_template.environment_variables + assert "MAX_RETRIES" in retrieved_template.environment_variables + + finally: + # Cleanup + try: + registry_client.deployment_templates.delete(name=template_name, version="1") + except Exception as e: + logger.warning(f"Failed to cleanup template: {e}") + + def test_deployment_template_version_management( + self, + registry_client: MLClient, + deployment_template_name: str, + ) -> None: + """Test managing multiple versions of the same deployment template. + + Args:`n registry_client: ML registry client fixture + deployment_template_name: Generated unique template name + """ + template_name = deployment_template_name + # Clean up all versions from previous runs + for i in range(1, 6): + cleanup_template(registry_client, template_name, str(i)) + + created_versions = [] + + try: + # Create multiple versions with different configurations + for i in range(1, 6): + template = DeploymentTemplate( + name=template_name, + version=str(i), + description=f"Version {i} of the template", + deployment_template_type="Managed", + environment="azureml://registries/testFeed/environments/test-sklearn-env/versions/1", + instance_count=i, # Different instance count per version + allowed_instance_types="Standard_DS2_v2", + default_instance_type="Standard_DS2_v2", + tags={"version_number": str(i), "test": "version_management"} + ) + + logger.info(f"Creating version {i} of template: {template_name}") + created = registry_client.deployment_templates.create_or_update(template) + created_versions.append(created) + assert created.version == str(i) + assert created.instance_count == i + sleep_if_live(2) + + sleep_if_live(5) + + # Verify all versions exist + for i in range(1, 6): + retrieved = registry_client.deployment_templates.get( + name=template_name, + version=str(i) + ) + assert retrieved.version == str(i) + assert retrieved.instance_count == i + assert retrieved.tags["version_number"] == str(i) + + # List all versions + template_list = registry_client.deployment_templates.list(name=template_name) + versions = list(template_list) + + # Verify we have at least our 5 versions + assert len(versions) >= 5, f"Expected at least 5 versions but got {len(versions)}" + + finally: + # Cleanup all versions + for i in range(1, 6): + try: + registry_client.deployment_templates.delete(name=template_name, version=str(i)) + except Exception as e: + logger.warning(f"Failed to cleanup version {i}: {e}") + + def test_deployment_template_with_tags( + self, + registry_client: MLClient, + deployment_template_name: str, + ) -> None: + """Test deployment template with various tag combinations. + + Args:`n registry_client: ML registry client fixture + deployment_template_name: Generated unique template name + """ + template_name = deployment_template_name + cleanup_template(registry_client, template_name, "1") + + try: + # Create template with multiple tags + tags = { + "environment": "production", + "team": "ml-ops", + "project": "recommendation-system", + "cost-center": "ML-001", + "compliance": "pci-dss", + "version": "2.0", + "owner": "ml-team@example.com", + "created-by": "e2e-test" + } + + template = DeploymentTemplate( + name=template_name, + version="1", + description="Template with comprehensive tags", + deployment_template_type="Managed", + environment="azureml://registries/testFeed/environments/test-sklearn-env/versions/1", + tags=tags, + instance_count=1, + allowed_instance_types="Standard_DS2_v2", + default_instance_type="Standard_DS2_v2", + ) + + logger.info(f"Creating deployment template with tags: {template_name}") + created_template = registry_client.deployment_templates.create_or_update(template) + + # Verify tags + assert created_template is not None + assert len(created_template.tags) == 8 + assert created_template.tags["environment"] == "production" + assert created_template.tags["team"] == "ml-ops" + + sleep_if_live(5) + + # Update tags + retrieved_template = registry_client.deployment_templates.get( + name=template_name, + version="1" + ) + retrieved_template.tags["updated"] = "true" + retrieved_template.tags["update-date"] = "2024-01-01" + + updated_template = registry_client.deployment_templates.create_or_update(retrieved_template) + assert "updated" in updated_template.tags + assert len(updated_template.tags) == 10 + + finally: + # Cleanup + try: + registry_client.deployment_templates.delete(name=template_name, version="1") + except Exception as e: + logger.warning(f"Failed to cleanup template: {e}") + + def test_deployment_template_minimal_to_full_update( + self, + registry_client: MLClient, + deployment_template_name: str, + ) -> None: + """Test updating a minimal template to a full-featured template. + + Args:`n registry_client: ML registry client fixture + deployment_template_name: Generated unique template name + """ + template_name = deployment_template_name + cleanup_template(registry_client, template_name, "1") + + try: + # Create minimal template + minimal_template = DeploymentTemplate( + name=template_name, + version="1", + deployment_template_type="Managed", + environment="azureml://registries/testFeed/environments/test-sklearn-env/versions/1", + instance_count=1, + allowed_instance_types="Standard_DS2_v2", + default_instance_type="Standard_DS2_v2", + ) + + logger.info(f"Creating minimal deployment template: {template_name}") + created_template = registry_client.deployment_templates.create_or_update(minimal_template) + assert created_template.description is None + assert created_template.tags in [None, {}] + + sleep_if_live(5) + + # Create version 2 with full features (templates are immutable, can't update existing version) + full_template = DeploymentTemplate( + name=template_name, + version="2", + deployment_template_type="Managed", + environment="azureml://registries/testFeed/environments/test-sklearn-env/versions/1", + description="Upgraded to full-featured template", + tags={"upgraded": "true", "feature_level": "full"}, + environment_variables={ + "FEATURE_SET": "complete", + "UPGRADED": "true" + }, + request_settings=OnlineRequestSettings( + request_timeout_ms=8000, + max_concurrent_requests_per_instance=3 + ), + instance_count=2, + allowed_instance_types="Standard_DS2_v2", + default_instance_type="Standard_DS2_v2", + ) + + logger.info(f"Creating full-featured template version 2: {template_name}") + updated_template = registry_client.deployment_templates.create_or_update(full_template) + + # Verify upgrade + assert updated_template.description == "Upgraded to full-featured template" + assert updated_template.tags["upgraded"] == "true" + assert updated_template.environment_variables["FEATURE_SET"] == "complete" + assert updated_template.request_settings.request_timeout_ms == 8000 + + finally: + # Cleanup both versions + for version in ["1", "2"]: + try: + registry_client.deployment_templates.delete(name=template_name, version=version) + except Exception as e: + logger.warning(f"Failed to cleanup template version {version}: {e}") + + def test_deployment_template_concurrent_versions( + self, + registry_client: MLClient, + deployment_template_name: str, + ) -> None: + """Test creating and managing concurrent versions of deployment templates. + + Args:`n registry_client: ML registry client fixture + deployment_template_name: Generated unique template name + """ + template_name = deployment_template_name + versions = ["1", "2", "3"] + # Clean up all versions from previous runs + for version in versions: + cleanup_template(registry_client, template_name, version) + + try: + # Create all versions + for version in versions: + template = DeploymentTemplate( + name=template_name, + version=version, + description=f"Concurrent version {version}", + deployment_template_type="Managed", + environment="azureml://registries/testFeed/environments/test-sklearn-env/versions/1", + instance_count=int(version), + allowed_instance_types="Standard_DS2_v2", + default_instance_type="Standard_DS2_v2", + tags={"version": version} + ) + + logger.info(f"Creating concurrent version {version}: {template_name}") + registry_client.deployment_templates.create_or_update(template) + sleep_if_live(2) + + sleep_if_live(5) + + # Verify all versions coexist + for version in versions: + retrieved = registry_client.deployment_templates.get( + name=template_name, + version=version + ) + assert retrieved.version == version + assert retrieved.instance_count == int(version) + + # Archive one version while others remain active + logger.info(f"Archiving version 2 of {template_name}") + registry_client.deployment_templates.archive(name=template_name, version="2") + sleep_if_live(3) + + # Verify version 2 is archived, others are not + v1 = registry_client.deployment_templates.get(name=template_name, version="1") + v2 = registry_client.deployment_templates.get(name=template_name, version="2") + v3 = registry_client.deployment_templates.get(name=template_name, version="3") + + assert v1.stage != "Archived" + assert v2.stage == "Archived" + assert v3.stage != "Archived" + + finally: + # Cleanup all versions + for version in versions: + try: + registry_client.deployment_templates.delete(name=template_name, version=version) + except Exception as e: + logger.warning(f"Failed to cleanup version {version}: {e}") + + diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/deployment_template/basic_deployment_template.yaml b/sdk/ml/azure-ai-ml/tests/test_configs/deployment_template/basic_deployment_template.yaml new file mode 100644 index 000000000000..bd9add97aa18 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/test_configs/deployment_template/basic_deployment_template.yaml @@ -0,0 +1,20 @@ +$schema: https://azuremlschemas.azureedge.net/latest/deploymentTemplate.schema.json +name: test-basic-template +version: 1 +type: deploymenttemplates +deployment_template_type: Managed +description: Test deployment template for E2E testing +environment: azureml://registries/testFeed/environments/test-sklearn-env/versions/1 +tags: + env: test + type: basic +instance_count: 2 +default_instance_type: Standard_DS3_v2 +allowed_instance_types: Standard_DS2_v2,Standard_DS3_v2 +request_settings: + request_timeout_ms: 5000 + max_concurrent_requests_per_instance: 1 +environment_variables: + MODEL_PATH: /var/azureml-app/azureml-models/model/1 + LOG_LEVEL: INFO + MODEL_NAME: test_model diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/deployment_template/complete_deployment_template.yaml b/sdk/ml/azure-ai-ml/tests/test_configs/deployment_template/complete_deployment_template.yaml new file mode 100644 index 000000000000..ca81fd996d92 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/test_configs/deployment_template/complete_deployment_template.yaml @@ -0,0 +1,45 @@ +$schema: https://azuremlschemas.azureedge.net/latest/deploymentTemplate.schema.json +name: test-complete-template +version: 1 +type: deploymenttemplates +deployment_template_type: Managed +description: Complete deployment template with all features for E2E testing +tags: + test: e2e + type: complete + stage: development +environment: azureml://registries/testFeed/environments/test-sklearn-env/versions/1 +model_mount_path: /var/azureml-app +request_settings: + request_timeout_ms: 6000 + max_concurrent_requests_per_instance: 2 +scoring_path: /v1/models/test_model:predict +scoring_port: 8501 +liveness_probe: + initial_delay: 300 + period: 10 + timeout: 2 + failure_threshold: 30 + success_threshold: 1 + scheme: HTTP + method: GET + path: /v1/models/test_model + port: 8501 +readiness_probe: + initial_delay: 300 + period: 10 + timeout: 2 + failure_threshold: 30 + success_threshold: 1 + scheme: HTTP + method: GET + path: /v1/models/test_model + port: 8501 +environment_variables: + MODEL_BASE_PATH: /var/azureml-app/azureml-models/model/1 + MODEL_NAME: test_model + ENABLE_DIAGNOSTICS: "true" +allowed_instance_types: Standard_DS2_v2,Standard_DS3_v2,Standard_DS4_v2 +default_instance_type: Standard_DS3_v2 +instance_count: 2 +instance_type: Standard_DS3_v2 diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/deployment_template/minimal_deployment_template.yaml b/sdk/ml/azure-ai-ml/tests/test_configs/deployment_template/minimal_deployment_template.yaml new file mode 100644 index 000000000000..02ffcbefe8ae --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/test_configs/deployment_template/minimal_deployment_template.yaml @@ -0,0 +1,9 @@ +$schema: https://azuremlschemas.azureedge.net/latest/deploymentTemplate.schema.json +name: test-minimal-template +version: 1 +type: deploymenttemplates +deployment_template_type: Managed +description: Minimal deployment template for E2E testing +environment: azureml://registries/testFeed/environments/test-sklearn-env/versions/1 +instance_count: 1 +default_instance_type: Standard_DS2_v2