diff --git a/monitoring/benchmarker/configurations/actions/__init__.py b/monitoring/benchmarker/configurations/actions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/monitoring/benchmarker/configurations/actions.py b/monitoring/benchmarker/configurations/actions/action.py similarity index 93% rename from monitoring/benchmarker/configurations/actions.py rename to monitoring/benchmarker/configurations/actions/action.py index 3caabf3819..d9c5a716e4 100644 --- a/monitoring/benchmarker/configurations/actions.py +++ b/monitoring/benchmarker/configurations/actions/action.py @@ -2,6 +2,7 @@ from implicitdict import ImplicitDict +from monitoring.benchmarker.configurations.actions.f3411 import F3411ActionSpecification from monitoring.benchmarker.configurations.artifacts.artifact import ( ArtifactSpecification, ) @@ -49,4 +50,6 @@ class BenchmarkActionSpecification(ImplicitDict): run_command: Optional[RunCommandActionSpecification] + f3411: Optional[F3411ActionSpecification] + generate_artifacts: Optional[GenerateArtifactsActionSpecification] diff --git a/monitoring/benchmarker/configurations/actions/astm.py b/monitoring/benchmarker/configurations/actions/astm.py new file mode 100644 index 0000000000..274d08be41 --- /dev/null +++ b/monitoring/benchmarker/configurations/actions/astm.py @@ -0,0 +1,11 @@ +from enum import StrEnum + + +class SubscriptionCreationMode(StrEnum): + GetDeleteCreate = "GetDeleteCreate" + """First, attempt to get an existing subscription with this ID. If the subscription exists, delete it. Then, create the new subscription as specified.""" + + +class SubscriptionDeletionMode(StrEnum): + GetDeleteIfExist = "GetDeleteIfExist" + """First, attempt to get an existing subscription with this ID. If the subscription exists, delete it. If the subscription doesn't exist, do nothing.""" diff --git a/monitoring/benchmarker/configurations/actions/f3411.py b/monitoring/benchmarker/configurations/actions/f3411.py new file mode 100644 index 0000000000..561e6f8394 --- /dev/null +++ b/monitoring/benchmarker/configurations/actions/f3411.py @@ -0,0 +1,54 @@ +from typing import Optional + +from implicitdict import ImplicitDict, StringBasedTimeDelta + +from monitoring.benchmarker.configurations.actions.astm import ( + SubscriptionCreationMode, + SubscriptionDeletionMode, +) +from monitoring.monitorlib.geo import Altitude, LatLngBoundingBox +from monitoring.monitorlib.rid import RIDVersion + + +class Subscription(ImplicitDict): + subscription_id: str + """ID of the single subscription to create.""" + + rid_version: RIDVersion + + duration: StringBasedTimeDelta + """Duration of the subscription, from the time it is created.""" + + area: LatLngBoundingBox + """Horizontal area this subscription should cover.""" + + min_alt: Altitude + """Altitude below which this subscription should not apply.""" + + max_alt: Altitude + """Altitude above which this subscription should not apply.""" + + +class CreateSubscription(ImplicitDict): + """Create a subscription.""" + + subscription: Subscription + """Characteristics of subscription to create.""" + + mode: SubscriptionCreationMode + """Desired creation behavior.""" + + +class DeleteSubscription(ImplicitDict): + subscription_id: str + """ID of the subscription to delete.""" + + mode: SubscriptionDeletionMode + """Desired deletion behavior.""" + + +class F3411ActionSpecification(ImplicitDict): + """Actions pertaining to ASTM F3411 NetRID.""" + + create_subscription: Optional[CreateSubscription] + delete_subscription: Optional[DeleteSubscription] diff --git a/monitoring/benchmarker/configurations/configuration.py b/monitoring/benchmarker/configurations/configuration.py index 052e41fb33..bc650e3948 100644 --- a/monitoring/benchmarker/configurations/configuration.py +++ b/monitoring/benchmarker/configurations/configuration.py @@ -2,7 +2,8 @@ from implicitdict import ImplicitDict -from monitoring.benchmarker.configurations.actions import ( +from monitoring.benchmarker.configurations.actions.action import ( + BenchmarkActionName, BenchmarkActionSpecification, ) from monitoring.benchmarker.configurations.artifacts.artifact import ( @@ -34,3 +35,9 @@ class BenchmarkConfiguration(ImplicitDict): artifacts: Optional[list[ArtifactSpecification]] """Artifacts to produce from the data collected during the benchmarker run.""" + + setup_actions: Optional[list[BenchmarkActionName]] + """Actions to perform before beginning scenarios.""" + + teardown_actions: Optional[list[BenchmarkActionName]] + """Actions to perform after completing all scenarios.""" diff --git a/monitoring/benchmarker/configurations/interuss/netrid/isas_images.jsonnet b/monitoring/benchmarker/configurations/interuss/netrid/isas_images.jsonnet new file mode 100644 index 0000000000..c450dd9527 --- /dev/null +++ b/monitoring/benchmarker/configurations/interuss/netrid/isas_images.jsonnet @@ -0,0 +1,314 @@ +local num_uss = 3; +local num_nodes = 3; +local num_subscriptions = 8; +local dss_images = ['interuss-local/dss:master_590e569', 'interuss-local/dss:fix_1509_patch']; + +local nodeIndex = function(uss, node) std.format('%02d', node + num_nodes * (uss - 1)); + +{ + resources: { + resource_declarations: { + utm_auth: { + resource_type: 'resources.communications.AuthAdapterResource', + specification: { + auth_spec: 'DummyOAuth(http://localhost:8085/token,benchmarker)', + scopes_authorized: [ + 'rid.service_provider', + 'rid.display_provider', + ], + }, + }, + } + { + ["dss_pool_%d" % uss]: { + resource_type: 'resources.astm.f3411.DSSInstancesResource', + dependencies: { + auth_adapter: 'utm_auth', + }, + specification: { + dss_instances: [ + { + participant_id: 'uss%(uss)d_dss%(node)d' % { uss: uss, node: node }, + base_url: 'http://localhost:80%s/rid/v2' % nodeIndex(uss, node), + rid_version: 'F3411-22a', + } for node in std.range(1, num_nodes) + ], + }, + } for uss in std.range(1, num_uss) + }, + }, + + actions: [ + { + name: 'Bring up DSS pool: %s' % dss_image, + run_command: { + env: { + NUM_USS: std.toString(num_uss), + NUM_NODES: std.toString(num_nodes), + DSS_IMAGE: dss_image, + DB_TYPE: 'crdb', + INTRA_USS_NETEM_CONF: 'delay 600us 40us 25% distribution normal loss 0.0005%', + INTER_USS_NETEM_CONF: 'delay 10ms 2ms 50% distribution paretonormal loss 0.25% 5%', + CORE_SERVICE_EXTRA_FLAGS: '--enable_time_based_notification_index', + }, + path: '$REPO_ROOT', + command: 'make start-locally', + }, + } + for dss_image in dss_images + ] + [ + { + name: 'Tear down DSS pool', + run_command: { + env: { + NUM_USS: std.toString(num_uss), + NUM_NODES: std.toString(num_nodes), + DB_TYPE: 'crdb', + }, + path: '$REPO_ROOT', + command: 'make clean-locally', + }, + } + ] + [ + { + name: 'Create subscription %d' % sub_index, + f3411: { + create_subscription: { + subscription: { + subscription_id: '16b87239-6063-47d4-a2ff-%d05086859f32' % (sub_index - 1), + rid_version: 'F3411-22a', + duration: '23h', + area: { + lat_min: 34 - 0.00001, + lng_min: -118 - 0.00001, + lat_max: 34 + 0.00001, + lng_max: -118 + 0.00001, + }, + min_alt: {value: 0, units: 'M', reference: 'W84'}, + max_alt: {value: 3000, units: 'M', reference: 'W84'}, + }, + mode: 'GetDeleteCreate', + }, + }, + } for sub_index in std.range(1, num_subscriptions) + ] + [ + { + name: 'Delete subscription %d' % sub_index, + f3411: { + delete_subscription: { + subscription_id: '16b87239-6063-47d4-a2ff-%d05086859f32' % (sub_index - 1), + mode: 'GetDeleteIfExist', + }, + }, + } for sub_index in std.range(1, num_subscriptions) + ], + + user_types: [ + { + name: 'FPU%d' % uss, // Flight planner user, DSS instance/USS i + flight_planner: { + flight_generation: { + independent_time_location_shape: { + time: { + fixed_spacing: '0s', + }, + location: { + fixed_location: { + horizontal: {lat: 34, lng: -118}, + vertical: {value: 300, reference: 'W84', units: 'M'}, + }, + }, + shape: { + fixed_volumes: { + origin_horizontal: {lat: 0, lng: 0}, + origin_vertical: {value: 0, reference: 'W84', units: 'M'}, + origin_time: '2026-01-01T00:00:00Z', + volumes: [ + { + volume: { + outline_polygon: { + vertices: [ + {lat: -0.00001, lng: -0.00001}, + {lat: 0.00001, lng: -0.00001}, + {lat: 0.00001, lng: 0.00001}, + {lat: -0.00001, lng: 0.00001}, + ], + }, + altitude_lower: {value: 0, reference: 'W84', units: 'M'}, + altitude_upper: {value: 20, reference: 'W84', units: 'M'}, + }, + time_start: '2026-01-01T00:00:00Z', + time_end: '2026-01-01T00:00:05Z', + }, + ], + }, + }, + }, + }, + astm_netrid_behavior: { + rid_version: 'F3411-22a', + dss_pool: ['dss_pool_%d' % uss], + dss_selection_strategy: 'Random', + isa_strategy: { + isa_per_flight: { + before_flight_start: '0s', + after_flight_end: '2s', + }, + }, + }, + }, + } for uss in std.range(1, num_uss) + ], + + loads: [ + { + name: 'Flight planner ramp for DSS instance %d' % uss, + user_ramp: { + user_type: 'FPU%d' % uss, + initial_users: 7, + additional_users_per_step: 1, + throughput_stability_criteria: { + each_user_completed_at_least: { + count: 1, + operations: ['workflow.flight_planner.flight'], + }, + }, + step_completion_criteria: { + any_of: [ + { + sampling_duration_at_least: '30s', + }, + { + completed_at_least: { + count: 100, + operations: ['workflow.flight_planner.flight'], + }, + }, + { + average_duration_more_than: { + duration: '20s', + operations: ['workflow.flight_planner.flight'], + }, + }, + ], + sampling_duration_at_least: '10s', + completed_at_least: { + count: 5, + operations: ['workflow.flight_planner.flight'], + } + }, + load_completion_criteria: { + any_of: [ + { + throughput_lower_than_peak: { + operations: ['workflow.flight_planner.flight'], + fraction_of_peak: 0.7, + }, + }, + { + failures_more_than: { + count: 10, + operations: ['workflow.flight_planner.flight'], + } + }, + { + most_recent_step: { + average_duration_more_than: { + duration: '20s', + operations: ['workflow.flight_planner.flight'], + }, + }, + }, + { + most_recent_step: { + throughput_stability_took_longer_than: '30s', + }, + }, + ], + }, + }, + } for uss in std.range(1, num_uss) + ], + + scenarios: std.flattenArrays([ + [ + { + name: 'Instance %(uss)d %(img)s' % { uss: uss, img: dss_image }, + [if uss == 1 then "setup"]: ['Bring up DSS pool: %s' % dss_image] + ['Create subscription %d' % sub_index for sub_index in std.range(1, num_subscriptions)], + load: 'Flight planner ramp for DSS instance %d' % uss, + [if uss == num_uss then "teardown"]: ['Delete subscription %d' % sub_index for sub_index in std.range(1, num_subscriptions)] + ['Tear down DSS pool'], + metadata: { + dss_image: dss_image, + dss_instance: uss, + } + } for uss in std.range(1, num_uss) + ] for dss_image in dss_images + ]), + + artifacts: [ + { + raw_report: { + name: 'report', + }, + }, + { + matplotlib_figure: { + local aspect_ratio = 16 / 9, + local n_cols = std.ceil(std.sqrt(std.length(dss_images) * aspect_ratio)), + local n_rows = std.ceil(std.length(dss_images) / n_cols), + + name: 'throughput_by_dss_image', + n_subfigure_cols: n_cols, + n_subfigure_rows: n_rows, + subfigures: [ + { + title: dss_image, + subplots: [ + { + x_axis: { + label: 'Flight planners', + }, + y_axis: { + label: 'Throughput\n(Flights/s)', + }, + xy_plots: [ + { + type: 'Scatter', + label_expr: '"DSS %d"' % uss, + evaluation_context: [ + { + name: 'scenarios', + value: ('[' + + 's for s in report.report.scenarios ' + + 'if s.metadata["dss_image"] == "%(img)s" and s.metadata["dss_instance"] == %(uss)d' + + ']') % + {img: dss_image, uss: uss}, + }, + { + name: 'scale', + value: '[step.load_factor for step in scenarios[0].steps]', + }, + { + name: 'throughput', + value: '[throughput_of_step(scenarios[0], s, types=["workflow.flight_planner.flight"])' + + ' for s in range(len(scenarios[0].steps))]', + }, + ], + render_expr: 'scenarios', + x_data_expr: 'scale', + y_data_expr: 'throughput', + } for uss in std.range(1, num_uss) + ], + legend: { + location: 'upper left', + font_size: 'x-small', + label_spacing: 0.2, + border_padding: 0.2, + }, + } + ], + } for dss_image in dss_images + ], + }, + }, + ], +} diff --git a/monitoring/benchmarker/configurations/scenarios.py b/monitoring/benchmarker/configurations/scenarios.py index 1a8c573ee4..cb516eece1 100644 --- a/monitoring/benchmarker/configurations/scenarios.py +++ b/monitoring/benchmarker/configurations/scenarios.py @@ -2,7 +2,7 @@ from implicitdict import ImplicitDict -from monitoring.benchmarker.configurations.actions import BenchmarkActionName +from monitoring.benchmarker.configurations.actions.action import BenchmarkActionName from monitoring.benchmarker.configurations.loads import BenchmarkLoadName diff --git a/monitoring/benchmarker/engine/actions/actions.py b/monitoring/benchmarker/engine/actions/actions.py index 19eab83375..bc7a29b9f8 100644 --- a/monitoring/benchmarker/engine/actions/actions.py +++ b/monitoring/benchmarker/engine/actions/actions.py @@ -1,13 +1,17 @@ -from monitoring.benchmarker.configurations.actions import ( +from typing import Any + +from monitoring.benchmarker.configurations.actions.action import ( BenchmarkActionName, BenchmarkActionSpecification, ) from monitoring.benchmarker.configurations.configuration import BenchmarkConfiguration +from monitoring.benchmarker.engine.actions.f3411 import run_f3411_action from monitoring.benchmarker.engine.actions.generate_artifacts import ( generate_intermediate_artifacts, ) from monitoring.benchmarker.engine.actions.run_command import run_command from monitoring.benchmarker.reports.report import BenchmarkScenarioReport +from monitoring.uss_qualifier.resources.definitions import ResourceID def run_scenario_actions( @@ -19,6 +23,7 @@ def run_scenario_actions( output_dir: str, codebase_version: str, commit_hash: str, + resource_pool: dict[ResourceID, Any] | None = None, ) -> None: """Run a sequence of scenario setup or teardown actions by name.""" if not action_names: @@ -27,7 +32,7 @@ def run_scenario_actions( for action_name in action_names: if action_name not in action_specs: raise ValueError( - f"Scenario action '{action_name}' not defined in configuration.actions" + f"Action '{action_name}' not defined in configuration.actions" ) action_spec = action_specs[action_name] invocation = action_invocations.get(action_name, 0) @@ -51,6 +56,13 @@ def run_scenario_actions( commit_hash, ) action_performed = True + if "f3411" in action_spec and action_spec.f3411 is not None: + if resource_pool is None: + raise ValueError( + f"Resource pool is required to execute action '{action_name}' with F3411 specification" + ) + run_f3411_action(action_name, action_spec.f3411, resource_pool) + action_performed = True if not action_performed: raise NotImplementedError( diff --git a/monitoring/benchmarker/engine/actions/f3411.py b/monitoring/benchmarker/engine/actions/f3411.py new file mode 100644 index 0000000000..0572667696 --- /dev/null +++ b/monitoring/benchmarker/engine/actions/f3411.py @@ -0,0 +1,210 @@ +from datetime import UTC, datetime +from typing import Any + +from loguru import logger + +from monitoring.benchmarker.configurations.actions.action import BenchmarkActionName +from monitoring.benchmarker.configurations.actions.astm import ( + SubscriptionCreationMode, + SubscriptionDeletionMode, +) +from monitoring.benchmarker.configurations.actions.f3411 import ( + CreateSubscription, + DeleteSubscription, + F3411ActionSpecification, +) +from monitoring.monitorlib.fetch import rid as fetch_rid +from monitoring.monitorlib.mutate import rid as mutate_rid +from monitoring.monitorlib.rid import RIDVersion +from monitoring.monitorlib.testing import make_fake_url +from monitoring.uss_qualifier.resources.astm.f3411.dss import ( + DSSInstance, + DSSInstanceResource, + DSSInstancesResource, +) +from monitoring.uss_qualifier.resources.definitions import ResourceID + + +def get_dss_instances(resource_pool: dict[ResourceID, Any]) -> list[DSSInstance]: + """Retrieve all F3411 DSS instances from the resource pool.""" + dss_instances: list[DSSInstance] = [] + for res in resource_pool.values(): + if isinstance(res, DSSInstanceResource): + dss_instances.append(res.dss_instance) + elif isinstance(res, DSSInstancesResource): + dss_instances.extend(res.dss_instances) + return dss_instances + + +def select_dss_instance( + dss_instances: list[DSSInstance], rid_version: RIDVersion | None = None +) -> DSSInstance: + """Select a DSS instance matching the rid_version (or the first available).""" + if not dss_instances: + raise ValueError("No ASTM F3411 DSS instances found in resource pool") + if rid_version is not None: + matching = [dss for dss in dss_instances if dss.rid_version == rid_version] + if not matching: + raise ValueError( + f"No ASTM F3411 DSS instances found in resource pool matching RID version '{rid_version}'" + ) + return matching[0] + return dss_instances[0] + + +def create_subscription( + spec: CreateSubscription, + resource_pool: dict[ResourceID, Any], +) -> None: + dss_instances = get_dss_instances(resource_pool) + sub = spec.subscription + dss_instance = select_dss_instance(dss_instances, sub.rid_version) + + if spec.mode == SubscriptionCreationMode.GetDeleteCreate: + logger.info( + f"F3411 Action: Checking if subscription '{sub.subscription_id}' exists before creating..." + ) + fetched_sub = fetch_rid.subscription( + subscription_id=sub.subscription_id, + rid_version=sub.rid_version, + session=dss_instance.client, + participant_id=dss_instance.participant_id, + ) + if fetched_sub.status_code == 200 and fetched_sub.subscription: + logger.info( + f"F3411 Action: Existing subscription '{sub.subscription_id}' found (version {fetched_sub.subscription.version}); deleting it..." + ) + del_result = mutate_rid.delete_subscription( + subscription_id=sub.subscription_id, + subscription_version=fetched_sub.subscription.version, + rid_version=sub.rid_version, + utm_client=dss_instance.client, + participant_id=dss_instance.participant_id, + ) + if not del_result.success: + raise RuntimeError( + f"Failed to delete existing subscription '{sub.subscription_id}' during GetDeleteCreate: {del_result.errors}" + ) + elif fetched_sub.status_code == 404: + logger.info( + f"F3411 Action: Subscription '{sub.subscription_id}' does not exist; proceeding to create." + ) + else: + raise RuntimeError( + f"Failed to query subscription '{sub.subscription_id}' during GetDeleteCreate: {fetched_sub.errors}" + ) + + logger.info(f"F3411 Action: Creating subscription '{sub.subscription_id}'...") + uss_base_url = make_fake_url() + t0 = datetime.now(UTC) + create_result = mutate_rid.upsert_subscription( + area_vertices=sub.area.to_vertices(), + alt_lo=sub.min_alt.to_w84_m(), + alt_hi=sub.max_alt.to_w84_m(), + start_time=t0, + end_time=t0 + sub.duration.timedelta, + uss_base_url=uss_base_url, + subscription_id=sub.subscription_id, + rid_version=sub.rid_version, + utm_client=dss_instance.client, + participant_id=dss_instance.participant_id, + ) + if not create_result.success: + raise RuntimeError( + f"Failed to create subscription '{sub.subscription_id}': {create_result.errors}" + ) + logger.info( + f"F3411 Action: Successfully created subscription '{sub.subscription_id}'." + ) + else: + raise NotImplementedError( + f"Unsupported subscription creation mode '{spec.mode}'" + ) + + +def delete_subscription( + spec: DeleteSubscription, + resource_pool: dict[ResourceID, Any], +) -> None: + dss_instances = get_dss_instances(resource_pool) + if not dss_instances: + raise ValueError("No ASTM F3411 DSS instances found in resource pool") + + if spec.mode == SubscriptionDeletionMode.GetDeleteIfExist: + logger.info( + f"F3411 Action: Checking if subscription '{spec.subscription_id}' exists before deleting..." + ) + checked_versions: set[RIDVersion] = set() + deleted = False + for dss_instance in dss_instances: + if dss_instance.rid_version in checked_versions: + continue + checked_versions.add(dss_instance.rid_version) + + fetched_sub = fetch_rid.subscription( + subscription_id=spec.subscription_id, + rid_version=dss_instance.rid_version, + session=dss_instance.client, + participant_id=dss_instance.participant_id, + ) + if fetched_sub.status_code == 200 and fetched_sub.subscription: + logger.info( + f"F3411 Action: Existing subscription '{spec.subscription_id}' found (version {fetched_sub.subscription.version}); deleting it..." + ) + del_result = mutate_rid.delete_subscription( + subscription_id=spec.subscription_id, + subscription_version=fetched_sub.subscription.version, + rid_version=dss_instance.rid_version, + utm_client=dss_instance.client, + participant_id=dss_instance.participant_id, + ) + if not del_result.success: + raise RuntimeError( + f"Failed to delete subscription '{spec.subscription_id}': {del_result.errors}" + ) + logger.info( + f"F3411 Action: Successfully deleted subscription '{spec.subscription_id}'." + ) + deleted = True + break + elif fetched_sub.status_code == 404: + continue + else: + raise RuntimeError( + f"Failed to query subscription '{spec.subscription_id}' during GetDeleteIfExist: {fetched_sub.errors}" + ) + + if not deleted: + logger.info( + f"F3411 Action: Subscription '{spec.subscription_id}' did not exist; nothing to delete." + ) + else: + raise NotImplementedError( + f"Unsupported subscription deletion mode '{spec.mode}'" + ) + + +def run_f3411_action( + action_name: BenchmarkActionName, + f3411_spec: F3411ActionSpecification, + resource_pool: dict[ResourceID, Any], +) -> None: + action_performed = False + if ( + "create_subscription" in f3411_spec + and f3411_spec.create_subscription is not None + ): + logger.info(f"Action '{action_name}': Creating F3411 subscription...") + create_subscription(f3411_spec.create_subscription, resource_pool) + action_performed = True + if ( + "delete_subscription" in f3411_spec + and f3411_spec.delete_subscription is not None + ): + logger.info(f"Action '{action_name}': Deleting F3411 subscription...") + delete_subscription(f3411_spec.delete_subscription, resource_pool) + action_performed = True + if not action_performed: + raise ValueError( + f"Action '{action_name}' F3411ActionSpecification did not specify any supported action" + ) diff --git a/monitoring/benchmarker/engine/actions/generate_artifacts.py b/monitoring/benchmarker/engine/actions/generate_artifacts.py index bce3dba7eb..0269d366cd 100644 --- a/monitoring/benchmarker/engine/actions/generate_artifacts.py +++ b/monitoring/benchmarker/engine/actions/generate_artifacts.py @@ -4,7 +4,7 @@ from loguru import logger from monitoring.benchmarker.artifacts.generation import generate_artifacts -from monitoring.benchmarker.configurations.actions import ( +from monitoring.benchmarker.configurations.actions.action import ( BenchmarkActionName, GenerateArtifactsActionSpecification, ) diff --git a/monitoring/benchmarker/engine/actions/run_command.py b/monitoring/benchmarker/engine/actions/run_command.py index 85e88ff60d..0ecd7a98a7 100644 --- a/monitoring/benchmarker/engine/actions/run_command.py +++ b/monitoring/benchmarker/engine/actions/run_command.py @@ -3,7 +3,9 @@ from loguru import logger -from monitoring.benchmarker.configurations.actions import RunCommandActionSpecification +from monitoring.benchmarker.configurations.actions.action import ( + RunCommandActionSpecification, +) from monitoring.monitorlib.versioning import get_repo_root diff --git a/monitoring/benchmarker/engine/engine.py b/monitoring/benchmarker/engine/engine.py index ca60f036af..24e5ae9d0f 100644 --- a/monitoring/benchmarker/engine/engine.py +++ b/monitoring/benchmarker/engine/engine.py @@ -3,7 +3,7 @@ from loguru import logger -from monitoring.benchmarker.configurations.actions import BenchmarkActionName +from monitoring.benchmarker.configurations.actions.action import BenchmarkActionName from monitoring.benchmarker.configurations.configuration import BenchmarkConfiguration from monitoring.benchmarker.engine.actions.actions import run_scenario_actions from monitoring.benchmarker.engine.coordination import Coordinator @@ -48,6 +48,22 @@ async def _run_benchmark_async( coordinator = Coordinator(coordination_groups) try: + # Run benchmark setup actions + config_setup_actions = ( + config.setup_actions if "setup_actions" in config else None + ) + run_scenario_actions( + config_setup_actions, + action_specs, + action_invocations, + config, + scenarios_reports, + output_dir, + codebase_version, + commit_hash, + resource_pool, + ) + for scenario_spec in config.scenarios: logger.info( f"========== Starting Scenario '{scenario_spec.name}' ==========" @@ -64,6 +80,7 @@ async def _run_benchmark_async( output_dir, codebase_version, commit_hash, + resource_pool, ) # Run load @@ -107,13 +124,31 @@ async def _run_benchmark_async( output_dir, codebase_version, commit_hash, + resource_pool, ) logger.info( f"========== Completed Scenario '{scenario_spec.name}' ==========" ) finally: - executor.shutdown(wait=True) + try: + # Run benchmark teardown actions + config_teardown_actions = ( + config.teardown_actions if "teardown_actions" in config else None + ) + run_scenario_actions( + config_teardown_actions, + action_specs, + action_invocations, + config, + scenarios_reports, + output_dir, + codebase_version, + commit_hash, + resource_pool, + ) + finally: + executor.shutdown(wait=True) run_report = BenchmarkRunReport( codebase_version=codebase_version, diff --git a/monitoring/benchmarker/engine/users/flight_planner/astm_net_rid.py b/monitoring/benchmarker/engine/users/flight_planner/astm_net_rid.py index 26d5f2fabe..dab470bbfe 100644 --- a/monitoring/benchmarker/engine/users/flight_planner/astm_net_rid.py +++ b/monitoring/benchmarker/engine/users/flight_planner/astm_net_rid.py @@ -20,7 +20,7 @@ from monitoring.monitorlib.fetch.rid import ISA from monitoring.monitorlib.geo import get_latlngrect_vertices from monitoring.monitorlib.mutate.rid import ISAChange, delete_isa, put_isa -from monitoring.monitorlib.testing import make_fake_url +from monitoring.monitorlib.testing import TESTDUMMY_URL_PREFIX, make_fake_url from monitoring.uss_qualifier.resources.astm.f3411.dss import ( DSSInstance, DSSInstanceResource, @@ -140,6 +140,7 @@ async def create_isa(self, flight: Flight, isa_id: str) -> list[FlightAction]: utm_client=dss_instance.client, isa_version=None, participant_id=dss_instance.participant_id, + do_not_notify=TESTDUMMY_URL_PREFIX, ) isa_success = isa_change.dss_query.success @@ -186,6 +187,7 @@ async def delete_isa(self, flight: Flight, isa: ISA) -> list[FlightAction]: rid_version=dss_instance.rid_version, utm_client=dss_instance.client, participant_id=dss_instance.participant_id, + do_not_notify=TESTDUMMY_URL_PREFIX, ) del_success = del_change.dss_query.success diff --git a/monitoring/monitorlib/testing.py b/monitoring/monitorlib/testing.py index 30a082ddf5..7642f3f4c2 100644 --- a/monitoring/monitorlib/testing.py +++ b/monitoring/monitorlib/testing.py @@ -3,6 +3,8 @@ from monitoring.monitorlib.formatting import make_datetime +TESTDUMMY_URL_PREFIX = "https://testdummy.interuss.org/interuss/" + def assert_datetimes_are_equal(t1, t2, tolerance_seconds: float = 0) -> None: try: @@ -34,4 +36,4 @@ def make_fake_url(suffix: str | None = None, frames_above: int = 1) -> str: layers = layers[layers.index("monitoring") :] if suffix is not None: layers.append(suffix) - return "https://testdummy.interuss.org/interuss/" + "/".join(layers) + return TESTDUMMY_URL_PREFIX + "/".join(layers) diff --git a/schemas/monitoring/benchmarker/configurations/actions/BenchmarkActionSpecification.json b/schemas/monitoring/benchmarker/configurations/actions/action/BenchmarkActionSpecification.json similarity index 67% rename from schemas/monitoring/benchmarker/configurations/actions/BenchmarkActionSpecification.json rename to schemas/monitoring/benchmarker/configurations/actions/action/BenchmarkActionSpecification.json index 112ddcb6e3..98c7650835 100644 --- a/schemas/monitoring/benchmarker/configurations/actions/BenchmarkActionSpecification.json +++ b/schemas/monitoring/benchmarker/configurations/actions/action/BenchmarkActionSpecification.json @@ -1,12 +1,22 @@ { - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/actions/BenchmarkActionSpecification.json", + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/actions/action/BenchmarkActionSpecification.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "monitoring.benchmarker.configurations.actions.BenchmarkActionSpecification, as defined in monitoring/benchmarker/configurations/actions.py", + "description": "monitoring.benchmarker.configurations.actions.action.BenchmarkActionSpecification, as defined in monitoring/benchmarker/configurations/actions/action.py", "properties": { "$ref": { "description": "Path to content that replaces the $ref", "type": "string" }, + "f3411": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "../f3411/F3411ActionSpecification.json" + } + ] + }, "generate_artifacts": { "oneOf": [ { diff --git a/schemas/monitoring/benchmarker/configurations/actions/GenerateArtifactsActionSpecification.json b/schemas/monitoring/benchmarker/configurations/actions/action/GenerateArtifactsActionSpecification.json similarity index 81% rename from schemas/monitoring/benchmarker/configurations/actions/GenerateArtifactsActionSpecification.json rename to schemas/monitoring/benchmarker/configurations/actions/action/GenerateArtifactsActionSpecification.json index 8dc26bcbff..5a5ebaefb9 100644 --- a/schemas/monitoring/benchmarker/configurations/actions/GenerateArtifactsActionSpecification.json +++ b/schemas/monitoring/benchmarker/configurations/actions/action/GenerateArtifactsActionSpecification.json @@ -1,7 +1,7 @@ { - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/actions/GenerateArtifactsActionSpecification.json", + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/actions/action/GenerateArtifactsActionSpecification.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Generate an intermediate artifact (prior to final artifact generation) during benchmarker execution.\n\nmonitoring.benchmarker.configurations.actions.GenerateArtifactsActionSpecification, as defined in monitoring/benchmarker/configurations/actions.py", + "description": "Generate an intermediate artifact (prior to final artifact generation) during benchmarker execution.\n\nmonitoring.benchmarker.configurations.actions.action.GenerateArtifactsActionSpecification, as defined in monitoring/benchmarker/configurations/actions/action.py", "properties": { "$ref": { "description": "Path to content that replaces the $ref", @@ -10,7 +10,7 @@ "custom_artifacts": { "description": "Generate these custom artifacts when this action is run.", "items": { - "$ref": "../artifacts/artifact/ArtifactSpecification.json" + "$ref": "../../artifacts/artifact/ArtifactSpecification.json" }, "type": [ "array", diff --git a/schemas/monitoring/benchmarker/configurations/actions/RunCommandActionSpecification.json b/schemas/monitoring/benchmarker/configurations/actions/action/RunCommandActionSpecification.json similarity index 80% rename from schemas/monitoring/benchmarker/configurations/actions/RunCommandActionSpecification.json rename to schemas/monitoring/benchmarker/configurations/actions/action/RunCommandActionSpecification.json index 23c3d73161..975f6d15a2 100644 --- a/schemas/monitoring/benchmarker/configurations/actions/RunCommandActionSpecification.json +++ b/schemas/monitoring/benchmarker/configurations/actions/action/RunCommandActionSpecification.json @@ -1,7 +1,7 @@ { - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/actions/RunCommandActionSpecification.json", + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/actions/action/RunCommandActionSpecification.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Shell command to run as a benchmark action.\n\nmonitoring.benchmarker.configurations.actions.RunCommandActionSpecification, as defined in monitoring/benchmarker/configurations/actions.py", + "description": "Shell command to run as a benchmark action.\n\nmonitoring.benchmarker.configurations.actions.action.RunCommandActionSpecification, as defined in monitoring/benchmarker/configurations/actions/action.py", "properties": { "$ref": { "description": "Path to content that replaces the $ref", diff --git a/schemas/monitoring/benchmarker/configurations/actions/f3411/CreateSubscription.json b/schemas/monitoring/benchmarker/configurations/actions/f3411/CreateSubscription.json new file mode 100644 index 0000000000..fbbcd49f3a --- /dev/null +++ b/schemas/monitoring/benchmarker/configurations/actions/f3411/CreateSubscription.json @@ -0,0 +1,27 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/actions/f3411/CreateSubscription.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Create a subscription.\n\nmonitoring.benchmarker.configurations.actions.f3411.CreateSubscription, as defined in monitoring/benchmarker/configurations/actions/f3411.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "mode": { + "description": "Desired creation behavior.", + "enum": [ + "GetDeleteCreate" + ], + "type": "string" + }, + "subscription": { + "$ref": "Subscription.json", + "description": "Characteristics of subscription to create." + } + }, + "required": [ + "mode", + "subscription" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/benchmarker/configurations/actions/f3411/DeleteSubscription.json b/schemas/monitoring/benchmarker/configurations/actions/f3411/DeleteSubscription.json new file mode 100644 index 0000000000..a23d6d6e96 --- /dev/null +++ b/schemas/monitoring/benchmarker/configurations/actions/f3411/DeleteSubscription.json @@ -0,0 +1,27 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/actions/f3411/DeleteSubscription.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.benchmarker.configurations.actions.f3411.DeleteSubscription, as defined in monitoring/benchmarker/configurations/actions/f3411.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "mode": { + "description": "Desired deletion behavior.", + "enum": [ + "GetDeleteIfExist" + ], + "type": "string" + }, + "subscription_id": { + "description": "ID of the subscription to delete.", + "type": "string" + } + }, + "required": [ + "mode", + "subscription_id" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/benchmarker/configurations/actions/f3411/F3411ActionSpecification.json b/schemas/monitoring/benchmarker/configurations/actions/f3411/F3411ActionSpecification.json new file mode 100644 index 0000000000..d35e9e22a7 --- /dev/null +++ b/schemas/monitoring/benchmarker/configurations/actions/f3411/F3411ActionSpecification.json @@ -0,0 +1,32 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/actions/f3411/F3411ActionSpecification.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Actions pertaining to ASTM F3411 NetRID.\n\nmonitoring.benchmarker.configurations.actions.f3411.F3411ActionSpecification, as defined in monitoring/benchmarker/configurations/actions/f3411.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "create_subscription": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "CreateSubscription.json" + } + ] + }, + "delete_subscription": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "DeleteSubscription.json" + } + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/benchmarker/configurations/actions/f3411/Subscription.json b/schemas/monitoring/benchmarker/configurations/actions/f3411/Subscription.json new file mode 100644 index 0000000000..cf96c74dbc --- /dev/null +++ b/schemas/monitoring/benchmarker/configurations/actions/f3411/Subscription.json @@ -0,0 +1,48 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/actions/f3411/Subscription.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.benchmarker.configurations.actions.f3411.Subscription, as defined in monitoring/benchmarker/configurations/actions/f3411.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "area": { + "$ref": "../../../../monitorlib/geo/LatLngBoundingBox.json", + "description": "Horizontal area this subscription should cover." + }, + "duration": { + "description": "Duration of the subscription, from the time it is created.", + "format": "duration", + "type": "string" + }, + "max_alt": { + "$ref": "../../../../monitorlib/geo/Altitude.json", + "description": "Altitude above which this subscription should not apply." + }, + "min_alt": { + "$ref": "../../../../monitorlib/geo/Altitude.json", + "description": "Altitude below which this subscription should not apply." + }, + "rid_version": { + "enum": [ + "F3411-19", + "F3411-22a" + ], + "type": "string" + }, + "subscription_id": { + "description": "ID of the single subscription to create.", + "type": "string" + } + }, + "required": [ + "area", + "duration", + "max_alt", + "min_alt", + "rid_version", + "subscription_id" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/benchmarker/configurations/configuration/BenchmarkConfiguration.json b/schemas/monitoring/benchmarker/configurations/configuration/BenchmarkConfiguration.json index c8246b6260..42109cdbe8 100644 --- a/schemas/monitoring/benchmarker/configurations/configuration/BenchmarkConfiguration.json +++ b/schemas/monitoring/benchmarker/configurations/configuration/BenchmarkConfiguration.json @@ -10,7 +10,7 @@ "actions": { "description": "Actions available to be performed during the benchmarker run.", "items": { - "$ref": "../actions/BenchmarkActionSpecification.json" + "$ref": "../actions/action/BenchmarkActionSpecification.json" }, "type": [ "array", @@ -52,6 +52,26 @@ }, "type": "array" }, + "setup_actions": { + "description": "Actions to perform before beginning scenarios.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "teardown_actions": { + "description": "Actions to perform after completing all scenarios.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, "user_types": { "description": "Types of users available to load the system under test during the benchmarker run.", "items": {