diff --git a/CMakeLists.txt b/CMakeLists.txt index 67e47b6eaebd..04454c0c0311 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -307,16 +307,19 @@ set( ) set( - CCF_NETWORK_TEST_DEFAULT_CONSTITUTION - --constitution + CCF_NETWORK_TEST_DEFAULT_CONSTITUTION_FILES ${CCF_DIR}/samples/constitutions/default/actions.js - --constitution ${CCF_DIR}/samples/constitutions/default/validate.js - --constitution ${CCF_DIR}/samples/constitutions/default/resolve.js - --constitution ${CCF_DIR}/samples/constitutions/default/apply.js ) +set(CCF_NETWORK_TEST_DEFAULT_CONSTITUTION "") +foreach(CONSTITUTION_FILE IN LISTS CCF_NETWORK_TEST_DEFAULT_CONSTITUTION_FILES) + list( + APPEND CCF_NETWORK_TEST_DEFAULT_CONSTITUTION --constitution + ${CONSTITUTION_FILE} + ) +endforeach() set( CCF_NETWORK_TEST_ARGS --log-level @@ -325,6 +328,27 @@ set( ${WORKER_THREADS} ) +# For fast e2e runs, tick node faster than default value (except for +# instrumented builds which may process ticks slower). +if(SAN) + set(NODE_TICK_MS 10) +else() + set(NODE_TICK_MS 1) +endif() + +list( + TRANSFORM CCF_NETWORK_TEST_DEFAULT_CONSTITUTION_FILES + REPLACE "(.+)" "\"\\1\"" + OUTPUT_VARIABLE QUOTED_DEFAULT_CONSTITUTION_FILES +) +list(JOIN QUOTED_DEFAULT_CONSTITUTION_FILES ", " DEFAULT_CONSTITUTION_JSON) +set(DEFAULT_CONSTITUTION_JSON "[${DEFAULT_CONSTITUTION_JSON}]") +configure_file( + ${CCF_DIR}/cmake/test_config.json.in + ${CMAKE_BINARY_DIR}/test_config.json + @ONLY +) + # SNIPPET_START: JS generic application add_ccf_app( js_generic @@ -1258,6 +1282,27 @@ if(BUILD_TESTS) ) set_tests_properties(schema_test PROPERTIES TIMEOUT 900) + if(BUILD_END_TO_END_TESTS) + add_test( + NAME e2e_unittests + COMMAND ${PYTHON} -m unittest discover -s ${CMAKE_SOURCE_DIR}/tests -v + ) + set_property( + TEST e2e_unittests + APPEND + PROPERTY ENVIRONMENT "PYTHONPATH=${CCF_DIR}/tests:$ENV{PYTHONPATH}" + ) + set_property( + TEST e2e_unittests + APPEND + PROPERTY ENVIRONMENT "CCF_TEST_CONFIG=${CMAKE_BINARY_DIR}/test_config.json" + ) + set_property(TEST e2e_unittests APPEND PROPERTY LABELS e2e) + set_property(TEST e2e_unittests APPEND PROPERTY LABELS e2e_unittest) + set_property(TEST e2e_unittests APPEND PROPERTY LABELS bucket_a) + add_san_test_properties(e2e_unittests) + endif() + add_e2e_test( NAME snp_platform_tests PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/amd_snp.py diff --git a/cmake/common.cmake b/cmake/common.cmake index a8e619faef0b..7fbfa7781366 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -112,14 +112,6 @@ function(add_e2e_test) if(BUILD_END_TO_END_TESTS) set(PYTHON_WRAPPER ${PYTHON}) - # For fast e2e runs, tick node faster than default value (except for - # instrumented builds which may process ticks slower). - if(SAN) - set(NODE_TICK_MS 10) - else() - set(NODE_TICK_MS 1) - endif() - if(NOT PARSED_ARGS_PERF_LABEL) set(PARSED_ARGS_PERF_LABEL ${PARSED_ARGS_NAME}) endif() diff --git a/cmake/test_config.json.in b/cmake/test_config.json.in new file mode 100644 index 000000000000..2ddd41e5fa4d --- /dev/null +++ b/cmake/test_config.json.in @@ -0,0 +1,9 @@ +{ + "binary_dir": "@CMAKE_BINARY_DIR@", + "log_level": "@TEST_LOGGING_LEVEL@", + "worker_threads": @WORKER_THREADS@, + "tick_ms": @NODE_TICK_MS@, + "default_constitution": @DEFAULT_CONSTITUTION_JSON@, + "historical_testdata": "@CMAKE_SOURCE_DIR@/tests/testdata", + "jinja_templates_path": "@CMAKE_SOURCE_DIR@/samples/templates" +} diff --git a/python/src/ccf/ledger.py b/python/src/ccf/ledger.py index 0b145b0d59a9..b97607a0f3aa 100644 --- a/python/src/ccf/ledger.py +++ b/python/src/ccf/ledger.py @@ -399,7 +399,8 @@ def clone(self, at_loc: int = 0): @staticmethod def from_file(filename): - return SimpleBuffer(filename, open(filename, "rb").read()) + with open(filename, "rb") as f: + return SimpleBuffer(filename, f.read()) def _byte_read_safe(file: SimpleBuffer, num_of_bytes): diff --git a/tests/ci-buckets.txt b/tests/ci-buckets.txt index 853617cdba6c..9f7102cae531 100644 --- a/tests/ci-buckets.txt +++ b/tests/ci-buckets.txt @@ -15,6 +15,7 @@ # ./scripts/test-buckets-checks.sh -f bucket_a: + e2e_unittests lts_compatibility bucket_b: diff --git a/tests/infra/e2e_args.py b/tests/infra/e2e_args.py index 9a135f836fe2..4e990089e68b 100644 --- a/tests/infra/e2e_args.py +++ b/tests/infra/e2e_args.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the Apache 2.0 License. import argparse +import json import os import infra.interfaces import infra.path @@ -46,11 +47,26 @@ def default_platform(): return "virtual" +def _load_test_config(): + config_path = os.getenv("CCF_TEST_CONFIG") + if config_path: + with open(config_path, encoding="utf-8") as f: + return json.load(f) + + config_path = os.path.join(os.getcwd(), "test_config.json") + if os.path.isfile(config_path): + with open(config_path, encoding="utf-8") as f: + return json.load(f) + + return None + + def cli_args( add=lambda x: None, parser=None, accept_unknown=False, ledger_chunk_bytes_override=None, + argv=None, ): LOG.remove() LOG.add( @@ -400,10 +416,20 @@ def cli_args( ) add(parser) + test_config = _load_test_config() + if test_config is not None: + config_defaults = { + k: v for k, v in test_config.items() if k != "default_constitution" + } + parser.set_defaults(**config_defaults) + if accept_unknown: - args, unknown_args = parser.parse_known_args() + args, unknown_args = parser.parse_known_args(argv) else: - args = parser.parse_args() + args = parser.parse_args(argv) + + if test_config is not None and not args.constitution: + args.constitution = test_config.get("default_constitution", []) args.binary_dir = os.path.abspath(args.binary_dir) diff --git a/tests/infra/network.py b/tests/infra/network.py index 75696a38e4cf..9b59fa252355 100644 --- a/tests/infra/network.py +++ b/tests/infra/network.py @@ -6,6 +6,7 @@ from contextlib import contextmanager from enum import Enum, IntEnum, auto +import unittest from infra.clients import flush_info, CCFConnectionException, CCFIOException import infra.crypto import infra.member @@ -2229,3 +2230,80 @@ def network( ) if init_partitioner: net.partitioner.cleanup() + + +class NetworkTestCase(unittest.TestCase): + label = None + package = "samples/apps/logging/logging" + test_config_overrides = lambda args: {} + network_kwargs = lambda args: {} + start_and_open_kwargs = {} + failure_stop_kwargs = { + "skip_verification": True, + "accept_ledger_diff": True, + "skip_verify_chunking": True, + } + success_stop_kwargs = { + "skip_verification": False, + "accept_ledger_diff": False, + "skip_verify_chunking": False, + } + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + if cls.label is None: + raise TypeError(f"{cls.__name__} must define a label") + if not isinstance(cls.label, str): + raise TypeError(f"{cls.__name__}.label must be a string") + + @classmethod + def resolve_args_overrides(cls, args): + return cls.test_config_overrides(args) + + @classmethod + def _cleanup(cls): + if cls._failing_test_id is None: + cls.network.stop_all_nodes(**cls.success_stop_kwargs) + else: + cls.network.txs = None + cls.network.log_stack_traces(timeout=10) + cls.network.stop_all_nodes(**cls.failure_stop_kwargs) + + @classmethod + def setUpClass(cls): + cls.args = infra.e2e_args.cli_args(argv=[]) + cls.args.label = cls.label + cls.args.package = cls.package + if not os.path.isabs(cls.args.package): + cls.args.package = os.path.join(cls.args.binary_dir, cls.args.package) + for name, value in cls.resolve_args_overrides(cls.args).items(): + setattr(cls.args, name, value) + + cls.network = infra.network.Network( + cls.args.nodes, + cls.args.binary_dir, + cls.args.debug_nodes, + **cls.network_kwargs(cls.args), + ) + with infra.network.close_on_error(cls.network, pdb=cls.args.pdb): + cls.network.start_and_open(cls.args, **cls.start_and_open_kwargs) + cls._failing_test_id = None + cls.addClassCleanup(cls._cleanup) + + # Record traces on exceptions + def _callTestMethod(self, method): + if type(self)._failing_test_id is not None: + raise unittest.SkipTest( + f"Skipping test due to previous failure: {type(self)._failing_test_id}" + ) + try: + return method() + except unittest.SkipTest: + raise + except Exception: + type(self)._failing_test_id = self.id() + if self.args.pdb: + import pdb + + pdb.post_mortem() + raise diff --git a/tests/test_e2e_operations.py b/tests/test_e2e_operations.py new file mode 100644 index 000000000000..2a5a940c56cc --- /dev/null +++ b/tests/test_e2e_operations.py @@ -0,0 +1,40 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +import infra.logging_app as app +import infra +from e2e_operations import ( + test_backup_snapshot_fetch, + test_backup_snapshot_fetch_max_size, + test_error_message_on_failure_to_fetch_snapshot, + test_join_idempotency_short_circuits_on_backup, + test_join_time_snapshot_fetch_failure, +) + + +class BackupSnapshotDownload(infra.network.NetworkTestCase): + label = "backup_snapshot_download" + test_config_overrides = lambda args: { + "snapshot_tx_interval": 30, + "nodes": infra.e2e_args.max_nodes(args, f=0), + } + start_and_open_kwargs = {"backup_snapshot_fetch_enabled": True} + network_kwargs = lambda _: {"txs": app.LoggingTxs("user0")} + success_stop_kwargs = { + "skip_verification": True, + } + + def test_backup_snapshot_fetch(self): + test_backup_snapshot_fetch(self.network, self.args) + + def test_backup_snapshot_fetch_max_size(self): + test_backup_snapshot_fetch_max_size(self.network, self.args) + + def test_join_idempotency_short_circuits_on_backup(self): + test_join_idempotency_short_circuits_on_backup(self.network, self.args) + + def test_join_time_snapshot_fetch_failure(self): + test_join_time_snapshot_fetch_failure(self.network, self.args) + + def test_error_message_on_failure_to_fetch_snapshot(self): + test_error_message_on_failure_to_fetch_snapshot(self.network, self.args)