From bf64fea6024a4d86cd9fcbac50b94b21c1a87e62 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 4 Mar 2026 16:29:09 +0000 Subject: [PATCH 1/6] feat: record the testplan reference in the sim result summary This information will be needed for the Markdown CLI report, and might also be worth surfacing in the HTML report as well. Signed-off-by: Alex Jones --- src/dvsim/sim/data.py | 2 ++ src/dvsim/sim/flow.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/dvsim/sim/data.py b/src/dvsim/sim/data.py index 0272da8d..6288f790 100644 --- a/src/dvsim/sim/data.py +++ b/src/dvsim/sim/data.py @@ -212,6 +212,8 @@ class SimFlowResults(BaseModel): build_seed: int | None """Build seed.""" + testplan_ref: str | None + """A reference (HTML link or relative HJSON path) to the testplan for this flow.""" stages: Mapping[str, TestStage] """Results per test stage.""" diff --git a/src/dvsim/sim/flow.py b/src/dvsim/sim/flow.py index 6c3579bf..3b499fac 100644 --- a/src/dvsim/sim/flow.py +++ b/src/dvsim/sim/flow.py @@ -150,6 +150,10 @@ def __init__(self, flow_cfg_file, hjson_data, args, mk_config) -> None: self.regressions = [] self.supported_wave_formats = None + # Options from cfg files used for reports / documentation + self.testplan_doc_path = "" + self.book = "" + # Options from tools - for building and running tests self.build_cmd = "" self.flist_gen_cmd = "" @@ -711,6 +715,21 @@ def _gen_json_results( build_seed = self.build_seed if not self.run_only else None + # Build up a reference to the testplan, which might be overridden. + if self.testplan_doc_path: + rel_path = Path(self.testplan_doc_path).relative_to(Path(self.proj_root)) + else: + # TODO: testplan variants frequently override `rel_path` for reporting + # and build reasons, but do not update the `testplan_doc_path`, meaning + # that they point to a variant testplan path that is not available + # in the book, unlike the original (non-variant). + rel_path = Path(self.rel_path).parent / "data" / f"{self.name}_testplan.hjson" + + if self.book: + testplan_ref = "https://{}/{}".format(self.book, str(rel_path.with_suffix(".html"))) + else: + testplan_ref = str(rel_path) + # --- Build stages only from testpoints that have at least one executed test --- stage_to_tps: defaultdict[str, dict[str, Testpoint]] = defaultdict(dict) @@ -811,6 +830,7 @@ def make_test_result(tr) -> TestResult | None: tool=tool, timestamp=timestamp, build_seed=build_seed, + testplan_ref=testplan_ref, stages=stages, coverage=coverage_model, failed_jobs=failures, From 028d1d16926233d97bed8999010f1b50b611e904 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 4 Mar 2026 16:30:06 +0000 Subject: [PATCH 2/6] feat: add the coverage report dashboard page to sim summary This information needs to be surfaced in the Markdown CLI report, and might also be worth presenting in the HTML report later. Signed-off-by: Alex Jones --- src/dvsim/sim/data.py | 2 ++ src/dvsim/sim/flow.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/src/dvsim/sim/data.py b/src/dvsim/sim/data.py index 6288f790..b0445140 100644 --- a/src/dvsim/sim/data.py +++ b/src/dvsim/sim/data.py @@ -219,6 +219,8 @@ class SimFlowResults(BaseModel): """Results per test stage.""" coverage: CoverageMetrics | None """Coverage metrics.""" + cov_report_page: Path | None + """Optional path linking to the generated coverage report dashboard page.""" failed_jobs: BucketedFailures """Bucketed failed job overview.""" diff --git a/src/dvsim/sim/flow.py b/src/dvsim/sim/flow.py index 3b499fac..d45957c2 100644 --- a/src/dvsim/sim/flow.py +++ b/src/dvsim/sim/flow.py @@ -153,6 +153,8 @@ def __init__(self, flow_cfg_file, hjson_data, args, mk_config) -> None: # Options from cfg files used for reports / documentation self.testplan_doc_path = "" self.book = "" + self.cov_report_dir = "" + self.cov_report_page = "" # Options from tools - for building and running tests self.build_cmd = "" @@ -820,6 +822,12 @@ def make_test_result(tr) -> TestResult | None: raw_metrics=coverage, ) + # Link to the coverage report page, if one exists + cov_report_page = None + if self.cov_report_page: + cov_report_dir = self.cov_report_dir or "cov_report" + cov_report_page = Path(cov_report_dir, self.cov_report_page) + failures = BucketedFailures.from_job_status(results=run_results) if failures.buckets: self.errors_seen = True @@ -833,6 +841,7 @@ def make_test_result(tr) -> TestResult | None: testplan_ref=testplan_ref, stages=stages, coverage=coverage_model, + cov_report_page=cov_report_page, failed_jobs=failures, passed=total_passed, total=total_runs, From 2a0401854f638f543816f487f9f5eb16a68e93cd Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 4 Mar 2026 18:30:13 +0000 Subject: [PATCH 3/6] refactor: store qualified name and log path in bucket items This information will be needed for reporting in the Markdown CLI logs. Signed-off-by: Alex Jones --- src/dvsim/sim_results.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/dvsim/sim_results.py b/src/dvsim/sim_results.py index 9c753482..7570d3f5 100644 --- a/src/dvsim/sim_results.py +++ b/src/dvsim/sim_results.py @@ -6,6 +6,7 @@ import re from collections.abc import Mapping, Sequence +from pathlib import Path from typing import TYPE_CHECKING from pydantic import BaseModel, ConfigDict @@ -97,12 +98,18 @@ class JobFailureOverview(BaseModel): name: str """Name of the job.""" + qual_name: str + """Qualified name to disambiguate name with other instances of the same name.""" + seed: int | None """Test seed.""" line: int | None """Line number within the log if there is one.""" + log_path: Path | None + """Path to the log for this failed job.""" + log_context: Sequence[str] """Context within the log.""" @@ -134,8 +141,10 @@ def from_job_status(results: Sequence["CompletedJobStatus"]) -> "BucketedFailure buckets[bucket].append( JobFailureOverview( name=job_status.name, + qual_name=job_status.qual_name, seed=job_status.seed, line=job_status.fail_msg.line_number, + log_path=job_status.log_path, log_context=job_status.fail_msg.context, ), ) From 73ce0bc9f0d0b6bcc83973bb1365266a1dd66475 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 4 Mar 2026 17:35:03 +0000 Subject: [PATCH 4/6] feat: add stubbed Markdown report renderer with CLI summary Reproduce the behaviour of the original DVSim within the new reporting abstractions by introducing a `MarkdownReportRenderer` (stubbed for now, to be filled out in future commits) and relevant display logic - with per-block results shown as INFO logs, and summary results shown as a print (if the summary is a primary config file). Signed-off-by: Alex Jones --- src/dvsim/logging.py | 6 ++++ src/dvsim/sim/report.py | 77 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/src/dvsim/logging.py b/src/dvsim/logging.py index 2891beac..620fe0d3 100644 --- a/src/dvsim/logging.py +++ b/src/dvsim/logging.py @@ -79,6 +79,12 @@ def set_logfile( fh.setFormatter(logging.Formatter(_PLAIN_FORMAT, datefmt=_DATE_FORMAT)) self.addHandler(fh) + def log_raw(self, contents: str) -> None: + """Log raw string contents without any added log formatting.""" + for handler in self.handlers: + handler.stream.write(contents) + handler.flush() + def _build_logger() -> DVSimLogger: """Build a DVSim logger.""" diff --git a/src/dvsim/sim/report.py b/src/dvsim/sim/report.py index 41ad7161..4a218ae7 100644 --- a/src/dvsim/sim/report.py +++ b/src/dvsim/sim/report.py @@ -4,17 +4,20 @@ """Generate reports.""" +from collections.abc import Callable from pathlib import Path from typing import Protocol, TypeAlias from dvsim.logging import log -from dvsim.sim.data import SimResultsSummary +from dvsim.sim.data import SimFlowResults, SimResultsSummary from dvsim.templates.render import render_static, render_template __all__ = ( "HtmlReportRenderer", "JsonReportRenderer", + "MarkdownReportRenderer", "ReportRenderer", + "display_report", "gen_reports", "write_report", ) @@ -123,6 +126,39 @@ def render_static_content(self, outdir: Path | None = None) -> ReportArtifacts: return artifacts +class MarkdownReportRenderer: + """Renders a Markdown report of the sim results.""" + + format_name = "markdown" + + def render(self, summary: SimResultsSummary, outdir: Path | None = None) -> ReportArtifacts: + """Render a Markdown report of the sim flow results.""" + if outdir is not None: + outdir.mkdir(parents=True, exist_ok=True) + + report_md = [ + self.render_block(flow_result)["report.md"] + for flow_result in summary.flow_results.values() + ] + report_md.append(self.render_summary(summary)["report.md"]) + + report = "\n".join(report_md) + if outdir is not None: + (outdir / "report.md").write_text(report) + + return {"report.md": report} + + def render_block(self, results: SimFlowResults) -> ReportArtifacts: + """Render a Markdown report of the sim flow results for a given block/flow.""" + _results = results + return {"report.md": "TODO: Markdown block report"} + + def render_summary(self, summary: SimResultsSummary) -> ReportArtifacts: + """Render a Markdown report of a summary of the sim flow results (overall).""" + _summary = summary + return {"report.md": "TODO: Markdown summary report"} + + def write_report(files: ReportArtifacts, root: Path) -> None: """Write rendered report artifacts to the file system, relative to a given path. @@ -137,8 +173,29 @@ def write_report(files: ReportArtifacts, root: Path) -> None: path.write_text(content) +def display_report( + files: ReportArtifacts, sink: Callable[[str], None] = print, *, with_headers: bool = False +) -> None: + """Emit the report artifacts to some textual sink. + + Prints to stdout by default, but can also write to a logger by overriding the sink. + + Args: + files: the output report artifacts from rendering simulation results. + sink: a callable that accepts a string. Default is `print` to stdout. + with_headers: a boolean controlling whether to emit artifact path names as headers. + + """ + for path, content in files.items(): + header = f"\n--- {path} ---\n" if with_headers else "" + sink(header + content + "\n") + + def gen_reports(summary: SimResultsSummary, path: Path) -> None: - """Generate a full set of reports for the given regression run. + """Generate and display a full set of reports for the given regression run. + + This helper currently saves JSON and HTML reports to disk (relative to the given path), + and outputs a Markdown report to the CLI. Args: summary: overview of the block results @@ -147,3 +204,19 @@ def gen_reports(summary: SimResultsSummary, path: Path) -> None: """ for renderer in (JsonReportRenderer(), HtmlReportRenderer()): renderer.render(summary, outdir=path) + + renderer = MarkdownReportRenderer() + + # Per-block CLI results are displayed to the `INFO` log + if log.isEnabledFor(log.INFO): + for flow_result in summary.flow_results.values(): + block_name = flow_result.block.variant_name() + log.info("[results]: [%s]", block_name) + cli_block = renderer.render_block(flow_result) + display_report(cli_block, sink=log.log_raw) + log.log_raw("\n") + + # Summary CLI results are displayed to stdout, so long as this is a primary cfg + if summary.top is not None: + cli_summary = renderer.render_summary(summary) + display_report(cli_summary) From 4376c72a4fe21ed5d542e2fe21ad3c1e9c24b51f Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 4 Mar 2026 18:35:28 +0000 Subject: [PATCH 5/6] feat: add Markdown block report generation for CLI Generally intended to resemble the original reports that were removed from DVSim as closely as possible. Note the TODOs - currently the progress table is missing because this information is not being computed nor captured correctly here. Also the stage/overall totals in the result table are not correct, but this is because the SimCfg calculations are not correct, instead of the Markdown presentation logic being incorrect. Signed-off-by: Alex Jones --- src/dvsim/sim/data.py | 12 +++ src/dvsim/sim/report.py | 216 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 224 insertions(+), 4 deletions(-) diff --git a/src/dvsim/sim/data.py b/src/dvsim/sim/data.py index b0445140..508f91c4 100644 --- a/src/dvsim/sim/data.py +++ b/src/dvsim/sim/data.py @@ -156,6 +156,18 @@ def empty(self) -> bool: v is None for v in self.model_dump(exclude_unset=True, exclude={"code"}).values() ) + def flattened(self) -> dict[str, float | None]: + """Convert the coverage metrics to a flattened dictionary. + + This dictionary will contain all the stored metrics, and a computed "total" average item. + """ + average = self.average + items = {} if average is None else {"total": average} + if self.code: + items.update(self.code.model_dump(exclude_none=True)) + items.update(self.model_dump(exclude={"code"})) + return items + class FlowResults(BaseModel): """Flow results data.""" diff --git a/src/dvsim/sim/report.py b/src/dvsim/sim/report.py index 4a218ae7..a68ed3a7 100644 --- a/src/dvsim/sim/report.py +++ b/src/dvsim/sim/report.py @@ -4,13 +4,19 @@ """Generate reports.""" -from collections.abc import Callable +from collections import defaultdict +from collections.abc import Callable, Collection, Iterable +from datetime import datetime from pathlib import Path -from typing import Protocol, TypeAlias +from typing import Any, Protocol, TypeAlias + +from tabulate import tabulate from dvsim.logging import log +from dvsim.report.data import IPMeta from dvsim.sim.data import SimFlowResults, SimResultsSummary from dvsim.templates.render import render_static, render_template +from dvsim.utils import TS_FORMAT_LONG __all__ = ( "HtmlReportRenderer", @@ -22,6 +28,18 @@ "write_report", ) + +def _plural(item: str, n: int | Collection[Any], suffix: str = "s") -> str: + if not isinstance(n, int): + n = len(n) + return item if n == 1 else item + suffix + + +def _indent_by_levels(lines: Iterable[tuple[int, str]], indent_spaces: int = 4) -> str: + """Format per-line indentation of (0-indexed level, msg) log messages.""" + return "\n".join(" " * lvl * indent_spaces + msg for lvl, msg in lines) + + # Report rendering returns mappings of relative report paths to (string) contents. ReportArtifacts: TypeAlias = dict[str, str] @@ -131,6 +149,9 @@ class MarkdownReportRenderer: format_name = "markdown" + MAX_TESTS_PER_BUCKET = 5 + MAX_RESEEDS_PER_BUCKETED_TEST = 2 + def render(self, summary: SimResultsSummary, outdir: Path | None = None) -> ReportArtifacts: """Render a Markdown report of the sim flow results.""" if outdir is not None: @@ -150,8 +171,195 @@ def render(self, summary: SimResultsSummary, outdir: Path | None = None) -> Repo def render_block(self, results: SimFlowResults) -> ReportArtifacts: """Render a Markdown report of the sim flow results for a given block/flow.""" - _results = results - return {"report.md": "TODO: Markdown block report"} + # Generate block result metadata information + report_md = self.render_metadata(results.block, results.timestamp, results.build_seed) + testplan_ref = (results.testplan_ref or "").strip() + if len(results.stages) > 0 and testplan_ref: + report_md += f"\n### [Testplan]({testplan_ref})" + report_md += f"\n### Simulator: {results.tool.name.upper()}" + + # Record a summary of the simulation results, coverage, and failure buckets, if applicable. + result_summary = self.render_block_results(results) + if result_summary: + report_md += "\n\n" + result_summary + + return {"report.md": report_md} + + def render_metadata( + self, + scope: IPMeta, + timestamp: datetime, + seed: int | None, + title: str = "Simulation Results", + ) -> str: + """Generate a Markdown string summary of the result metadata. + + Args: + scope: The scope (block/top) to generate metadata from. + timestamp: The timestamp metadata info to include. + seed: The build seed, if one was used in this run. + title: The title to use as a suffix (to "NAME %s"). Defaults to "Simulation Results". + + """ + name = scope.variant_name(sep="/") + report_md = f"## {name.upper()} {title}" + report_md += f"\n### {timestamp.strftime(TS_FORMAT_LONG)}" + + revision = (scope.revision_info or "").strip() + if not revision: + revision = f"Github Revision: [`{scope.commit_short}`]({scope.url})" + report_md += f"\n### {revision}" + report_md += f"\n### Branch: {scope.branch}" + + if seed is not None: + report_md += f"\n### Build randomization enabled with --build-seed {seed}" + + return report_md + + def render_block_results(self, results: SimFlowResults) -> str: + """Generate a Markdown string covering the results, coverage and failure buckets.""" + report_md = self.render_result_table(results) if results.total else "No results to display." + + # TODO: need to optionally generate a progress table if `--map-full-testplan` was set. + # This can be passed through and set when instantiating the markdown renderer, but + # right now we don't record the correct information for testplan progress in the sim + # results, so we leave this incomplete for now. + + if results.coverage: + coverage_table = self.render_coverage_table(results) + if coverage_table: + report_md += "\n\n" + coverage_table + + if results.failed_jobs.buckets: + bucket_summary = self.render_bucket_summary(results) + if bucket_summary: + report_md += "\n\n" + bucket_summary + + return report_md + + def render_result_table(self, results: SimFlowResults) -> str: + """Generate a Markdown string containing a table of the testplan results.""" + column_info = [ + ("Stage", "center"), + ("Name", "center"), + ("Tests", "left"), + ("Max Job Runtime", "center"), + ("Simulated Time", "center"), + ("Passing", "center"), + ("Total", "center"), + ("Pass Rate", "center"), + ] + table = [] + hidden_names = ("n.a.", "unmapped") + + for stage_key, stage in results.stages.items(): + # Coalesce result information to default values if necessary + stage_name = "" if stage_key.lower() in hidden_names else stage_key + + for tp_key, tp in stage.testpoints.items(): + tp_name = "" if tp_key.lower() in hidden_names else tp_key + for test_name, result in tp.tests.items(): + job_runtime = "" if result.max_time is None else f"{result.max_time:.3f}s" + sim_time = "" if result.sim_time is None else f"{result.sim_time:.3f}us" + pass_rate = "-- %" if result.total == 0 else f"{result.percent:.2f} %" + + row = [ + stage_name, + tp_name, + test_name, + job_runtime, + sim_time, + result.passed, + result.total, + pass_rate, + ] + table.append(row) + + pass_rate = "-- %" if stage.total == 0 else f"{stage.percent:.2f} %" + # TODO: note the calculated stage totals are currently not correct. + table.append( + [stage_name, None, "**TOTAL**", None, None, stage.passed, stage.total, pass_rate] + ) + + # TODO: note the calculated overall totals are currently not correct. + pass_rate = "-- %" if results.total == 0 else f"{results.percent:.2f} %" + table.append( + [None, None, "**TOTAL**", None, None, results.passed, results.total, pass_rate] + ) + + if not table: + return "" + + return "### Test Results\n\n" + tabulate( + table, + headers=[c[0] for c in column_info], + tablefmt="pipe", + colalign=[c[1] for c in column_info], + ) + + def render_coverage_table(self, results: SimFlowResults) -> str: + """Generate a Markdown string containing a table of the coverage results.""" + if results.coverage is None: + return "" + + cov_results = { + k.upper().replace("_", "/"): f"{v:.2f} %" + for k, v in results.coverage.flattened().items() + if v is not None + } + if not cov_results and not results.cov_report_page: + return "" + + report_md = "## Coverage Results" + if results.cov_report_page: + report_md += f"\n### [Coverage Dashboard]({results.cov_report_page})" + if cov_results: + colalign = ("center",) * len(cov_results) + report_md += "\n\n" + tabulate( + [cov_results], headers="keys", tablefmt="pipe", colalign=colalign + ) + + return report_md + + def render_bucket_summary(self, results: SimFlowResults) -> str: + """Generate a Markdown string with a summary of the buckets (failures/killed).""" + lines = [(0, "## Failure Buckets")] + + for bucket, tests in sorted( + results.failed_jobs.buckets.items(), + key=lambda kv: len(kv[1]), + reverse=True, + ): + lines.append((0, f"* `{bucket}` has {len(tests)} {_plural('failure', tests)}:")) + + grouped_tests = defaultdict(list) + for job in tests: + grouped_tests[job.name].append(job) + + displayed = list(grouped_tests.items())[: self.MAX_TESTS_PER_BUCKET] + for name, reseeds in displayed: + lines.append( + (1, f"* Test {name} has {len(reseeds)} {_plural('failure', reseeds)}.") + ) + + for failure in reseeds[: self.MAX_RESEEDS_PER_BUCKETED_TEST]: + lines.append((2, f"* {failure.qual_name}\\")) + line_context = "Log" if failure.line is None else f"Line {failure.line}, in log" + lines.append((2, f" {line_context} {failure.log_path}")) + if failure.log_context: + lines.append((0, "")) + lines.extend((4, line.rstrip()) for line in failure.log_context) + lines.append((0, "")) + + extra = len(reseeds) - self.MAX_RESEEDS_PER_BUCKETED_TEST + if extra > 0: + lines.append((2, f"* ... and {extra} more {_plural('failure', extra)}.")) + + extra = len(grouped_tests) - self.MAX_TESTS_PER_BUCKET + if extra > 0: + lines.append((2, f"* ... and {extra} more {_plural('test', extra)}.")) + + return _indent_by_levels(lines) def render_summary(self, summary: SimResultsSummary) -> ReportArtifacts: """Render a Markdown report of a summary of the sim flow results (overall).""" From 8279cb06e59f20b2c2906e32ebd4237400f72160 Mon Sep 17 00:00:00 2001 From: Alex Jones Date: Wed, 4 Mar 2026 18:52:03 +0000 Subject: [PATCH 6/6] feat: add Markdown summary report generation for CLI If a link to a directory containing an output HTML report is provided, then the generated table will use Markdown links to the generated reports. This is using different logic to the original DVSim in OpenTitan, because that did not make sense, for two reasons: 1. Reports were relative to the generated Markdown file, but we no longer write a Markdown report to disk. 2. The report structure as a whole has changed. Instead, while we still provide the ability to customise the relative path to report paths, if unspecified (which it is currently), we just print CLI report links relative to the CWD. It might be the case that this doesn't have much value in current DVSim and so should just be removed altogether. Signed-off-by: Alex Jones --- src/dvsim/sim/report.py | 65 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/src/dvsim/sim/report.py b/src/dvsim/sim/report.py index a68ed3a7..8d0ebe9e 100644 --- a/src/dvsim/sim/report.py +++ b/src/dvsim/sim/report.py @@ -152,6 +152,18 @@ class MarkdownReportRenderer: MAX_TESTS_PER_BUCKET = 5 MAX_RESEEDS_PER_BUCKETED_TEST = 2 + def __init__(self, html_link_base: Path | None = None, relative_to: Path | None = None) -> None: + """Construct a Markdown report renderer. + + Args: + html_link_base: The path to the dir that HTML reports are written into, if using HTML + links. If not provided, no HTML links will be generated in the summary report. + relative_to: The path that HTML report links should be relative to. + + """ + self.html_link_base = html_link_base + self.relative_to = relative_to + def render(self, summary: SimResultsSummary, outdir: Path | None = None) -> ReportArtifacts: """Render a Markdown report of the sim flow results.""" if outdir is not None: @@ -363,8 +375,53 @@ def render_bucket_summary(self, results: SimFlowResults) -> str: def render_summary(self, summary: SimResultsSummary) -> ReportArtifacts: """Render a Markdown report of a summary of the sim flow results (overall).""" - _summary = summary - return {"report.md": "TODO: Markdown summary report"} + # Generate result metadata information + if summary.top is not None: + report_md = self.render_metadata( + summary.top, + summary.timestamp, + summary.build_seed, + title="Simulation Results (Summary)", + ) + else: + report_md = "" + + # Generate a table aggregating and mapping block-level reports + table = [] + for name, flow_result in summary.flow_results.items(): + coverage = "--" + if flow_result.coverage is not None: + average = flow_result.coverage.average + if average is not None: + coverage = f"{average:.2f} %" + file_name = flow_result.block.variant_name() + + # Optionally display links to the block HTML reports, relative to the CWD + if self.html_link_base is not None: + relative = self.relative_to if self.relative_to is not None else Path(Path.cwd()) + block_report = self.html_link_base / f"{file_name}.html" + html_report_path = block_report.relative_to(relative) + name_link = f"[{name.upper()}]({html_report_path!s})" + else: + name_link = name.upper() + + table.append( + { + "Name": name_link, + "Passing": flow_result.passed, + "Total": flow_result.total, + "Pass Rate": f"{flow_result.percent:.2f} %", + "Coverage": coverage, + } + ) + + if table: + colalign = ("center",) * len(table[0]) + report_md += "\n\n" + tabulate( + table, headers="keys", tablefmt="pipe", colalign=colalign + ) + + return {"report.md": report_md} def write_report(files: ReportArtifacts, root: Path) -> None: @@ -413,7 +470,7 @@ def gen_reports(summary: SimResultsSummary, path: Path) -> None: for renderer in (JsonReportRenderer(), HtmlReportRenderer()): renderer.render(summary, outdir=path) - renderer = MarkdownReportRenderer() + renderer = MarkdownReportRenderer(path) # Per-block CLI results are displayed to the `INFO` log if log.isEnabledFor(log.INFO): @@ -422,7 +479,7 @@ def gen_reports(summary: SimResultsSummary, path: Path) -> None: log.info("[results]: [%s]", block_name) cli_block = renderer.render_block(flow_result) display_report(cli_block, sink=log.log_raw) - log.log_raw("\n") + log.log_raw("\n" if summary.top is None else "\n\n") # Summary CLI results are displayed to stdout, so long as this is a primary cfg if summary.top is not None: