diff --git a/Project.toml b/Project.toml index 0e14ada..60258eb 100644 --- a/Project.toml +++ b/Project.toml @@ -10,6 +10,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DefaultApplication = "3f0dd361-4fe0-5fc6-8523-80b14ec94d85" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" EzXML = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -22,6 +23,7 @@ Dates = "1.10" DefaultApplication = "1" DocStringExtensions = "0.8, 0.9" EzXML = "1" +JSON = "0.21, 1" LibGit2 = "1.10" OrderedCollections = "1" Pkg = "1.10" diff --git a/README.md b/README.md index cc0f8b5..4413ccf 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,15 @@ This is a collection of trivial functions to facilitate generating and exploring ```julia Pkg.add("LocalCoverage") ``` + or `]add LocalCoverage` from the Julia REPL. ### Optional Dependencies + The package has several optional features which require additional dependencies. [`lcov`](https://github.com/linux-test-project/lcov) is required for generating HTML output. You can install it via + - Debian/Ubuntu: `sudo apt install lcov` - Arch/Manjaro: `yay -S lcov` @@ -25,6 +28,7 @@ Note that the code in this package assumes a reasonably recent `lcov` version wh `LocalCoverage` also provides an option to generate a [Cobertura](https://cobertura.github.io/cobertura/) XML, which is used by JVM-related test suites such as Jenkins. This can be done either with: + - The native Julia implementation via the `write_lcov_to_xml` function, - The original implementation using the `generate_xml` function. @@ -39,20 +43,129 @@ with packages added with the `Pkg.dev` installation option (which allows for eas manipulation of the package directory). To generate test coverage data do + ```julia using LocalCoverage # pkg is the package name as a string, e.g. "LocalCoverage" generate_coverage(pkg = nothing; run_test = true) # defaults shown ``` + +You'll see output during testing like: + +```text +┌─────────────────────┬───────┬─────┬──────┬──────┬──────────┐ +│ Filename │ Lines │ Hit │ Miss │ % │ Gaps │ +├─────────────────────┼───────┼─────┼──────┼──────┼──────────┤ +│ src/DummyPackage.jl │ 1 │ 0 │ 1 │ 0% │ 6 │ +│ src/bar.jl │ 2 │ 1 │ 1 │ 50% │ 3 │ +│ src/corge/corge.jl │ 1 │ 1 │ 0 │ 100% │ │ +│ src/corge/grault.jl │ 1 │ 0 │ 1 │ 0% │ 2 │ +│ src/qux.jl │ 2 │ 0 │ 2 │ 0% │ 2, 5 │ +├─────────────────────┼───────┼─────┼──────┼──────┼──────────┤ +│ TOTAL │ 7 │ 2 │ 5 │ 29% │ │ +└─────────────────────┴───────┴─────┴──────┴──────┴──────────┘ +``` + You can then navigate to the `coverage` subdirectory of the package directory (e.g. `~/.julia/dev/PackageName/coverage`) and see the generated coverage summaries. Note that the test execution step may be skipped if `*.cov` files were already generated (possibly by some external package). To generate, and optionally open, the coverage report HTML do + ```julia html_coverage(coverage::PackageCoverage; open = false, dir = tempdir()) # defaults shown ``` +### Coverage summary JSON output + +To generate a Jest-compatible `coverage-summary.json` report, pass a `JSONSummaryOptions` instance to `generate_coverage()` via the `json_summary` keyword argument: + +```julia +generate_coverage(pkg = nothing; json_summary = JSONSummaryOptions()) # defaults shown +``` + +If `test_args` are provided, the top level `"total"` key is replaced with the name of the test set. + +To generate a Jest-compatible `coverage-summary.json` report from existing `PackageCoverage` data do + +```julia +generate_json_summary(coverage::PackageCoverage, filename = "coverage-summary.json"; test_args = [""]) # defaults shown +``` + +#### `JSONSummaryOptions` Fields + +`JSONSummaryOptions` has the following fields (with their default values when calling `JSONSummaryOptions()`): + +* `write::Bool = true`: Whether to write the coverage JSON summary to disk. +* `filename::String = "coverage-summary.json"`: Filename of the generated JSON summary. +* `comparison_filename::Union{Nothing,String} = nothing`: Path/string of previous coverage JSON summary to compare against. +* `comparison_fail_on_decrease::Bool = false`: Determines whether to fail (throw an error) if the coverage has decreased. + +#### JSON Summary Schema + +The JSON summary mimics [Jest's standard `json-summary` reporter format](https://stackoverflow.com/a/60968723) ([relevant section of Jest docs](https://jestjs.io/docs/configuration#coveragereporters-arraystring--string-options)). It contains a top-level key for the package total (or the name of the test set if customized), followed by keys for each individual tracked file path. + +Each section contains sub-metrics for `lines`, `statements`, `functions`, and `branches` with the following structure: +* `total`: Total number of lines tracked. +* `covered`: Number of lines hit. +* `skipped`: Always `0` (included for compatibility). +* `pct`: Coverage percentage (as a float). + +```json +{ + "total": { + "lines": { "total": 10, "covered": 5, "skipped": 0, "pct": 50.0 }, + "statements": { "total": 10, "covered": 5, "skipped": 0, "pct": 50.0 }, + "functions": { "total": 10, "covered": 5, "skipped": 0, "pct": 50.0 }, + "branches": { "total": 10, "covered": 5, "skipped": 0, "pct": 50.0 } + }, + "src/bar.jl": { + "lines": { "total": 5, "covered": 2, "skipped": 0, "pct": 40.0 }, + "statements": { "total": 5, "covered": 2, "skipped": 0, "pct": 40.0 }, + "functions": { "total": 5, "covered": 2, "skipped": 0, "pct": 40.0 }, + "branches": { "total": 5, "covered": 2, "skipped": 0, "pct": 40.0 } + } +} +``` + + +### Coverage Delta and Comparison (CI) + +You can compare two coverage JSON summaries (either from file paths or directly from JSON strings) to print a delta table and check if coverage has decreased: + +```julia +# Compare two summaries. Returns true if coverage did not decrease, false if it did. +compare_coverage_json_summaries("old-summary.json", "new-summary.json") +``` + +You can also integrate this check directly into `generate_coverage()`, which is very useful for continuous integration (CI) environments to fail a build if coverage drops. If the `comparison_filename` file is missing, it's an automatic pass: + +```julia +# Runs tests, generates the current summary, prints a comparison table against +# "previous-summary.json", and throws an error if overall coverage decreased. +generate_coverage(pkg; + json_summary = JSONSummaryOptions( + comparison_filename = "previous-summary.json", + comparison_fail_on_decrease = true + )) +``` + +The comparison output looks like: + +```text +┌─────────────────────┬──────────────┬──────────────┬────────┐ +│ File/Section │ Old Coverage │ New Coverage │ Delta │ +├─────────────────────┼──────────────┼──────────────┼────────┤ +│ total │ 20.0% │ 28.57% │ +8.57% │ +│ src/DummyPackage.jl │ - │ 0.0% │ - │ +│ src/bar.jl │ 20.0% │ 50.0% │ +30.0% │ +│ src/corge/corge.jl │ - │ 100.0% │ - │ +│ src/corge/grault.jl │ - │ 0.0% │ - │ +│ src/qux.jl │ - │ 0.0% │ - │ +└─────────────────────┴──────────────┴──────────────┴────────┘ +``` + A utility method is also provided to easily print coverage statistics and exit with a status reflecting if some given target coverage was met. It can be used from a shell by doing + ```bash julia --project -e'using LocalCoverage; report_coverage_and_exit(target_coverage=90)' ``` diff --git a/src/LocalCoverage.jl b/src/LocalCoverage.jl index 3839bef..6f1e6ca 100644 --- a/src/LocalCoverage.jl +++ b/src/LocalCoverage.jl @@ -10,9 +10,11 @@ import Pkg import Dates using EzXML using OrderedCollections +import JSON export generate_coverage, process_coverage, clean_coverage, report_coverage_and_exit, - html_coverage, generate_xml, write_lcov_to_xml + html_coverage, generate_xml, write_lcov_to_xml, generate_json_summary, + compare_coverage_json_summaries, JSONSummaryOptions #### #### helper functions and constants @@ -74,6 +76,22 @@ Base.@kwdef struct PackageCoverage lines_tracked::Int end +""" +JSON coverage summary configuration options. + +$(FIELDS) +""" +Base.@kwdef struct JSONSummaryOptions + "Whether to write the coverage JSON summary to disk" + write::Bool = true + "Filename of the generated JSON summary" + filename::String = "coverage-summary.json" + "Path/string of previous coverage JSON summary to compare against" + comparison_filename::Union{Nothing,String} = nothing + "Whether to fail (throw an error) if the coverage has decreased" + comparison_fail_on_decrease::Bool = false +end + function Base.getproperty(summary::Union{PackageCoverage,FileCoverageSummary}, sym::Symbol) if sym ≡ :coverage_percentage 100 * summary.lines_hit / summary.lines_tracked @@ -238,6 +256,8 @@ step is skipped allowing an easier use in combination with other test packages. a test is run on files/folders that *are not* in the list, then those files will be shown as having 0% coverage. +- `json_summary::Union{Nothing,JSONSummaryOptions} = nothing` determines whether and how a JSON summary of coverage is generated and compared. See [`JSONSummaryOptions`](@ref). + Coverage of subsets of tests/files can be generated by specifying the list of testsets to run along with corresponding lists of files/folders. @@ -263,7 +283,8 @@ function generate_coverage(pkg = nothing; run_test = true, test_args = [""], folder_list = ["src"], - file_list = [])::PackageCoverage + file_list = [], + json_summary::Union{Nothing,JSONSummaryOptions} = nothing)::PackageCoverage try if run_test @@ -278,7 +299,19 @@ function generate_coverage(pkg = nothing; println(stdout, coverage) rethrow(e) end - return process_coverage(pkg; folder_list, file_list) + coverage = process_coverage(pkg; folder_list, file_list) + if !isnothing(json_summary) + if json_summary.write || !isnothing(json_summary.comparison_filename) + out_path = generate_json_summary(coverage, json_summary.filename; test_args = test_args) + if !isnothing(json_summary.comparison_filename) + passed = compare_coverage_json_summaries(json_summary.comparison_filename, out_path, stdout) + if !passed && json_summary.comparison_fail_on_decrease + error("Coverage comparison failed: overall coverage decreased from previous run.") + end + end + end + end + return coverage end """ @@ -449,6 +482,93 @@ function generate_xml(coverage::PackageCoverage, filename="cov.xml") @info("generated cobertura XML $(joinpath(coverage.package_dir, COVDIR, filename)).") end +""" +$(SIGNATURES) + +Generate a coverage JSON summary in the package `coverage` directory, mimicking the structure +of the [Jest](https://jestjs.io/) `json-summary` reporter. + +If `test_args` are provided, the top level `"total"` key is replaced with the name of the test set. + +Args: + - `coverage` (PackageCoverage): Coverage metrics evaluated for the package. + - `filename` (str, optional): Filename of the generated JSON summary. Defaults to "coverage-summary.json". + - `test_args` (list of str, optional): Arguments passed to `Pkg.test`. Defaults to `[""]`. + +Returns: + The absolute path to the generated JSON summary file. + +JSON Schema: + The output JSON file maps each source file (and a package `"total"` or custom test set name) + to a set of coverage metrics (`lines`, `statements`, `functions`, and `branches`): + +```json + { + "total": { + "lines": {"total": int, "covered": int, "skipped": int, "pct": float}, + "statements": {"total": int, "covered": int, "skipped": int, "pct": float}, + "functions": {"total": int, "covered": int, "skipped": int, "pct": float}, + "branches": {"total": int, "covered": int, "skipped": int, "pct": float} + }, + "src/file.jl": { + "lines": {"total": int, "covered": int, "skipped": int, "pct": float}, + "statements": {"total": int, "covered": int, "skipped": int, "pct": float}, + "functions": {"total": int, "covered": int, "skipped": int, "pct": float}, + "branches": {"total": int, "covered": int, "skipped": int, "pct": float} + } + } +``` +Note: + Since Julia only natively tracks line-level coverage, the `statements`, `functions`, + and `branches` fields are populated using the line coverage statistics to conform to + the Jest JSON schema. +""" +function generate_json_summary(coverage::PackageCoverage, filename="coverage-summary.json"; test_args=[""]) + # Determine the top-level key name + top_key = (isempty(test_args) || (length(test_args) == 1 && test_args[1] == "")) ? "total" : join(test_args, " ") + + # Helper to construct a metrics dictionary + function make_metrics(total, covered) + pct = total == 0 ? 100.0 : round(covered / total * 100, digits=2) + m = OrderedDict( + "total" => total, + "covered" => covered, + "skipped" => 0, + "pct" => pct + ) + return OrderedDict( + "lines" => m, + "statements" => m, + "functions" => m, + "branches" => m + ) + end + + data = OrderedDict() + data[top_key] = make_metrics(coverage.lines_tracked, coverage.lines_hit) + + for f in coverage.files + # Normalize path slashes to forward slashes for cross-platform JSON summary + f_name = replace(f.filename, '\\' => '/') + data[f_name] = make_metrics(f.lines_tracked, f.lines_hit) + end + + # Write to the file in package coverage dir + out_dir = joinpath(coverage.package_dir, COVDIR) + mkpath(out_dir) + out_path = joinpath(out_dir, filename) + + open(out_path, "w") do io + JSON.print(io, data, 2) + println(io) + end + + @info("generated JSON summary $(out_path).") + return out_path +end + + + """ $(SIGNATURES) @@ -734,6 +854,138 @@ Get the percentage of lines covered in the total, with formatting _percent(lines_total::Integer, lines_covered::Integer) = lines_total == 0 ? "0.0" : string(lines_covered / lines_total) +""" +$(SIGNATURES) + +Check if a string looks like JSON content (starts with `{` or `[`, ends with `}` or `]`). +""" +function is_json_content(s::AbstractString) + t = strip(s) + (startswith(t, "{") && endswith(t, "}")) || (startswith(t, "[") && endswith(t, "]")) +end + +""" +$(SIGNATURES) + +Parse a JSON summary from either a file path or inline JSON string. +""" +function parse_json_summary(input::AbstractString) + if is_json_content(input) + return JSON.parse(input; dicttype=OrderedDict{String,Any}) + elseif isfile(input) + return JSON.parsefile(input; dicttype=OrderedDict{String,Any}) + else + # Treat as JSON string (fallback for backwards compatibility) + return JSON.parse(input; dicttype=OrderedDict{String,Any}) + end +end + +""" +$(SIGNATURES) + +Compare two coverage JSON summaries (either as file paths or JSON strings). +Prints a table of coverage delta information to `io` (defaults to `stdout`). +Returns `true` if coverage did not decrease, and `false` if it decreased. +""" +function compare_coverage_json_summaries(old::AbstractString, new::AbstractString, io::IO = stdout) + is_json(s) = begin + t = strip(s) + (startswith(t, "{") && endswith(t, "}")) || (startswith(t, "[") && endswith(t, "]")) + end + if isempty(strip(old)) || (!is_json(old) && !isfile(old)) + println(io, "No previous coverage summary found. Automatic pass.") + return true + end + old_data = parse_json_summary(old) + new_data = parse_json_summary(new) + return compare_coverage_json_summaries(old_data, new_data, io) +end + +function compare_coverage_json_summaries(old_data::AbstractDict, new_data::AbstractDict, io::IO = stdout) + # Helper to find overall summary key (not ending with .jl) + function find_overall_key(dict) + for k in keys(dict) + if !endswith(k, ".jl") + return k + end + end + return "total" # fallback + end + + # Helper to safely get lines.pct from a data dict, returning default if not found + function get_pct(data, key; default=nothing) + haskey(data, key) ? data[key]["lines"]["pct"] : default + end + + old_key = find_overall_key(old_data) + new_key = find_overall_key(new_data) + + # Extract overall coverage percentages + old_pct = get_pct(old_data, old_key; default=nothing) + new_pct = get_pct(new_data, new_key; default=nothing) + + # Get all keys to compare using Set union + keys_all = collect(keys(new_data) ∪ keys(old_data)) + + # Helper to format coverage values for display: returns (old_str, new_str, delta_str) + function format_coverage_row(old_val, new_val) + # Calculate delta if both are numbers + if old_val isa Number && new_val isa Number + delta = round(new_val - old_val, digits=2) + delta_str = delta >= 0 ? "+$delta%" : "$delta%" + else + delta_str = "-" + end + + # Format percentage strings + old_pct_str = old_val isa Number ? "$old_val%" : old_val + new_pct_str = new_val isa Number ? "$new_val%" : new_val + + return (old_pct_str, new_pct_str, delta_str) + end + + # Build rows for the PrettyTable as a 2D Matrix + rows = Matrix{Any}(undef, length(keys_all), 4) + for (i, k) in enumerate(keys_all) + old_val = get_pct(old_data, k; default="-") + new_val = get_pct(new_data, k; default="-") + + old_pct_str, new_pct_str, delta_str = format_coverage_row(old_val, new_val) + + rows[i, 1] = k + rows[i, 2] = old_pct_str + rows[i, 3] = new_pct_str + rows[i, 4] = delta_str + end + + # Print the table using version-appropriate kwargs + @static if pkgversion(PrettyTables) < v"3.0.0" + pretty_table( + io, + rows; + title = "Coverage Comparison", + header = ["File/Section", "Old Coverage", "New Coverage", "Delta"], + alignment = [:l, :r, :r, :r] + ) + else + pretty_table( + io, + rows; + title = "Coverage Comparison", + column_labels = ["File/Section", "Old Coverage", "New Coverage", "Delta"], + alignment = [:l, :r, :r, :r] + ) + end + + decreased = false + if !isnothing(old_pct) && !isnothing(new_pct) + if new_pct < old_pct + decreased = true + end + end + + return !decreased +end end # module diff --git a/test/Project.toml b/test/Project.toml index 195c520..b5cb3a2 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,5 +1,6 @@ [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" FileCmp = "343a5541-a696-4ae7-8102-bc61c09e1896" +LocalCoverage = "5f6e1e16-694c-5876-87ef-16b5274f298e" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/runtests.jl b/test/runtests.jl index 5163ccb..c07df28 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -54,6 +54,34 @@ function test_coverage(pkg; @info "Printing coverage information for visual debugging" show(stdout, cov) show(IOContext(stdout, :print_gaps => true), cov) + + # Testing JSON summary generation (low-level) + jsonsummary = joinpath(covdir, "coverage-summary.json") + @test !isfile(jsonsummary) + generate_json_summary(cov, "coverage-summary.json"; test_args = test_args) + @test isfile(jsonsummary) + json_content = read(jsonsummary, String) + @test occursin(test_args == [""] ? "\"total\":" : "\"" * join(test_args, " ") * "\":", json_content) + @test occursin("\"lines\":", json_content) + @test occursin("\"statements\":", json_content) + @test occursin("\"functions\":", json_content) + @test occursin("\"branches\":", json_content) + @test occursin("\"total\":", json_content) + @test occursin("\"pct\":", json_content) + rm(jsonsummary) + + # Test generate_coverage with json_summary=true + direct_json = joinpath(covdir, "coverage-summary-direct.json") + @test !isfile(direct_json) + generate_coverage(pkg; + run_test = false, + test_args = test_args, + folder_list = folder_list, + file_list = file_list, + json_summary = JSONSummaryOptions(filename = "coverage-summary-direct.json")) + @test isfile(direct_json) + direct_content = read(direct_json, String) + @test occursin(test_args == [""] ? "\"total\":" : "\"" * join(test_args, " ") * "\":", direct_content) end xmltrace = joinpath(covdir,"lcov.xml") @@ -112,9 +140,180 @@ end end end +@testset "compare_coverage_json_summaries" begin + # Mocking old and new JSON string summaries + old_json = """ + { + "total": { + "lines": {"total": 10, "covered": 5, "skipped": 0, "pct": 50.0}, + "statements": {"total": 10, "covered": 5, "skipped": 0, "pct": 50.0}, + "functions": {"total": 10, "covered": 5, "skipped": 0, "pct": 50.0}, + "branches": {"total": 10, "covered": 5, "skipped": 0, "pct": 50.0} + }, + "src/bar.jl": { + "lines": {"total": 5, "covered": 2, "skipped": 0, "pct": 40.0}, + "statements": {"total": 5, "covered": 2, "skipped": 0, "pct": 40.0}, + "functions": {"total": 5, "covered": 2, "skipped": 0, "pct": 40.0}, + "branches": {"total": 5, "covered": 2, "skipped": 0, "pct": 40.0} + } + } + """ + + # Increased coverage scenario + inc_json = """ + { + "total": { + "lines": {"total": 10, "covered": 8, "skipped": 0, "pct": 80.0}, + "statements": {"total": 10, "covered": 8, "skipped": 0, "pct": 80.0}, + "functions": {"total": 10, "covered": 8, "skipped": 0, "pct": 80.0}, + "branches": {"total": 10, "covered": 8, "skipped": 0, "pct": 80.0} + }, + "src/bar.jl": { + "lines": {"total": 5, "covered": 4, "skipped": 0, "pct": 80.0}, + "statements": {"total": 5, "covered": 4, "skipped": 0, "pct": 80.0}, + "functions": {"total": 5, "covered": 4, "skipped": 0, "pct": 80.0}, + "branches": {"total": 5, "covered": 4, "skipped": 0, "pct": 80.0} + } + } + """ + + # Decreased coverage scenario + dec_json = """ + { + "total": { + "lines": {"total": 10, "covered": 2, "skipped": 0, "pct": 20.0}, + "statements": {"total": 10, "covered": 2, "skipped": 0, "pct": 20.0}, + "functions": {"total": 10, "covered": 2, "skipped": 0, "pct": 20.0}, + "branches": {"total": 10, "covered": 2, "skipped": 0, "pct": 20.0} + }, + "src/bar.jl": { + "lines": {"total": 5, "covered": 1, "skipped": 0, "pct": 20.0}, + "statements": {"total": 5, "covered": 1, "skipped": 0, "pct": 20.0}, + "functions": {"total": 5, "covered": 1, "skipped": 0, "pct": 20.0}, + "branches": {"total": 5, "covered": 1, "skipped": 0, "pct": 20.0} + } + } + """ + + # Test JSON string parsing and direct comparison + @test compare_coverage_json_summaries(old_json, old_json) == true # unchanged is true (not decreased) + @test compare_coverage_json_summaries(old_json, inc_json) == true # increased is true + @test compare_coverage_json_summaries(old_json, dec_json) == false # decreased is false + @test compare_coverage_json_summaries("", old_json) == true # empty string is automatic pass + @test compare_coverage_json_summaries("nonexistent_summary_file_abc123.json", old_json) == true # missing file is automatic pass + + # Test file-based comparison + mktempdir() do tmp_dir + old_file = joinpath(tmp_dir, "old.json") + inc_file = joinpath(tmp_dir, "inc.json") + dec_file = joinpath(tmp_dir, "dec.json") + + write(old_file, old_json) + write(inc_file, inc_json) + write(dec_file, dec_json) + + @test compare_coverage_json_summaries(old_file, old_file) == true + @test compare_coverage_json_summaries(old_file, inc_file) == true + @test compare_coverage_json_summaries(old_file, dec_file) == false + + # Test integrated generate_coverage comparison + cov_dir = joinpath(dirname(@__FILE__), "DummyPackage", "coverage") + rm(cov_dir; force=true, recursive=true) + + # Scenario: old file coverage is lower (20.0%) than generated coverage -> should pass + cov = generate_coverage("DummyPackage"; run_test=true, json_summary=JSONSummaryOptions(comparison_filename=dec_file, comparison_fail_on_decrease=false)) + @test isfile(joinpath(cov_dir, "coverage-summary.json")) + + # Scenario: old file coverage is higher (80.0%) than generated coverage (~28.57%) -> should fail + @test_throws ErrorException generate_coverage("DummyPackage"; run_test=true, json_summary=JSONSummaryOptions(comparison_filename=inc_file, comparison_fail_on_decrease=true)) + + # Scenario: old file is missing -> should pass automatically + cov = generate_coverage("DummyPackage"; run_test=true, json_summary=JSONSummaryOptions(comparison_filename="missing_file_abc.json", comparison_fail_on_decrease=true)) + @test isfile(joinpath(cov_dir, "coverage-summary.json")) + + # Cleanup + rm(cov_dir; force=true, recursive=true) + end +end + @test LocalCoverage.find_gaps([nothing, 0, 0, 0, 2, 3, 0, nothing, 0, 3, 0, 6, 2]) == [2:4, 7:7, 9:9, 11:11] +@testset "report_coverage_and_exit" begin + pkg_root = dirname(@__DIR__) + dummy_pkg_dir = escape_string(joinpath(@__DIR__, "DummyPackage")) + + # 1. Test report_coverage_and_exit(pkg; target_coverage) + # 1a. Target met (10% target coverage on DummyPackage should meet it and exit 0) + cmd_met = `$(Base.julia_cmd()) --project=$(pkg_root) -e "using LocalCoverage; using Pkg; Pkg.activate(\"$(dummy_pkg_dir)\"); report_coverage_and_exit(\"DummyPackage\"; target_coverage=10)"` + p_met = open(cmd_met, "r") + out_met = read(p_met, String) + wait(p_met) + close(p_met) + @test p_met.exitcode == 0 + @test occursin("Target coverage was met", out_met) + @test occursin("10%", out_met) + @test occursin("TOTAL", out_met) # Summary was printed + + # 1b. Target wasn't met (100% target coverage on DummyPackage should fail it and exit 1) + cmd_unmet = `$(Base.julia_cmd()) --project=$(pkg_root) -e "using LocalCoverage; using Pkg; Pkg.activate(\"$(dummy_pkg_dir)\"); report_coverage_and_exit(\"DummyPackage\"; target_coverage=100)"` + p_unmet = open(cmd_unmet, "r") + out_unmet = read(p_unmet, String) + wait(p_unmet) + close(p_unmet) + @test p_unmet.exitcode == 1 + @test occursin("Target coverage wasn't met", out_unmet) + @test occursin("100%", out_unmet) + @test occursin("TOTAL", out_unmet) # Summary was printed + + # 1c. Target met but with print_summary = false (summary should not be printed) + cmd_no_summary = `$(Base.julia_cmd()) --project=$(pkg_root) -e "using LocalCoverage; using Pkg; Pkg.activate(\"$(dummy_pkg_dir)\"); report_coverage_and_exit(\"DummyPackage\"; target_coverage=10, print_summary=false)"` + p_no_summary = open(cmd_no_summary, "r") + out_no_summary = read(p_no_summary, String) + wait(p_no_summary) + close(p_no_summary) + @test p_no_summary.exitcode == 0 + @test occursin("Target coverage was met", out_no_summary) + @test occursin("10%", out_no_summary) + @test !occursin("TOTAL", out_no_summary) # Summary was NOT printed + + # 1d. Target met with print_gaps = true + cmd_gaps = `$(Base.julia_cmd()) --project=$(pkg_root) -e "using LocalCoverage; using Pkg; Pkg.activate(\"$(dummy_pkg_dir)\"); report_coverage_and_exit(\"DummyPackage\"; target_coverage=10, print_gaps=true)"` + p_gaps = open(cmd_gaps, "r") + out_gaps = read(p_gaps, String) + wait(p_gaps) + close(p_gaps) + @test p_gaps.exitcode == 0 + @test occursin("Target coverage was met", out_gaps) + @test occursin("10%", out_gaps) + @test occursin("TOTAL", out_gaps) + + # 2. Test report_coverage_and_exit(coverage::PackageCoverage; target_coverage) + # 2a. Target met (exit 0) + cmd_cov_met = `$(Base.julia_cmd()) --project=$(pkg_root) -e "using LocalCoverage; using Pkg; Pkg.activate(\"$(dummy_pkg_dir)\"); cov = generate_coverage(\"DummyPackage\"); report_coverage_and_exit(cov; target_coverage=10, print_summary=false)"` + p_cov_met = open(cmd_cov_met, "r") + out_cov_met = read(p_cov_met, String) + wait(p_cov_met) + close(p_cov_met) + @test p_cov_met.exitcode == 0 + @test occursin("Target coverage was met", out_cov_met) + @test occursin("10%", out_cov_met) + + # 2b. Target wasn't met (exit 1) + cmd_cov_unmet = `$(Base.julia_cmd()) --project=$(pkg_root) -e "using LocalCoverage; using Pkg; Pkg.activate(\"$(dummy_pkg_dir)\"); cov = generate_coverage(\"DummyPackage\"); report_coverage_and_exit(cov; target_coverage=100, print_summary=false)"` + p_cov_unmet = open(cmd_cov_unmet, "r") + out_cov_unmet = read(p_cov_unmet, String) + wait(p_cov_unmet) + close(p_cov_unmet) + @test p_cov_unmet.exitcode == 1 + @test occursin("Target coverage wasn't met", out_cov_unmet) + @test occursin("100%", out_cov_unmet) + + # Cleanup generated coverage directory for DummyPackage + rm(joinpath(@__DIR__, "DummyPackage", "coverage"); force=true, recursive=true) +end + + # automated QA import Aqua Aqua.test_all(LocalCoverage)