diff --git a/scripts/report-test-coverage.sh b/scripts/report-test-coverage.sh new file mode 100755 index 0000000..90d5150 --- /dev/null +++ b/scripts/report-test-coverage.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env sh +# Per-workspace-crate line and branch coverage report. +# +# This is deliberately a report over every host-testable workspace package, +# not a misleading aggregate. STM32F7 platform/board images are target-only: +# their `panic-halt` handler cannot link into the host test harness, so they +# are reported as N/A and must be covered by target checks plus bench +# evidence instead. +# +# Prerequisites (kept out of the firmware dependency graph): +# cargo install cargo-llvm-cov --locked +# rustup toolchain install nightly --component llvm-tools-preview +# python3 (standard library only, for strict source-level accounting) +set -eu + +if ! cargo llvm-cov --version >/dev/null 2>&1; then + echo "cargo-llvm-cov is required; install it with: cargo install cargo-llvm-cov --locked" >&2 + exit 127 +fi + +if ! rustup toolchain list | grep -q '^nightly'; then + echo "nightly with llvm-tools-preview is required for branch coverage" >&2 + exit 127 +fi + +if ! python3 --version >/dev/null 2>&1; then + echo "python3 is required for strict source-level coverage accounting" >&2 + exit 127 +fi + +python3 scripts/test-summarize-test-coverage.py + +# Keep instrumentation products outside a possibly shared target directory. +: "${CARGO_TARGET_DIR:=/tmp/kaptan-test-coverage}" +export CARGO_TARGET_DIR +mkdir -p "$CARGO_TARGET_DIR" + +# `cargo llvm-cov` merges every instrumented object below `CARGO_TARGET_DIR`. +# A shared target directory therefore makes a later strict-core report include +# an unexecuted copy of a shared dependency built while testing an earlier +# package. Keep reports together but isolate each strict-core invocation. The +# non-core packages retain the shared directory: several intentionally +# target-only crates have no host-instrumented object to export by themselves. +coverage_root=$CARGO_TARGET_DIR + +core_incomplete=0 + +report_package() { + package=$1 + source_path=$2 + shift 2 + case "$package" in + kaptan-math|kaptan-navigation|kaptan-control|kaptan-safety) + package_target_dir="$coverage_root/$package" + lcov_report="$coverage_root/$package.lcov" + json_report="$coverage_root/$package.json" + report_format=strict-core + ;; + *) + package_target_dir=$coverage_root + lcov_report="$coverage_root/$package.lcov" + report_format=lcov + ;; + esac + + case "$report_format" in + strict-core) + # Keep the strict core isolated, then export both representations + # from the same test run. LCOV `DA` defines source lines; JSON + # coordinates define branch identities across generic instances. + RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov \ + --branch \ + --no-report \ + -p "$package" \ + "$@" + RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov report \ + --lcov \ + --output-path "$lcov_report" + RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov report \ + --json \ + --output-path "$json_report" + totals=$(python3 scripts/summarize-test-coverage.py \ + "$lcov_report" "$json_report" "$source_path") + ;; + lcov) + RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov \ + --branch \ + --lcov \ + --output-path "$lcov_report" \ + -p "$package" \ + "$@" + totals=$(awk -v source_path="$source_path" ' + /^SF:/ { selected = index($0, source_path) != 0 } + selected && /^LF:/ { lines_found += substr($0, 4) } + selected && /^LH:/ { lines_hit += substr($0, 4) } + selected && /^BRF:/ { branches_found += substr($0, 5) } + selected && /^BRH:/ { branches_hit += substr($0, 5) } + END { + printf "%d %d %d %d", lines_hit, lines_found, branches_hit, branches_found + } + ' "$lcov_report") + ;; + esac + set -- $totals + printf '%-26s lines %s/%s, branches %s/%s\n' "$package" "$1" "$2" "$3" "$4" + + case "$package" in + kaptan-math|kaptan-navigation|kaptan-control|kaptan-safety) + if [ "$1" -ne "$2" ] || [ "$3" -ne "$4" ]; then + core_incomplete=1 + fi + ;; + esac +} + +echo "Host test coverage (reports: $CARGO_TARGET_DIR; strict core unions LCOV DA lines and JSON branches)" +report_package kaptan-actuator /libs/kaptan-actuator/ +report_package kaptan-board /libs/kaptan-board/ +report_package kaptan-control /libs/kaptan-control/ +report_package kaptan-data-channel /libs/kaptan-data-channel/ +report_package kaptan-dma /libs/kaptan-dma/ +report_package kaptan-drivers /libs/kaptan-drivers/ +report_package kaptan-i2c /libs/kaptan-i2c/ +report_package kaptan-math /libs/kaptan-math/ +report_package kaptan-mavlink /libs/kaptan-mavlink/ --features dialect-kaptan +report_package kaptan-navigation /libs/kaptan-navigation/ --features sitl +report_package kaptan-safety /libs/kaptan-safety/ +report_package kaptan-sensor /libs/kaptan-sensor/ +report_package kaptan-spi /libs/kaptan-spi/ +report_package kaptan-time /libs/kaptan-time/ +report_package kaptan-tx /libs/kaptan-tx/ +report_package kaptan-sitl /tools/sitl/ + +echo "platform-stm32f7 target-only/N/A (run cargo check -p platform-stm32f7 --lib)" +echo "kaptan-geprc-f722-v2 target-only/N/A (run its thumbv7em-none-eabihf check)" +echo "kaptan-kakute-f7-mini-v3 target-only/N/A (run its thumbv7em-none-eabihf check)" + +if [ "$core_incomplete" -ne 0 ]; then + echo "Strict core coverage is incomplete; reports above identify each crate." >&2 + exit 1 +fi diff --git a/scripts/summarize-test-coverage.py b/scripts/summarize-test-coverage.py new file mode 100644 index 0000000..929edb8 --- /dev/null +++ b/scripts/summarize-test-coverage.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Summarize strict-core LCOV lines and LLVM JSON branches by source location. + +`llvm-cov` emits one coverage record for every generic monomorphization. Its +per-file totals add those records, which can report an uncovered duplicate even +when the corresponding source line or branch has been exercised by another +monomorphization. This helper instead applies the gate to source locations: + +* a source line is identified by its LCOV `DA` file and line number; +* a branch outcome is identified by its full source span; and +* a location is covered when any matching monomorphization executed it. + +JSON segments are deliberately not used as a line denominator: they mark LLVM +instrumentation-region boundaries, not every source line represented by an +LCOV `DA` record. Replacing `DA` with segments would silently lower the strict +line denominator. + +Macro-expansion records are deliberately rejected. Their source span denotes +the invocation, not necessarily the expanded predicate, so folding them into +the source-level key would make the gate unsound. Extend this helper with an +explicit expansion identity before adding macro-expanded strict-core coverage. +""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Iterable +from pathlib import Path +import sys + + +class CoverageFormatError(ValueError): + """Coverage records cannot be safely summarized by source location.""" + + +def selected_files(report: dict[object, object], source_path: str) -> Iterable[dict[object, object]]: + """Yield selected file records after rejecting ambiguous macro expansion.""" + exports = report.get("data") + if not isinstance(exports, list): + raise CoverageFormatError("LLVM JSON has no data array") + + for export in exports: + if not isinstance(export, dict): + raise CoverageFormatError("LLVM JSON data array contains a non-object") + files = export.get("files") + if not isinstance(files, list): + raise CoverageFormatError("LLVM JSON export has no files array") + for file_record in files: + if not isinstance(file_record, dict): + raise CoverageFormatError("LLVM JSON files array contains a non-object") + filename = file_record.get("filename") + if not isinstance(filename, str) or source_path not in filename: + continue + expansions = file_record.get("expansions", []) + if not isinstance(expansions, list): + raise CoverageFormatError(f"{filename}: expansions is not an array") + if expansions: + raise CoverageFormatError( + f"{filename}: macro-expansion coverage is present; " + "refusing to merge it by invocation source span" + ) + yield file_record + + +def lcov_line_totals(report: str, source_path: str) -> tuple[int, int, set[str]]: + """Return the union of selected LCOV `DA` lines and their source files.""" + lines: dict[tuple[str, int], bool] = {} + files: set[str] = set() + filename: str | None = None + + for record in report.splitlines(): + if record.startswith("SF:"): + filename = record[3:] + if source_path in filename: + files.add(filename) + continue + if record == "end_of_record": + filename = None + continue + if filename is None or source_path not in filename or not record.startswith("DA:"): + continue + fields = record[3:].split(",", 2) + if len(fields) < 2: + raise CoverageFormatError(f"{filename}: malformed LCOV DA record") + try: + line = int(fields[0]) + count = int(fields[1]) + except ValueError as error: + raise CoverageFormatError(f"{filename}: non-integer LCOV DA record") from error + if line <= 0 or count < 0: + raise CoverageFormatError(f"{filename}: invalid LCOV DA location or count") + key = (filename, line) + lines[key] = lines.get(key, False) or count > 0 + + if not files: + raise CoverageFormatError(f"no LCOV source file contains {source_path!r}") + if not lines: + raise CoverageFormatError(f"no LCOV DA records found under {source_path!r}") + return sum(lines.values()), len(lines), files + + +def branch_totals(report: dict[object, object], source_path: str) -> tuple[int, int, set[str]]: + """Return the union of selected JSON branch outcomes and source files.""" + branches: dict[tuple[str, int, int, int, int], list[bool]] = {} + files: set[str] = set() + + for file_record in selected_files(report, source_path): + filename = file_record["filename"] + assert isinstance(filename, str) + files.add(filename) + + raw_branches = file_record.get("branches") + if not isinstance(raw_branches, list): + raise CoverageFormatError(f"{filename}: branches is not an array") + for branch in raw_branches: + if ( + not isinstance(branch, list) + or len(branch) != 9 + or not all(type(value) is int for value in branch) + ): + raise CoverageFormatError(f"{filename}: invalid branch record") + ( + start_line, + start_column, + end_line, + end_column, + true_count, + false_count, + file_id, + expanded_file_id, + _kind, + ) = branch + if ( + start_line <= 0 + or start_column <= 0 + or end_line < start_line + or end_column <= 0 + or (end_line == start_line and end_column < start_column) + or true_count < 0 + or false_count < 0 + or file_id < 0 + or expanded_file_id < 0 + ): + raise CoverageFormatError(f"{filename}: invalid branch location or count") + if file_id != expanded_file_id: + raise CoverageFormatError( + f"{filename}:{start_line}:{start_column}: macro-expanded branch is present; " + "refusing to merge it by invocation source span" + ) + key = (filename, start_line, start_column, end_line, end_column) + outcomes = branches.setdefault(key, [False, False]) + outcomes[0] = outcomes[0] or true_count > 0 + outcomes[1] = outcomes[1] or false_count > 0 + + if not files: + raise CoverageFormatError(f"no LLVM JSON source file contains {source_path!r}") + branch_hit = sum(sum(outcomes) for outcomes in branches.values()) + return branch_hit, 2 * len(branches), files + + +def coverage_totals( + lcov_report: str, json_report: dict[object, object], source_path: str +) -> tuple[int, int, int, int]: + """Return strict source-level line and branch coverage for one core crate.""" + lines_hit, lines_found, lcov_files = lcov_line_totals(lcov_report, source_path) + branch_hit, branches_found, json_files = branch_totals(json_report, source_path) + if lcov_files != json_files: + raise CoverageFormatError( + "LCOV/JSON source-file mismatch: " + f"LCOV={sorted(lcov_files)!r}, JSON={sorted(json_files)!r}" + ) + return lines_hit, lines_found, branch_hit, branches_found + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("lcov_report", type=Path, help="LCOV report from cargo llvm-cov") + parser.add_argument("json_report", type=Path, help="LLVM JSON report from cargo llvm-cov") + parser.add_argument("source_path", help="workspace-relative path fragment to include") + arguments = parser.parse_args() + + try: + lcov_report = arguments.lcov_report.read_text(encoding="utf-8") + with arguments.json_report.open(encoding="utf-8") as report_file: + json_report = json.load(report_file) + if not isinstance(json_report, dict): + raise CoverageFormatError("LLVM JSON root is not an object") + print(*coverage_totals(lcov_report, json_report, arguments.source_path)) + except (CoverageFormatError, OSError, json.JSONDecodeError) as error: + print(f"coverage summary error: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-summarize-test-coverage.py b/scripts/test-summarize-test-coverage.py new file mode 100644 index 0000000..8a7206d --- /dev/null +++ b/scripts/test-summarize-test-coverage.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Synthetic regression tests for strict source-level coverage accounting.""" + +from __future__ import annotations + +import importlib.util +import unittest + + +def load_summary_module(): + spec = importlib.util.spec_from_file_location( + "summarize_test_coverage", "scripts/summarize-test-coverage.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +SUMMARY = load_summary_module() +SOURCE = "/libs/kaptan-navigation/" +FILENAME = "/repo/libs/kaptan-navigation/src/eskf.rs" + + +def json_report(*, expansions=None, expanded_file_id=0): + return { + "data": [ + { + "files": [ + { + "filename": FILENAME, + "expansions": [] if expansions is None else expansions, + # Segments intentionally differ from DA lines: strict + # line coverage must not lower its denominator to them. + "segments": [[10, 1, 3, True, True, False]], + "branches": [ + [10, 5, 10, 10, 0, 3, 0, expanded_file_id, 4], + [10, 5, 10, 10, 2, 0, 0, expanded_file_id, 4], + [11, 5, 11, 10, 0, 0, 0, expanded_file_id, 4], + ], + } + ] + } + ] + } + + +LCOV = """SF:/repo/libs/kaptan-navigation/src/eskf.rs +DA:10,0 +DA:11,0 +end_of_record +SF:/repo/libs/kaptan-navigation/src/eskf.rs +DA:10,3 +DA:12,1 +end_of_record +""" + + +class CoverageSummaryTests(unittest.TestCase): + def test_unions_da_lines_and_full_span_branch_outcomes(self): + self.assertEqual( + SUMMARY.coverage_totals(LCOV, json_report(), SOURCE), + # DA lines: 10 is covered by the second generic instance; 11 is + # uncovered; 12 is covered. Branch outcomes: the two duplicate + # source spans cover both outcomes, while line 11 covers neither. + (2, 3, 2, 4), + ) + + def test_rejects_macro_expansion_records(self): + with self.assertRaises(SUMMARY.CoverageFormatError): + SUMMARY.coverage_totals(LCOV, json_report(expansions=[{}]), SOURCE) + with self.assertRaises(SUMMARY.CoverageFormatError): + SUMMARY.coverage_totals(LCOV, json_report(expanded_file_id=1), SOURCE) + + +if __name__ == "__main__": + unittest.main()