build: replace argument dispatch and retain source-bound verification
Replace argument recovery with direct build targets. Run host, target, board, coverage, Miri and selected Kani checks while retaining source identities, logs and artifacts, including failed attempts. Keep the PID proof diagnostic exception explicit and match its exact identity alongside passing contract assertions and covers. Validation: final-source verification matrix, assembled workflow tests and documentation build.
This commit is contained in:
Onur Özkan committed
1 parent
4953f0a974
commit
62b18d5b2e
26 files changed
+2177
-1052
No files matched your search
@@ -1,94 +1,90 @@
|
||||
#!/usr/bin/env sh
|
||||
# Per-workspace-crate line and branch coverage report.
|
||||
#
|
||||
# This is deliberately a report over every host-testable workspace package,
|
||||
# not a misleading aggregate. STM32F7 platform/board images are target-only:
|
||||
# their `panic-halt` handler cannot link into the host test harness, so they
|
||||
# are reported as N/A and must be covered by target checks plus bench
|
||||
# evidence instead.
|
||||
#
|
||||
# Prerequisites (kept out of the firmware dependency graph):
|
||||
# cargo install cargo-llvm-cov --locked
|
||||
# rustup toolchain install nightly --component llvm-tools-preview
|
||||
# python3 (standard library only, for strict source-level accounting)
|
||||
# Host workspace coverage. The four strict-core packages require 100% source
|
||||
# lines and branch outcomes; all other package totals are informational.
|
||||
# Run from firmware/ in the locked Nix shell (cargo-llvm-cov 0.8.7).
|
||||
# Install the separate coverage compiler with:
|
||||
# rustup toolchain install nightly-2025-08-01 --profile minimal --component llvm-tools-preview
|
||||
set -eu
|
||||
|
||||
if ! cargo llvm-cov --version >/dev/null 2>&1; then
|
||||
echo "cargo-llvm-cov is required; install it with: cargo install cargo-llvm-cov --locked" >&2
|
||||
export RUSTUP_TOOLCHAIN=nightly-2025-08-01
|
||||
export CARGO_BUILD_JOBS=2
|
||||
if [ "$(cargo llvm-cov --version)" != "cargo-llvm-cov 0.8.7" ]; then
|
||||
echo "Use cargo-llvm-cov 0.8.7 from nix develop (flake.lock)." >&2
|
||||
exit 127
|
||||
fi
|
||||
|
||||
if ! rustup toolchain list | grep -q '^nightly'; then
|
||||
echo "nightly with llvm-tools-preview is required for branch coverage" >&2
|
||||
exit 127
|
||||
if [ -n "${RUSTC_BOOTSTRAP:-}" ] || [ -n "${RUSTC:-}" ] || [ -n "${RUSTDOC:-}" ]; then
|
||||
echo "Coverage requires the pinned compiler without RUSTC_BOOTSTRAP/RUSTC/RUSTDOC overrides." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ! python3 --version >/dev/null 2>&1; then
|
||||
echo "python3 is required for strict source-level coverage accounting" >&2
|
||||
exit 127
|
||||
fi
|
||||
|
||||
sysroot=$(rustc --print sysroot)
|
||||
host=$(rustc -vV | awk '/^host:/ { print $2 }')
|
||||
export LLVM_COV="$sysroot/lib/rustlib/$host/bin/llvm-cov"
|
||||
export LLVM_PROFDATA="$sysroot/lib/rustlib/$host/bin/llvm-profdata"
|
||||
rustc -vV
|
||||
cargo llvm-cov --version
|
||||
"$LLVM_COV" --version
|
||||
"$LLVM_PROFDATA" --version
|
||||
echo "Coverage profile: dev; compiler and LLVM tools: $RUSTUP_TOOLCHAIN"
|
||||
python3 scripts/test-summarize-test-coverage.py
|
||||
|
||||
# Keep instrumentation products outside a possibly shared target directory.
|
||||
: "${CARGO_TARGET_DIR:=/tmp/kaptan-test-coverage}"
|
||||
export CARGO_TARGET_DIR
|
||||
mkdir -p "$CARGO_TARGET_DIR"
|
||||
|
||||
# `cargo llvm-cov` merges every instrumented object below `CARGO_TARGET_DIR`.
|
||||
# A shared target directory therefore makes a later strict-core report include
|
||||
# an unexecuted copy of a shared dependency built while testing an earlier
|
||||
# package. Keep reports together but isolate each strict-core invocation. The
|
||||
# non-core packages retain the shared directory: several intentionally
|
||||
# target-only crates have no host-instrumented object to export by themselves.
|
||||
# Keep coverage out of ordinary build artifacts. Strict-core invocations also
|
||||
# need separate object directories to avoid unexecuted dependency duplicates.
|
||||
: "${CARGO_TARGET_DIR:=$PWD/target/test-coverage}"
|
||||
coverage_root=$CARGO_TARGET_DIR
|
||||
mkdir -p "$coverage_root"
|
||||
|
||||
core_incomplete=0
|
||||
# Discover current workspace members, including host platform contracts and
|
||||
# proc-macro runtime harnesses. Only bare-metal boards are excluded.
|
||||
# Put the strict core first so unrelated test failures cannot hide its gaps.
|
||||
cargo metadata --no-deps --format-version 1 --locked --offline >"$coverage_root/metadata.json"
|
||||
python3 - "$coverage_root/metadata.json" >"$coverage_root/packages.txt" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
metadata = json.loads(Path(sys.argv[1]).read_text())
|
||||
excluded = {"kaptan-geprc-f722-v2", "kaptan-kakute-f7-mini-v3"}
|
||||
core = {"kaptan-math", "kaptan-navigation", "kaptan-control", "kaptan-safety"}
|
||||
packages = [p for p in metadata["packages"] if p["id"] in metadata["workspace_members"] and p["name"] not in excluded]
|
||||
if not core <= {p["name"] for p in packages}:
|
||||
raise ValueError("missing strict-core package")
|
||||
for package in sorted(packages, key=lambda p: (p["name"] not in core, p["name"])):
|
||||
source = Path(package["manifest_path"]).parent.relative_to(metadata["workspace_root"])
|
||||
print(package["name"], f"/{source}/")
|
||||
PY
|
||||
|
||||
report_package() {
|
||||
package=$1
|
||||
source_path=$2
|
||||
shift 2
|
||||
lcov_report="$coverage_root/$package.lcov"
|
||||
case "$package" in
|
||||
kaptan-math | kaptan-navigation | kaptan-control | kaptan-safety)
|
||||
package_target_dir="$coverage_root/$package"
|
||||
lcov_report="$coverage_root/$package.lcov"
|
||||
json_report="$coverage_root/$package.json"
|
||||
report_format=strict-core
|
||||
;;
|
||||
*)
|
||||
package_target_dir=$coverage_root
|
||||
lcov_report="$coverage_root/$package.lcov"
|
||||
package_target_dir="$coverage_root/other"
|
||||
report_format=lcov
|
||||
;;
|
||||
esac
|
||||
|
||||
printf '\nCoverage package: %s; extra arguments: %s\n' "$package" "$*"
|
||||
CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov --profile dev \
|
||||
--locked --offline --branch --no-report -p "$package" "$@" || return 1
|
||||
CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov report --profile dev \
|
||||
--locked --offline --lcov --output-path "$lcov_report" || return 1
|
||||
case "$report_format" in
|
||||
strict-core)
|
||||
# Keep the strict core isolated, then export both representations
|
||||
# from the same test run. LCOV `DA` defines source lines; JSON
|
||||
# coordinates define branch identities across generic instances.
|
||||
RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov \
|
||||
--branch \
|
||||
--no-report \
|
||||
-p "$package" \
|
||||
"$@"
|
||||
RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov report \
|
||||
--lcov \
|
||||
--output-path "$lcov_report"
|
||||
RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov report \
|
||||
--json \
|
||||
--output-path "$json_report"
|
||||
# LCOV DA defines source lines. JSON spans identify branch outcomes
|
||||
# across generic instances. Both exports use the same compiler/profile.
|
||||
CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov report --profile dev \
|
||||
--locked --offline --json --output-path "$json_report" || return 1
|
||||
totals=$(python3 scripts/summarize-test-coverage.py \
|
||||
"$lcov_report" "$json_report" "$source_path")
|
||||
"$lcov_report" "$json_report" "$source_path") || return 1
|
||||
;;
|
||||
lcov)
|
||||
RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov \
|
||||
--branch \
|
||||
--lcov \
|
||||
--output-path "$lcov_report" \
|
||||
-p "$package" \
|
||||
"$@"
|
||||
totals=$(awk -v source_path="$source_path" '
|
||||
/^SF:/ { selected = index($0, source_path) != 0 }
|
||||
selected && /^LF:/ { lines_found += substr($0, 4) }
|
||||
@@ -98,42 +94,45 @@ report_package() {
|
||||
END {
|
||||
printf "%d %d %d %d", lines_hit, lines_found, branches_hit, branches_found
|
||||
}
|
||||
' "$lcov_report")
|
||||
' "$lcov_report") || return 1
|
||||
;;
|
||||
esac
|
||||
# Both summarizers emit exactly four space-separated integer counters.
|
||||
# shellcheck disable=SC2086
|
||||
set -- $totals
|
||||
printf '%-26s lines %s/%s, branches %s/%s\n' "$package" "$1" "$2" "$3" "$4"
|
||||
|
||||
case "$package" in
|
||||
kaptan-math | kaptan-navigation | kaptan-control | kaptan-safety)
|
||||
if [ "$1" -ne "$2" ] || [ "$3" -ne "$4" ]; then
|
||||
core_incomplete=1
|
||||
printf '%-30s lines %s/%s, branches %s/%s\n' "$package" "$1" "$2" "$3" "$4"
|
||||
if [ "$report_format" = strict-core ]; then
|
||||
if [ "$2" -eq 0 ] || [ "$4" -eq 0 ] || [ "$1" -ne "$2" ] || [ "$3" -ne "$4" ]; then
|
||||
echo "$package: FAIL strict 100% line/branch gate" >&2
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
elif [ "$2" -eq 0 ]; then
|
||||
echo "$package: no instrumented host source; coverage unavailable"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Host test coverage (reports: $CARGO_TARGET_DIR; strict core unions LCOV DA lines and JSON branches)"
|
||||
report_package kaptan-actuator /libs/kaptan-actuator/
|
||||
report_package kaptan-control /libs/kaptan-control/
|
||||
report_package kaptan-data-channel /libs/kaptan-data-channel/
|
||||
report_package kaptan-dma /libs/kaptan-dma/
|
||||
report_package kaptan-drivers /libs/kaptan-drivers/
|
||||
report_package kaptan-i2c /libs/kaptan-i2c/
|
||||
report_package kaptan-math /libs/kaptan-math/
|
||||
report_package kaptan-mavlink /libs/kaptan-mavlink/ --features sensor-telemetry
|
||||
report_package kaptan-navigation /libs/kaptan-navigation/ --features sitl
|
||||
report_package kaptan-safety /libs/kaptan-safety/
|
||||
report_package kaptan-sensor /libs/kaptan-sensor/
|
||||
report_package kaptan-spi /libs/kaptan-spi/
|
||||
report_package kaptan-time /libs/kaptan-time/
|
||||
report_package kaptan-tx /libs/kaptan-tx/
|
||||
echo "Host test coverage (reports: $coverage_root; strict core unions LCOV DA lines and JSON branches)"
|
||||
echo "UI/compile-fail suites run under Rust 1.90.0 in make test, not under coverage nightly."
|
||||
echo "Proc-macro runtime-harness totals do not establish compiler-phase macro coverage."
|
||||
incomplete=0
|
||||
while read -r package source_path; do
|
||||
set --
|
||||
case "$package" in
|
||||
kaptan-navigation) set -- --features sensor-snapshots ;;
|
||||
kaptan-mavlink) set -- --features sensor-telemetry ;;
|
||||
# Diagnostic snapshots describe the production compiler's output. Keep
|
||||
# those suites in stable make test; instrument the runtime harnesses here.
|
||||
kaptan-config-derive) set -- --lib --test section ;;
|
||||
kaptan-err-reportable-macros) set -- --lib ;;
|
||||
esac
|
||||
if ! report_package "$package" "$source_path" "$@"; then
|
||||
echo "$package: coverage/test failure (see diagnostics above)" >&2
|
||||
incomplete=1
|
||||
fi
|
||||
done <"$coverage_root/packages.txt"
|
||||
|
||||
echo "platform-stm32f7 target-only/N/A (run cargo check -p platform-stm32f7 --lib)"
|
||||
echo "kaptan-geprc-f722-v2 target-only/N/A (run its thumbv7em-none-eabihf check)"
|
||||
echo "kaptan-kakute-f7-mini-v3 target-only/N/A (run its thumbv7em-none-eabihf check)"
|
||||
|
||||
if [ "$core_incomplete" -ne 0 ]; then
|
||||
echo "Strict core coverage is incomplete; reports above identify each crate." >&2
|
||||
echo "kaptan-geprc-f722-v2, kaptan-kakute-f7-mini-v3: target-only; make boards validates release images"
|
||||
if [ "$incomplete" -ne 0 ]; then
|
||||
echo "Coverage gate failed. Strict core gaps and test/report failures are listed above." >&2
|
||||
exit 1
|
||||
fi
|
||||
Reference in new issue
Block a user