build: replace argument dispatch and retain source-bound verification
Replace argument recovery with direct build targets. Run host, target, board, coverage, Miri and selected Kani checks while retaining source identities, logs and artifacts, including failed attempts. Keep the PID proof diagnostic exception explicit and match its exact identity alongside passing contract assertions and covers. Validation: final-source verification matrix, assembled workflow tests and documentation build.
This commit is contained in:
Onur Özkan committed
1 parent
4953f0a974
commit
62b18d5b2e
26 files changed
+2177
-1052
No files matched your search
@@ -5,6 +5,7 @@ Usage:
|
||||
scripts/analyze-fw.py <firmware.elf> <memory.x>
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from collections import namedtuple
|
||||
@@ -33,7 +34,6 @@ def number(text):
|
||||
|
||||
|
||||
def run(command):
|
||||
"""Run a tool and return stdout."""
|
||||
return subprocess.run(command, check=True, text=True, capture_output=True).stdout
|
||||
|
||||
|
||||
@@ -128,9 +128,8 @@ def merged_span_size(spans):
|
||||
def region_usage(name, layout, section_list, load_segments):
|
||||
"""Return runtime or load-image bytes occupying one linker region.
|
||||
|
||||
Allocated-section VMAs account for runtime RAM, while PT_LOAD physical file
|
||||
spans account for initializers stored in flash (notably a future `.data`).
|
||||
Taking their union avoids counting ordinary flash sections twice.
|
||||
Section VMAs count runtime RAM. PT_LOAD physical spans count flash storage,
|
||||
including RAM initializers. Their union avoids counting flash sections twice.
|
||||
"""
|
||||
spans = [
|
||||
(section.address, section.address + section.size)
|
||||
@@ -260,11 +259,11 @@ def sram_control_flow_errors(section, disassembly):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
raise ValueError(f"usage: {sys.argv[0]} <firmware.elf> <memory.x>")
|
||||
|
||||
elf = Path(sys.argv[1])
|
||||
memory_x = Path(sys.argv[2])
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("elf", type=Path)
|
||||
parser.add_argument("memory_x", type=Path)
|
||||
args = parser.parse_args()
|
||||
elf, memory_x = args.elf, args.memory_x
|
||||
if not elf.is_file():
|
||||
raise ValueError(f"ELF does not exist: {elf}")
|
||||
if not memory_x.is_file():
|
||||
|
||||
@@ -1,94 +1,90 @@
|
||||
#!/usr/bin/env sh
|
||||
# Per-workspace-crate line and branch coverage report.
|
||||
#
|
||||
# This is deliberately a report over every host-testable workspace package,
|
||||
# not a misleading aggregate. STM32F7 platform/board images are target-only:
|
||||
# their `panic-halt` handler cannot link into the host test harness, so they
|
||||
# are reported as N/A and must be covered by target checks plus bench
|
||||
# evidence instead.
|
||||
#
|
||||
# Prerequisites (kept out of the firmware dependency graph):
|
||||
# cargo install cargo-llvm-cov --locked
|
||||
# rustup toolchain install nightly --component llvm-tools-preview
|
||||
# python3 (standard library only, for strict source-level accounting)
|
||||
# Host workspace coverage. The four strict-core packages require 100% source
|
||||
# lines and branch outcomes; all other package totals are informational.
|
||||
# Run from firmware/ in the locked Nix shell (cargo-llvm-cov 0.8.7).
|
||||
# Install the separate coverage compiler with:
|
||||
# rustup toolchain install nightly-2025-08-01 --profile minimal --component llvm-tools-preview
|
||||
set -eu
|
||||
|
||||
if ! cargo llvm-cov --version >/dev/null 2>&1; then
|
||||
echo "cargo-llvm-cov is required; install it with: cargo install cargo-llvm-cov --locked" >&2
|
||||
export RUSTUP_TOOLCHAIN=nightly-2025-08-01
|
||||
export CARGO_BUILD_JOBS=2
|
||||
if [ "$(cargo llvm-cov --version)" != "cargo-llvm-cov 0.8.7" ]; then
|
||||
echo "Use cargo-llvm-cov 0.8.7 from nix develop (flake.lock)." >&2
|
||||
exit 127
|
||||
fi
|
||||
|
||||
if ! rustup toolchain list | grep -q '^nightly'; then
|
||||
echo "nightly with llvm-tools-preview is required for branch coverage" >&2
|
||||
exit 127
|
||||
if [ -n "${RUSTC_BOOTSTRAP:-}" ] || [ -n "${RUSTC:-}" ] || [ -n "${RUSTDOC:-}" ]; then
|
||||
echo "Coverage requires the pinned compiler without RUSTC_BOOTSTRAP/RUSTC/RUSTDOC overrides." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ! python3 --version >/dev/null 2>&1; then
|
||||
echo "python3 is required for strict source-level coverage accounting" >&2
|
||||
exit 127
|
||||
fi
|
||||
|
||||
sysroot=$(rustc --print sysroot)
|
||||
host=$(rustc -vV | awk '/^host:/ { print $2 }')
|
||||
export LLVM_COV="$sysroot/lib/rustlib/$host/bin/llvm-cov"
|
||||
export LLVM_PROFDATA="$sysroot/lib/rustlib/$host/bin/llvm-profdata"
|
||||
rustc -vV
|
||||
cargo llvm-cov --version
|
||||
"$LLVM_COV" --version
|
||||
"$LLVM_PROFDATA" --version
|
||||
echo "Coverage profile: dev; compiler and LLVM tools: $RUSTUP_TOOLCHAIN"
|
||||
python3 scripts/test-summarize-test-coverage.py
|
||||
|
||||
# Keep instrumentation products outside a possibly shared target directory.
|
||||
: "${CARGO_TARGET_DIR:=/tmp/kaptan-test-coverage}"
|
||||
export CARGO_TARGET_DIR
|
||||
mkdir -p "$CARGO_TARGET_DIR"
|
||||
|
||||
# `cargo llvm-cov` merges every instrumented object below `CARGO_TARGET_DIR`.
|
||||
# A shared target directory therefore makes a later strict-core report include
|
||||
# an unexecuted copy of a shared dependency built while testing an earlier
|
||||
# package. Keep reports together but isolate each strict-core invocation. The
|
||||
# non-core packages retain the shared directory: several intentionally
|
||||
# target-only crates have no host-instrumented object to export by themselves.
|
||||
# Keep coverage out of ordinary build artifacts. Strict-core invocations also
|
||||
# need separate object directories to avoid unexecuted dependency duplicates.
|
||||
: "${CARGO_TARGET_DIR:=$PWD/target/test-coverage}"
|
||||
coverage_root=$CARGO_TARGET_DIR
|
||||
mkdir -p "$coverage_root"
|
||||
|
||||
core_incomplete=0
|
||||
# Discover current workspace members, including host platform contracts and
|
||||
# proc-macro runtime harnesses. Only bare-metal boards are excluded.
|
||||
# Put the strict core first so unrelated test failures cannot hide its gaps.
|
||||
cargo metadata --no-deps --format-version 1 --locked --offline >"$coverage_root/metadata.json"
|
||||
python3 - "$coverage_root/metadata.json" >"$coverage_root/packages.txt" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
metadata = json.loads(Path(sys.argv[1]).read_text())
|
||||
excluded = {"kaptan-geprc-f722-v2", "kaptan-kakute-f7-mini-v3"}
|
||||
core = {"kaptan-math", "kaptan-navigation", "kaptan-control", "kaptan-safety"}
|
||||
packages = [p for p in metadata["packages"] if p["id"] in metadata["workspace_members"] and p["name"] not in excluded]
|
||||
if not core <= {p["name"] for p in packages}:
|
||||
raise ValueError("missing strict-core package")
|
||||
for package in sorted(packages, key=lambda p: (p["name"] not in core, p["name"])):
|
||||
source = Path(package["manifest_path"]).parent.relative_to(metadata["workspace_root"])
|
||||
print(package["name"], f"/{source}/")
|
||||
PY
|
||||
|
||||
report_package() {
|
||||
package=$1
|
||||
source_path=$2
|
||||
shift 2
|
||||
lcov_report="$coverage_root/$package.lcov"
|
||||
case "$package" in
|
||||
kaptan-math | kaptan-navigation | kaptan-control | kaptan-safety)
|
||||
package_target_dir="$coverage_root/$package"
|
||||
lcov_report="$coverage_root/$package.lcov"
|
||||
json_report="$coverage_root/$package.json"
|
||||
report_format=strict-core
|
||||
;;
|
||||
*)
|
||||
package_target_dir=$coverage_root
|
||||
lcov_report="$coverage_root/$package.lcov"
|
||||
package_target_dir="$coverage_root/other"
|
||||
report_format=lcov
|
||||
;;
|
||||
esac
|
||||
|
||||
printf '\nCoverage package: %s; extra arguments: %s\n' "$package" "$*"
|
||||
CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov --profile dev \
|
||||
--locked --offline --branch --no-report -p "$package" "$@" || return 1
|
||||
CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov report --profile dev \
|
||||
--locked --offline --lcov --output-path "$lcov_report" || return 1
|
||||
case "$report_format" in
|
||||
strict-core)
|
||||
# Keep the strict core isolated, then export both representations
|
||||
# from the same test run. LCOV `DA` defines source lines; JSON
|
||||
# coordinates define branch identities across generic instances.
|
||||
RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov \
|
||||
--branch \
|
||||
--no-report \
|
||||
-p "$package" \
|
||||
"$@"
|
||||
RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov report \
|
||||
--lcov \
|
||||
--output-path "$lcov_report"
|
||||
RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov report \
|
||||
--json \
|
||||
--output-path "$json_report"
|
||||
# LCOV DA defines source lines. JSON spans identify branch outcomes
|
||||
# across generic instances. Both exports use the same compiler/profile.
|
||||
CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov report --profile dev \
|
||||
--locked --offline --json --output-path "$json_report" || return 1
|
||||
totals=$(python3 scripts/summarize-test-coverage.py \
|
||||
"$lcov_report" "$json_report" "$source_path")
|
||||
"$lcov_report" "$json_report" "$source_path") || return 1
|
||||
;;
|
||||
lcov)
|
||||
RUSTC_BOOTSTRAP=1 CARGO_TARGET_DIR="$package_target_dir" cargo llvm-cov \
|
||||
--branch \
|
||||
--lcov \
|
||||
--output-path "$lcov_report" \
|
||||
-p "$package" \
|
||||
"$@"
|
||||
totals=$(awk -v source_path="$source_path" '
|
||||
/^SF:/ { selected = index($0, source_path) != 0 }
|
||||
selected && /^LF:/ { lines_found += substr($0, 4) }
|
||||
@@ -98,42 +94,45 @@ report_package() {
|
||||
END {
|
||||
printf "%d %d %d %d", lines_hit, lines_found, branches_hit, branches_found
|
||||
}
|
||||
' "$lcov_report")
|
||||
' "$lcov_report") || return 1
|
||||
;;
|
||||
esac
|
||||
# Both summarizers emit exactly four space-separated integer counters.
|
||||
# shellcheck disable=SC2086
|
||||
set -- $totals
|
||||
printf '%-26s lines %s/%s, branches %s/%s\n' "$package" "$1" "$2" "$3" "$4"
|
||||
|
||||
case "$package" in
|
||||
kaptan-math | kaptan-navigation | kaptan-control | kaptan-safety)
|
||||
if [ "$1" -ne "$2" ] || [ "$3" -ne "$4" ]; then
|
||||
core_incomplete=1
|
||||
printf '%-30s lines %s/%s, branches %s/%s\n' "$package" "$1" "$2" "$3" "$4"
|
||||
if [ "$report_format" = strict-core ]; then
|
||||
if [ "$2" -eq 0 ] || [ "$4" -eq 0 ] || [ "$1" -ne "$2" ] || [ "$3" -ne "$4" ]; then
|
||||
echo "$package: FAIL strict 100% line/branch gate" >&2
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
elif [ "$2" -eq 0 ]; then
|
||||
echo "$package: no instrumented host source; coverage unavailable"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Host test coverage (reports: $CARGO_TARGET_DIR; strict core unions LCOV DA lines and JSON branches)"
|
||||
report_package kaptan-actuator /libs/kaptan-actuator/
|
||||
report_package kaptan-control /libs/kaptan-control/
|
||||
report_package kaptan-data-channel /libs/kaptan-data-channel/
|
||||
report_package kaptan-dma /libs/kaptan-dma/
|
||||
report_package kaptan-drivers /libs/kaptan-drivers/
|
||||
report_package kaptan-i2c /libs/kaptan-i2c/
|
||||
report_package kaptan-math /libs/kaptan-math/
|
||||
report_package kaptan-mavlink /libs/kaptan-mavlink/ --features sensor-telemetry
|
||||
report_package kaptan-navigation /libs/kaptan-navigation/ --features sitl
|
||||
report_package kaptan-safety /libs/kaptan-safety/
|
||||
report_package kaptan-sensor /libs/kaptan-sensor/
|
||||
report_package kaptan-spi /libs/kaptan-spi/
|
||||
report_package kaptan-time /libs/kaptan-time/
|
||||
report_package kaptan-tx /libs/kaptan-tx/
|
||||
echo "Host test coverage (reports: $coverage_root; strict core unions LCOV DA lines and JSON branches)"
|
||||
echo "UI/compile-fail suites run under Rust 1.90.0 in make test, not under coverage nightly."
|
||||
echo "Proc-macro runtime-harness totals do not establish compiler-phase macro coverage."
|
||||
incomplete=0
|
||||
while read -r package source_path; do
|
||||
set --
|
||||
case "$package" in
|
||||
kaptan-navigation) set -- --features sensor-snapshots ;;
|
||||
kaptan-mavlink) set -- --features sensor-telemetry ;;
|
||||
# Diagnostic snapshots describe the production compiler's output. Keep
|
||||
# those suites in stable make test; instrument the runtime harnesses here.
|
||||
kaptan-config-derive) set -- --lib --test section ;;
|
||||
kaptan-err-reportable-macros) set -- --lib ;;
|
||||
esac
|
||||
if ! report_package "$package" "$source_path" "$@"; then
|
||||
echo "$package: coverage/test failure (see diagnostics above)" >&2
|
||||
incomplete=1
|
||||
fi
|
||||
done <"$coverage_root/packages.txt"
|
||||
|
||||
echo "platform-stm32f7 target-only/N/A (run cargo check -p platform-stm32f7 --lib)"
|
||||
echo "kaptan-geprc-f722-v2 target-only/N/A (run its thumbv7em-none-eabihf check)"
|
||||
echo "kaptan-kakute-f7-mini-v3 target-only/N/A (run its thumbv7em-none-eabihf check)"
|
||||
|
||||
if [ "$core_incomplete" -ne 0 ]; then
|
||||
echo "Strict core coverage is incomplete; reports above identify each crate." >&2
|
||||
echo "kaptan-geprc-f722-v2, kaptan-kakute-f7-mini-v3: target-only; make boards validates release images"
|
||||
if [ "$incomplete" -ne 0 ]; then
|
||||
echo "Coverage gate failed. Strict core gaps and test/report failures are listed above." >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,24 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Summarize strict-core LCOV lines and LLVM JSON branches by source location.
|
||||
|
||||
`llvm-cov` emits one coverage record for every generic monomorphization. Its
|
||||
per-file totals add those records, which can report an uncovered duplicate even
|
||||
when the corresponding source line or branch has been exercised by another
|
||||
monomorphization. This helper instead applies the gate to source locations:
|
||||
Merge generic monomorphizations by source location. A location is covered if
|
||||
any instance executed it:
|
||||
|
||||
* a source line is identified by its LCOV `DA` file and line number;
|
||||
* a branch outcome is identified by its full source span; and
|
||||
* a location is covered when any matching monomorphization executed it.
|
||||
* lines use LCOV `DA` file and line numbers
|
||||
* branch outcomes use their full source spans
|
||||
|
||||
JSON segments are deliberately not used as a line denominator: they mark LLVM
|
||||
instrumentation-region boundaries, not every source line represented by an
|
||||
LCOV `DA` record. Replacing `DA` with segments would silently lower the strict
|
||||
line denominator.
|
||||
JSON segments mark instrumentation boundaries and omit some `DA` lines.
|
||||
Using them would undercount the line denominator.
|
||||
|
||||
Macro-expansion records are deliberately rejected. Their source span denotes
|
||||
the invocation, not necessarily the expanded predicate, so folding them into
|
||||
the source-level key would make the gate unsound. Extend this helper with an
|
||||
explicit expansion identity before adding macro-expanded strict-core coverage.
|
||||
Reject macro expansions: invocation spans cannot distinguish expanded predicates.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -3,11 +3,18 @@
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import contextlib
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
def load_analyzer_module():
|
||||
spec = importlib.util.spec_from_file_location("analyze_fw", "scripts/analyze-fw.py")
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"analyze_fw", Path(__file__).with_name("analyze-fw.py")
|
||||
)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
@@ -179,6 +186,72 @@ class FirmwareMemoryValidationTests(unittest.TestCase):
|
||||
ANALYZER.sram_control_flow_errors(section, literal)[0],
|
||||
)
|
||||
|
||||
def test_cli_requires_and_checks_executable_sram_code(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
elf, memory = root / "firmware", root / "memory.x"
|
||||
elf.touch()
|
||||
memory.write_text(
|
||||
"MEMORY {\nFLASH : ORIGIN = 0x08000000, LENGTH = 256\nRAM : ORIGIN = 0x20000000, LENGTH = 128\n}\n"
|
||||
)
|
||||
segments = [ANALYZER.LoadSegment(0x08000000, 0x08000000, 0x20, 0x20)]
|
||||
for executable in (False, True):
|
||||
with self.subTest(executable=executable):
|
||||
sections = [
|
||||
ANALYZER.Section(".data", 0x10, 0x20000000, True, executable)
|
||||
]
|
||||
with (
|
||||
patch("sys.argv", ["analyze-fw.py", str(elf), str(memory)]),
|
||||
patch.object(ANALYZER, "llvm_tool", return_value="llvm-tool"),
|
||||
patch.object(
|
||||
ANALYZER,
|
||||
"elf_layout",
|
||||
return_value=(sections, segments),
|
||||
),
|
||||
patch.object(
|
||||
ANALYZER, "run", return_value="20000000: bx lr"
|
||||
) as run,
|
||||
contextlib.redirect_stdout(io.StringIO()),
|
||||
):
|
||||
if executable:
|
||||
ANALYZER.main()
|
||||
self.assertEqual(run.call_count, 2)
|
||||
else:
|
||||
with self.assertRaisesRegex(
|
||||
ValueError,
|
||||
"Flash-operation placement validation failed",
|
||||
):
|
||||
ANALYZER.main()
|
||||
|
||||
def test_cli_rejects_common_memory_bounds(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
elf, memory = Path(temporary) / "firmware", Path(temporary) / "memory.x"
|
||||
elf.touch()
|
||||
memory.write_text(
|
||||
"FLASH : ORIGIN = 0x08000000, LENGTH = 256\nRAM : ORIGIN = 0x20000000, LENGTH = 128\n"
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"sys.argv",
|
||||
["analyze-fw.py", str(elf), str(memory)],
|
||||
),
|
||||
patch.object(ANALYZER, "llvm_tool", return_value="llvm-readobj"),
|
||||
patch.object(ANALYZER, "run", return_value=""),
|
||||
patch.object(
|
||||
ANALYZER,
|
||||
"elf_layout",
|
||||
return_value=(
|
||||
[ANALYZER.Section(".data", 0x81, 0x20000000, True, False)],
|
||||
[ANALYZER.LoadSegment(0x20000000, 0x080000FF, 2, 0x81)],
|
||||
),
|
||||
),
|
||||
contextlib.redirect_stdout(io.StringIO()),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
ValueError, "firmware memory validation failed"
|
||||
):
|
||||
ANALYZER.main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise board Makefiles with fake Cargo, image creation, and DFU tools."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
FIRMWARE = Path(__file__).resolve().parents[1]
|
||||
BOARDS = ("geprc-f722-v2", "kakute-f7-mini-v3")
|
||||
TARGET = "thumbv7em-none-eabihf"
|
||||
FAKE_TOOL = """#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
tool = Path(sys.argv[0]).name
|
||||
args = sys.argv[1:]
|
||||
with open(os.environ["WORKFLOW_LOG"], "a") as log:
|
||||
log.write(json.dumps({"tool": tool, "args": args}) + "\\n")
|
||||
if os.environ.get("FAIL_TOOL") == tool:
|
||||
sys.exit(1)
|
||||
if tool == "cargo":
|
||||
if args[0] != "build" or not all(
|
||||
flag in args for flag in ("--release", "--locked", "--offline")
|
||||
):
|
||||
raise ValueError("expected locked offline release build")
|
||||
directory = Path(args[args.index("--target-dir") + 1])
|
||||
directory /= args[args.index("--target") + 1]
|
||||
directory /= "release"
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
for index, arg in enumerate(args):
|
||||
if arg == "--bin":
|
||||
binary = args[index + 1]
|
||||
(directory / binary).write_text("fresh " + binary)
|
||||
elif tool == "create-dfu-image.sh":
|
||||
if len(args) != 1:
|
||||
raise ValueError("expected one ELF")
|
||||
elf = Path(args[0])
|
||||
if elf.read_text() != "fresh " + elf.name:
|
||||
raise ValueError("stale ELF")
|
||||
Path(str(elf) + ".bin").write_text(elf.read_text())
|
||||
elif tool == "dfu-util":
|
||||
image = Path(args[args.index("--download") + 1])
|
||||
if image.read_text() != "fresh " + image.stem:
|
||||
raise ValueError("stale image")
|
||||
else:
|
||||
raise AssertionError(tool)
|
||||
"""
|
||||
|
||||
|
||||
class BoardWorkflowTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temporary = tempfile.TemporaryDirectory(prefix="kaptan-workflow-")
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.root = Path(temporary.name)
|
||||
self.boards = self.root / "firmware" / "boards"
|
||||
self.boards.mkdir(parents=True)
|
||||
for relative in ("Makefile", "board.mk", *(f"{b}/Makefile" for b in BOARDS)):
|
||||
destination = self.boards / relative
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(FIRMWARE / "boards" / relative, destination)
|
||||
for relative in (
|
||||
"bin/cargo",
|
||||
"bin/dfu-util",
|
||||
"firmware/scripts/create-dfu-image.sh",
|
||||
):
|
||||
tool = self.root / relative
|
||||
tool.parent.mkdir(parents=True, exist_ok=True)
|
||||
tool.write_text(FAKE_TOOL)
|
||||
tool.chmod(0o755)
|
||||
self.log = self.root / "events.jsonl"
|
||||
self.environment = dict(os.environ)
|
||||
for name in ("CARGO_TARGET_DIR", "MAKEFLAGS", "MFLAGS", "MAKEOVERRIDES"):
|
||||
self.environment.pop(name, None)
|
||||
self.environment.update(
|
||||
PATH=f"{self.root / 'bin'}{os.pathsep}{os.environ['PATH']}",
|
||||
WORKFLOW_LOG=str(self.log),
|
||||
)
|
||||
# A wrong packaging path must fail even when old build outputs exist.
|
||||
self.seed_stale(self.root / "firmware" / "target")
|
||||
|
||||
def seed_stale(self, target_dir):
|
||||
release = target_dir / TARGET / "release"
|
||||
release.mkdir(parents=True, exist_ok=True)
|
||||
for board in BOARDS:
|
||||
for suffix in ("", ".bin"):
|
||||
(release / f"kaptan-{board}{suffix}").write_text("stale")
|
||||
|
||||
def run_make(self, directory, goal, target_dir=None, fail="", variables=()):
|
||||
self.log.unlink(missing_ok=True)
|
||||
environment = dict(self.environment, FAIL_TOOL=fail)
|
||||
if target_dir is not None:
|
||||
environment["CARGO_TARGET_DIR"] = target_dir
|
||||
result = subprocess.run(
|
||||
[
|
||||
"make",
|
||||
"--no-print-directory",
|
||||
"-j2",
|
||||
"-C",
|
||||
str(directory),
|
||||
goal,
|
||||
*variables,
|
||||
],
|
||||
env=environment,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
events = (
|
||||
[json.loads(line) for line in self.log.read_text().splitlines()]
|
||||
if self.log.exists()
|
||||
else []
|
||||
)
|
||||
return result, events
|
||||
|
||||
def test_selected_board_artifact_directory_and_identity(self):
|
||||
for board in BOARDS:
|
||||
directory = self.boards / board
|
||||
for target_dir in (
|
||||
None,
|
||||
"local artifacts",
|
||||
str(self.root / "external artifacts"),
|
||||
):
|
||||
with self.subTest(board=board, target_dir=target_dir):
|
||||
result, events = self.run_make(
|
||||
directory,
|
||||
"flash",
|
||||
target_dir,
|
||||
variables=(
|
||||
"BOARD=wrong",
|
||||
"PACKAGE=wrong",
|
||||
"EMBEDDED_TARGET=wrong",
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
result.returncode, 0, result.stdout + result.stderr
|
||||
)
|
||||
self.assertEqual(
|
||||
[event["tool"] for event in events],
|
||||
["cargo", "create-dfu-image.sh", "dfu-util"],
|
||||
)
|
||||
args = events[0]["args"]
|
||||
self.assertEqual(args[args.index("-p") + 1], f"kaptan-{board}")
|
||||
self.assertEqual(args[args.index("--bin") + 1], f"kaptan-{board}")
|
||||
self.assertEqual(args[args.index("--target") + 1], TARGET)
|
||||
expected = directory / (target_dir or "../../target")
|
||||
self.assertEqual(
|
||||
(directory / args[args.index("--target-dir") + 1]).resolve(),
|
||||
expected.resolve(),
|
||||
)
|
||||
elf = expected / TARGET / "release" / f"kaptan-{board}"
|
||||
self.assertEqual(
|
||||
(directory / events[1]["args"][0]).resolve(), elf.resolve()
|
||||
)
|
||||
self.assertEqual(
|
||||
(directory / events[2]["args"][-1]).resolve(),
|
||||
Path(str(elf) + ".bin").resolve(),
|
||||
)
|
||||
|
||||
def test_all_boards_use_one_artifact_directory(self):
|
||||
for target_dir in (
|
||||
None,
|
||||
"local artifacts",
|
||||
str(self.root / "external artifacts"),
|
||||
):
|
||||
with self.subTest(target_dir=target_dir):
|
||||
result, events = self.run_make(self.boards, "build", target_dir)
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertEqual(
|
||||
[event["tool"] for event in events],
|
||||
["cargo", "create-dfu-image.sh", "create-dfu-image.sh"],
|
||||
)
|
||||
expected = self.boards / (target_dir or "../target")
|
||||
args = events[0]["args"]
|
||||
self.assertEqual(
|
||||
(self.boards / args[args.index("--target-dir") + 1]).resolve(),
|
||||
expected.resolve(),
|
||||
)
|
||||
self.assertEqual(
|
||||
[
|
||||
(self.boards / event["args"][0]).resolve()
|
||||
for event in events[1:]
|
||||
],
|
||||
[
|
||||
(expected / TARGET / "release" / f"kaptan-{b}").resolve()
|
||||
for b in BOARDS
|
||||
],
|
||||
)
|
||||
|
||||
def test_failures_stop_packaging_and_flashing_even_with_stale_outputs(self):
|
||||
target_dir = self.root / "external artifacts"
|
||||
for directory, goal in (
|
||||
(self.boards, "build"),
|
||||
*((self.boards / board, "flash") for board in BOARDS),
|
||||
):
|
||||
for failed_tool in ("cargo", "create-dfu-image.sh"):
|
||||
with self.subTest(directory=directory, failed_tool=failed_tool):
|
||||
self.seed_stale(target_dir)
|
||||
result, events = self.run_make(
|
||||
directory, goal, str(target_dir), failed_tool
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(
|
||||
[event["tool"] for event in events],
|
||||
["cargo"] if failed_tool == "cargo" else ["cargo", failed_tool],
|
||||
)
|
||||
for board in BOARDS:
|
||||
image = target_dir / TARGET / "release" / f"kaptan-{board}.bin"
|
||||
self.assertEqual(image.read_text(), "stale")
|
||||
|
||||
|
||||
class PackagingValidationTests(unittest.TestCase):
|
||||
def test_real_packager_validates_sram_and_bounds_before_objcopy(self):
|
||||
fake = """#!/usr/bin/env python3
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
name = Path(sys.argv[0]).name
|
||||
if name == "rustc":
|
||||
print("host: test-host" if sys.argv[1] == "-vV" else os.environ["FAKE_SYSROOT"])
|
||||
elif name == "llvm-readobj":
|
||||
print(os.environ["ELF_METADATA"])
|
||||
elif name == "llvm-objdump":
|
||||
print("20000000: bx lr")
|
||||
elif name == "llvm-objcopy":
|
||||
Path(os.environ["OBJCOPY_LOG"]).touch()
|
||||
Path(sys.argv[-1]).write_bytes(b"validated image")
|
||||
else:
|
||||
raise AssertionError(name)
|
||||
"""
|
||||
with tempfile.TemporaryDirectory(prefix="kaptan-package-") as temporary:
|
||||
root = Path(temporary)
|
||||
tools = root / "lib/rustlib/test-host/bin"
|
||||
tools.mkdir(parents=True)
|
||||
for path in (
|
||||
root / "rustc",
|
||||
*(tools / f"llvm-{tool}" for tool in ("readobj", "objdump", "objcopy")),
|
||||
):
|
||||
path.write_text(fake)
|
||||
path.chmod(0o755)
|
||||
elf, memory, log = (
|
||||
root / "arbitrary-name.elf",
|
||||
root / "memory.x",
|
||||
root / "objcopy-called",
|
||||
)
|
||||
elf.touch()
|
||||
image = elf.with_suffix(".bin")
|
||||
memory.write_text(
|
||||
"FLASH : ORIGIN = 0x08000000, LENGTH = 256\nRAM : ORIGIN = 0x20000000, LENGTH = 128\n"
|
||||
)
|
||||
for executable, size, accepted in (
|
||||
(True, 16, True),
|
||||
(False, 16, False),
|
||||
(True, 129, False),
|
||||
):
|
||||
with self.subTest(executable=executable, size=size):
|
||||
image.write_bytes(b"previous image")
|
||||
log.unlink(missing_ok=True)
|
||||
flags = [{"Name": "SHF_ALLOC"}]
|
||||
if executable:
|
||||
flags.append({"Name": "SHF_EXECINSTR"})
|
||||
metadata = [
|
||||
{
|
||||
"Sections": [
|
||||
{
|
||||
"Section": {
|
||||
"Name": {"Name": ".data"},
|
||||
"Size": size,
|
||||
"Address": 0x20000000,
|
||||
"Flags": {"Flags": flags},
|
||||
}
|
||||
}
|
||||
],
|
||||
"ProgramHeaders": [
|
||||
{
|
||||
"ProgramHeader": {
|
||||
"Type": {"Name": "PT_LOAD"},
|
||||
"VirtualAddress": 0x20000000,
|
||||
"PhysicalAddress": 0x08000000,
|
||||
"FileSize": size,
|
||||
"MemSize": size,
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
result = subprocess.run(
|
||||
[
|
||||
str(FIRMWARE / "scripts/create-dfu-image.sh"),
|
||||
str(elf),
|
||||
str(memory),
|
||||
],
|
||||
env=dict(
|
||||
os.environ,
|
||||
PATH=f"{root}{os.pathsep}{os.environ['PATH']}",
|
||||
FAKE_SYSROOT=str(root),
|
||||
ELF_METADATA=json.dumps(metadata),
|
||||
OBJCOPY_LOG=str(log),
|
||||
),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
self.assertEqual(
|
||||
result.returncode == 0, accepted, result.stdout + result.stderr
|
||||
)
|
||||
self.assertEqual(log.exists(), accepted)
|
||||
self.assertEqual(
|
||||
image.read_bytes(),
|
||||
b"validated image" if accepted else b"previous image",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,443 @@
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("verify.py")
|
||||
SPEC = importlib.util.spec_from_file_location("verify", SCRIPT)
|
||||
verify = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(verify)
|
||||
|
||||
|
||||
class VerificationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.directory = Path(self.temporary.name)
|
||||
|
||||
def test_contract_cannot_remove_whole_proof_or_interpreter_gates(self):
|
||||
contract, _ = verify.load_contract()
|
||||
loader = verify.tomllib.load
|
||||
for section, field, value in (
|
||||
(None, "properties", []),
|
||||
("miri", "packages", []),
|
||||
("miri", "packages", ["kaptan-log", "kaptan-log"]),
|
||||
("miri", "seeds", []),
|
||||
("miri", "seeds", [0, 0]),
|
||||
("miri", "seeds", [-1]),
|
||||
("miri", "seeds", [True]),
|
||||
):
|
||||
damaged = copy.deepcopy(contract)
|
||||
(damaged[section] if section else damaged)[field] = value
|
||||
|
||||
def load(source):
|
||||
if Path(source.name).name == "verification.toml":
|
||||
return damaged
|
||||
return loader(source)
|
||||
|
||||
with (
|
||||
self.subTest(field=field, value=value),
|
||||
mock.patch.object(verify.tomllib, "load", side_effect=load),
|
||||
self.assertRaises(ValueError),
|
||||
):
|
||||
verify.load_contract()
|
||||
|
||||
def proof(self):
|
||||
prop = {
|
||||
"id": "example",
|
||||
"harnesses": ["proof"],
|
||||
"covers": [1],
|
||||
"package": "example",
|
||||
"source": "proof.rs",
|
||||
}
|
||||
report = {
|
||||
"metadata": {
|
||||
"version": "1.0",
|
||||
"kani_version": "0.68.0",
|
||||
"target": "x86_64-unknown-linux-gnu",
|
||||
},
|
||||
"tools": {"rustc": "pinned rustc"},
|
||||
"harness_metadata": [
|
||||
{
|
||||
"pretty_name": "proof",
|
||||
"crate_name": "example",
|
||||
"source": {"file": "proof.rs"},
|
||||
}
|
||||
],
|
||||
"verification_results": {
|
||||
"summary": {
|
||||
"total_harnesses": 1,
|
||||
"executed": 1,
|
||||
"status": "completed",
|
||||
"successful": 1,
|
||||
"failed": 0,
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"harness_id": "proof",
|
||||
"status": "Success",
|
||||
"checks": [
|
||||
{
|
||||
"category": "assertion",
|
||||
"status": "Success",
|
||||
"description": "assertion failed: invariant",
|
||||
"function": "proof",
|
||||
},
|
||||
{"category": "cover", "status": "Satisfied"},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
return prop, report
|
||||
|
||||
def validate(self, prop, report, exit_code=0):
|
||||
path = self.directory / "proof.json"
|
||||
path.write_text(json.dumps(report))
|
||||
return verify.validate_kani(
|
||||
path, prop, "0.68.0", "x86_64-unknown-linux-gnu", exit_code, "pinned rustc"
|
||||
)
|
||||
|
||||
def test_failed_missing_vacuous_and_unknown_proofs_cannot_pass(self):
|
||||
prop, report = self.proof()
|
||||
self.assertTrue(self.validate(prop, report)["raw_verifier_passed"])
|
||||
for status in (
|
||||
"Failure",
|
||||
"Undetermined",
|
||||
"SolverError",
|
||||
"Unreachable",
|
||||
"new-status",
|
||||
):
|
||||
damaged = copy.deepcopy(report)
|
||||
damaged["verification_results"]["results"][0]["checks"][0]["status"] = (
|
||||
status
|
||||
)
|
||||
with self.subTest(status=status), self.assertRaises(ValueError):
|
||||
self.validate(prop, damaged)
|
||||
for status in ("Unreachable", "Unsatisfiable", "Undetermined"):
|
||||
damaged = copy.deepcopy(report)
|
||||
damaged["verification_results"]["results"][0]["checks"][1]["status"] = (
|
||||
status
|
||||
)
|
||||
with self.subTest(cover=status), self.assertRaises(ValueError):
|
||||
self.validate(prop, damaged)
|
||||
damaged = copy.deepcopy(report)
|
||||
damaged["verification_results"]["results"][0]["checks"].pop()
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(prop, damaged)
|
||||
damaged["verification_results"]["results"] = []
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(prop, damaged)
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(prop, report, exit_code=1)
|
||||
damaged = copy.deepcopy(report)
|
||||
damaged["harness_metadata"][0]["source"]["file"] = "another.rs"
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(prop, damaged)
|
||||
damaged = copy.deepcopy(report)
|
||||
damaged["tools"]["rustc"] = "different compiler"
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(prop, damaged)
|
||||
|
||||
def test_pid_exception_preserves_raw_failure_and_rejects_other_failures(self):
|
||||
prop, report = self.proof()
|
||||
prop.update(
|
||||
id="control.pid-state",
|
||||
harnesses=["kani_proofs::pid_update_preserves_bounds_or_state"],
|
||||
intermediate_nan_line=196,
|
||||
)
|
||||
result = report["verification_results"]["results"][0]
|
||||
result.update(harness_id=prop["harnesses"][0], status="Failure")
|
||||
report["harness_metadata"][0]["pretty_name"] = prop["harnesses"][0]
|
||||
report["verification_results"]["summary"].update(successful=0, failed=1)
|
||||
diagnostic = {
|
||||
"category": "NaN",
|
||||
"status": "Failure",
|
||||
"description": "NaN on addition",
|
||||
"function": "Pid::update_with_valid_time",
|
||||
"location": {
|
||||
"file": "libs/kaptan-control/src/lib.rs",
|
||||
"line": "196",
|
||||
"column": "13",
|
||||
},
|
||||
}
|
||||
result["checks"].append(diagnostic)
|
||||
accepted = self.validate(prop, report, exit_code=1)
|
||||
self.assertFalse(accepted["raw_verifier_passed"])
|
||||
self.assertTrue(accepted["contract_checks_passed"])
|
||||
for field, value in (
|
||||
("category", "overflow"),
|
||||
("description", "NaN on division"),
|
||||
("function", "different"),
|
||||
):
|
||||
damaged = copy.deepcopy(report)
|
||||
damaged["verification_results"]["results"][0]["checks"][-1][field] = value
|
||||
with self.subTest(field=field), self.assertRaises(ValueError):
|
||||
self.validate(prop, damaged, exit_code=1)
|
||||
result["checks"].append(copy.deepcopy(diagnostic))
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(prop, report, exit_code=1)
|
||||
|
||||
def test_miri_cannot_pass_with_empty_ignored_or_filtered_suites(self):
|
||||
path = self.directory / "miri.log"
|
||||
template = "test result: ok. {} passed; 0 failed; {} ignored; 0 measured; {} filtered out"
|
||||
for passed, ignored, filtered in ((0, 0, 0), (1, 1, 0), (1, 0, 1)):
|
||||
path.write_text(template.format(passed, ignored, filtered))
|
||||
with self.assertRaises(ValueError):
|
||||
verify.validate_miri(path, 1)
|
||||
path.write_text(template.format(4, 0, 0))
|
||||
self.assertEqual(verify.validate_miri(path, 1), 4)
|
||||
with self.assertRaises(ValueError):
|
||||
verify.validate_miri(path, 2)
|
||||
|
||||
def test_snapshot_includes_untracked_bytes_modes_and_deletions(self):
|
||||
path = self.directory / "untracked.rs"
|
||||
path.write_text("original")
|
||||
with mock.patch.object(
|
||||
verify.subprocess,
|
||||
"check_output",
|
||||
return_value=b"untracked.rs\0deleted.rs\0",
|
||||
):
|
||||
before = verify.snapshot(self.directory)
|
||||
path.write_text("edited")
|
||||
self.assertNotEqual(before, verify.snapshot(self.directory))
|
||||
path.write_text("original")
|
||||
path.chmod(0o755)
|
||||
self.assertNotEqual(before, verify.snapshot(self.directory))
|
||||
path.unlink()
|
||||
self.assertTrue(
|
||||
all(
|
||||
entry["kind"] == "absent"
|
||||
for entry in verify.snapshot(self.directory)["entries"]
|
||||
)
|
||||
)
|
||||
|
||||
def test_compiler_and_target_environment_overrides_are_rejected(self):
|
||||
for name in (
|
||||
"RUSTFLAGS",
|
||||
"MIRIFLAGS",
|
||||
"CARGO_BUILD_RUSTFLAGS",
|
||||
"CARGO_BUILD_RUSTC_WRAPPER",
|
||||
"CARGO_ENCODED_RUSTDOCFLAGS",
|
||||
"CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS",
|
||||
"CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS",
|
||||
):
|
||||
with self.subTest(name=name), self.assertRaises(ValueError):
|
||||
verify.reject_overrides({name: "override"})
|
||||
verify.reject_overrides({"PATH": "/bin", "CARGO_TARGET_DIR": "/tmp/build"})
|
||||
|
||||
def test_make_controls_cannot_skip_gate_recipes_or_override_commands(self):
|
||||
(self.directory / "Makefile").write_text(
|
||||
"COMMAND = touch executed\n.PHONY: gate\ngate:\n\t$(COMMAND)\n"
|
||||
)
|
||||
injected = self.directory / "injected.mk"
|
||||
injected.write_text("override COMMAND = true\n")
|
||||
for controls in (
|
||||
{"MAKEFLAGS": "n"},
|
||||
{"MAKEFLAGS": "t"},
|
||||
{"MFLAGS": "-n", "GNUMAKEFLAGS": "-n"},
|
||||
{"MAKEFLAGS": "-- COMMAND=true", "MAKEOVERRIDES": "COMMAND=true"},
|
||||
{"MAKEFILES": str(injected)},
|
||||
):
|
||||
with self.subTest(controls=controls):
|
||||
marker = self.directory / "executed"
|
||||
marker.unlink(missing_ok=True)
|
||||
result = verify.run(
|
||||
["make", "-C", str(self.directory), "gate"],
|
||||
self.directory / "make.log",
|
||||
verify.make_environment(os.environ | controls),
|
||||
5,
|
||||
)
|
||||
self.assertTrue(result["passed"])
|
||||
self.assertTrue(marker.exists())
|
||||
|
||||
def test_optimized_final_link_rejects_invalid_outcomes(self):
|
||||
fixture = (
|
||||
SCRIPT.parents[1] / "libs/kaptan-err-reportable-macros/tests/final_link.py"
|
||||
)
|
||||
harness = """
|
||||
import importlib.util,subprocess,sys
|
||||
spec=importlib.util.spec_from_file_location('final_link',sys.argv[1])
|
||||
fixture=importlib.util.module_from_spec(spec); spec.loader.exec_module(fixture)
|
||||
failure=sys.argv[3]
|
||||
def run(arguments, cwd):
|
||||
linking=arguments[0]=='c++' or arguments[-1]=='error-id-app'
|
||||
duplicate=any('used-duplicate' in arg for arg in arguments)
|
||||
code=int(linking and duplicate and failure=='diagnostic')
|
||||
return subprocess.CompletedProcess(arguments,code,
|
||||
'host: test-host\\n' if arguments[0]=='rustc' else '',
|
||||
'unrelated link failure' if code else '')
|
||||
fixture.run=run
|
||||
sys.argv=[sys.argv[1],'--work-dir',sys.argv[2]]
|
||||
fixture.main()
|
||||
"""
|
||||
for case, diagnostic in (
|
||||
("success", "duplicate unexpectedly linked"),
|
||||
("diagnostic", "unrelated link failure"),
|
||||
):
|
||||
with self.subTest(case=case):
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-O",
|
||||
"-c",
|
||||
harness,
|
||||
str(fixture),
|
||||
str(self.directory / case),
|
||||
case,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(diagnostic, result.stderr)
|
||||
|
||||
def test_source_intake_drift_and_finalization_failures_leave_failed_receipts(self):
|
||||
baseline = {"sha256": "baseline", "entries": []}
|
||||
changed = {"sha256": "changed", "entries": []}
|
||||
for stage, snapshots in (
|
||||
("intake", [ValueError("unavailable source")]),
|
||||
("archive", [baseline, baseline]),
|
||||
("drift", [baseline, baseline, changed, changed]),
|
||||
("finalization", [baseline, baseline, baseline, OSError("final snapshot")]),
|
||||
):
|
||||
output = self.directory / stage
|
||||
result = {
|
||||
"completed": True,
|
||||
"passed": True,
|
||||
"exit_code": 0,
|
||||
"log": "unused",
|
||||
}
|
||||
with (
|
||||
self.subTest(stage=stage),
|
||||
mock.patch.object(
|
||||
sys, "argv", [str(SCRIPT), "host", "--output", str(output)]
|
||||
),
|
||||
mock.patch.object(verify.signal, "signal"),
|
||||
mock.patch.object(verify, "snapshot", side_effect=snapshots),
|
||||
mock.patch.object(verify, "capture", return_value="identity"),
|
||||
mock.patch.object(verify, "run", return_value=result),
|
||||
mock.patch.object(
|
||||
verify.tarfile,
|
||||
"open",
|
||||
wraps=verify.tarfile.open,
|
||||
side_effect=OSError("archive") if stage == "archive" else None,
|
||||
),
|
||||
):
|
||||
self.assertEqual(verify.main(), 1)
|
||||
receipt = json.loads((output / "receipt.json").read_text())
|
||||
self.assertFalse(receipt["passed"])
|
||||
if stage != "archive":
|
||||
self.assertFalse(receipt["source_stable"])
|
||||
|
||||
def test_launch_failure_and_timeout_are_recorded(self):
|
||||
failed = verify.run(
|
||||
["/no-such-verification-program"],
|
||||
self.directory / "missing.log",
|
||||
os.environ,
|
||||
1,
|
||||
)
|
||||
self.assertFalse(failed["completed"])
|
||||
timed = verify.run(
|
||||
[sys.executable, "-c", "import time; time.sleep(30)"],
|
||||
self.directory / "timeout.log",
|
||||
os.environ,
|
||||
0.05,
|
||||
)
|
||||
self.assertFalse(timed["passed"])
|
||||
self.assertFalse(timed["completed"])
|
||||
|
||||
def test_repeated_termination_kills_the_owned_process_group(self):
|
||||
pid_file = self.directory / "child.pid"
|
||||
receipt = self.directory / "result.json"
|
||||
child = (
|
||||
"import os,time,pathlib; pathlib.Path(%r).write_text(str(os.getpid())); time.sleep(30)"
|
||||
% str(pid_file)
|
||||
)
|
||||
harness = """
|
||||
import importlib.util,json,os,pathlib,signal,sys
|
||||
spec=importlib.util.spec_from_file_location('verify',sys.argv[1])
|
||||
v=importlib.util.module_from_spec(spec); spec.loader.exec_module(v)
|
||||
for s in (signal.SIGINT,signal.SIGTERM): signal.signal(s,v.interrupt)
|
||||
r=v.run([sys.executable,'-c',sys.argv[2]],pathlib.Path(sys.argv[3]),os.environ,30)
|
||||
pathlib.Path(sys.argv[4]).write_text(json.dumps(r))
|
||||
"""
|
||||
parent = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
harness,
|
||||
str(SCRIPT),
|
||||
child,
|
||||
str(self.directory / "signal.log"),
|
||||
str(receipt),
|
||||
]
|
||||
)
|
||||
|
||||
def cleanup():
|
||||
if pid_file.exists():
|
||||
try:
|
||||
os.killpg(int(pid_file.read_text()), signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
if parent.poll() is None:
|
||||
parent.kill()
|
||||
parent.wait()
|
||||
|
||||
self.addCleanup(cleanup)
|
||||
deadline = time.monotonic() + 5
|
||||
while not pid_file.exists() and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
self.assertTrue(pid_file.exists())
|
||||
child_pid = int(pid_file.read_text())
|
||||
os.kill(parent.pid, signal.SIGTERM)
|
||||
os.kill(parent.pid, signal.SIGTERM)
|
||||
self.assertEqual(parent.wait(timeout=5), 0)
|
||||
self.assertFalse(json.loads(receipt.read_text())["completed"])
|
||||
with self.assertRaises(ProcessLookupError):
|
||||
os.kill(child_pid, 0)
|
||||
|
||||
def test_cancellation_during_spawn_successful_wait_or_cleanup_cannot_pass(self):
|
||||
for stage in ("spawn", "wait", "cleanup"):
|
||||
process = mock.Mock(pid=999999)
|
||||
|
||||
def wait(**_kwargs):
|
||||
if stage == "wait":
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
return 0
|
||||
|
||||
def spawn(*_args, **_kwargs):
|
||||
if stage == "spawn":
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
return process
|
||||
|
||||
def kill(*_args):
|
||||
if stage == "cleanup":
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
|
||||
process.wait.side_effect = wait
|
||||
with (
|
||||
self.subTest(stage=stage),
|
||||
mock.patch.object(verify.subprocess, "Popen", side_effect=spawn),
|
||||
mock.patch.object(verify.os, "killpg", side_effect=kill),
|
||||
):
|
||||
result = verify.run(
|
||||
["fake-child"], self.directory / f"{stage}.log", os.environ, 1
|
||||
)
|
||||
self.assertFalse(result["passed"])
|
||||
self.assertFalse(result["completed"])
|
||||
process.wait.assert_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,710 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the fixed verification gates and retain source-bound evidence."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import time
|
||||
import tomllib
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
FIRMWARE = ROOT / "firmware"
|
||||
GATES = ("host", "targets", "boards", "coverage", "miri", "kani")
|
||||
|
||||
|
||||
class Interrupted(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def interrupt(signum, _frame):
|
||||
raise Interrupted(f"Received signal {signum}")
|
||||
|
||||
|
||||
def reject_overrides(environment):
|
||||
fixed = {
|
||||
"RUSTFLAGS",
|
||||
"CARGO_ENCODED_RUSTFLAGS",
|
||||
"RUSTC",
|
||||
"RUSTDOC",
|
||||
"RUSTDOCFLAGS",
|
||||
"CARGO_ENCODED_RUSTDOCFLAGS",
|
||||
"RUSTC_WRAPPER",
|
||||
"RUSTC_WORKSPACE_WRAPPER",
|
||||
"RUSTC_BOOTSTRAP",
|
||||
"MIRIFLAGS",
|
||||
"MIRI_SYSROOT",
|
||||
"MIRI_LIB_SRC",
|
||||
"MIRI_NO_STD",
|
||||
"KANIFLAGS",
|
||||
"CBMC",
|
||||
}
|
||||
rejected = [
|
||||
key
|
||||
for key, value in environment.items()
|
||||
if value
|
||||
and (
|
||||
key in fixed
|
||||
or (
|
||||
key.startswith("CARGO_BUILD_")
|
||||
and key not in ("CARGO_BUILD_JOBS", "CARGO_BUILD_TARGET_DIR")
|
||||
)
|
||||
or key.startswith("CARGO_PROFILE_")
|
||||
or (key.startswith("CARGO_TARGET_") and key != "CARGO_TARGET_DIR")
|
||||
)
|
||||
]
|
||||
if rejected:
|
||||
raise ValueError(
|
||||
f"Remove verification overrides: {', '.join(sorted(rejected))}"
|
||||
)
|
||||
|
||||
|
||||
def make_environment(environment):
|
||||
# A wrapper's dry-run, touch or command-line overrides must not suppress gates.
|
||||
controls = {"MAKEFLAGS", "MFLAGS", "MAKEOVERRIDES", "GNUMAKEFLAGS", "MAKEFILES"}
|
||||
return {key: value for key, value in environment.items() if key not in controls}
|
||||
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def capture(argv, cwd=FIRMWARE, env=None):
|
||||
return subprocess.check_output(argv, cwd=cwd, env=env, text=True).strip()
|
||||
|
||||
|
||||
def snapshot(root=ROOT):
|
||||
names = subprocess.check_output(
|
||||
["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"],
|
||||
cwd=root,
|
||||
).split(b"\0")
|
||||
entries = []
|
||||
for name in sorted({os.fsdecode(name) for name in names if name}):
|
||||
path = root / name
|
||||
if path.is_symlink():
|
||||
raise ValueError(
|
||||
f"Source symlink requires an explicit input boundary: {name}"
|
||||
)
|
||||
entry = {"path": name, "kind": "absent"}
|
||||
if path.is_file():
|
||||
entry.update(
|
||||
kind="file", sha256=sha256(path), mode=path.stat().st_mode & 0o777
|
||||
)
|
||||
elif path.exists():
|
||||
raise ValueError(f"Unsupported source entry: {name}")
|
||||
entries.append(entry)
|
||||
encoded = json.dumps(entries, sort_keys=True, separators=(",", ":")).encode()
|
||||
return {"sha256": hashlib.sha256(encoded).hexdigest(), "entries": entries}
|
||||
|
||||
|
||||
def load_contract():
|
||||
with (FIRMWARE / "verification.toml").open("rb") as source:
|
||||
contract = tomllib.load(source)
|
||||
if contract["format"] != 1:
|
||||
raise ValueError("Unsupported verification contract format")
|
||||
with (FIRMWARE / "Cargo.toml").open("rb") as source:
|
||||
workspace = tomllib.load(source)["workspace"]
|
||||
packages = {}
|
||||
for member in workspace["members"]:
|
||||
with (FIRMWARE / member / "Cargo.toml").open("rb") as source:
|
||||
name = tomllib.load(source)["package"]["name"]
|
||||
packages[name] = member
|
||||
declared = [name for area in contract["areas"] for name in area["packages"]]
|
||||
if len(declared) != len(set(declared)) or set(declared) != set(packages):
|
||||
raise ValueError(
|
||||
"Every workspace package must have exactly one verification area"
|
||||
)
|
||||
property_ids = [prop["id"] for prop in contract["properties"]]
|
||||
if not property_ids or len(property_ids) != len(set(property_ids)):
|
||||
raise ValueError("Empty or duplicate property inventory")
|
||||
for prop in contract["properties"]:
|
||||
if (
|
||||
prop["package"] not in packages
|
||||
or not prop["harnesses"]
|
||||
or len(prop["harnesses"]) != len(set(prop["harnesses"]))
|
||||
or len(prop["covers"]) != len(prop["harnesses"])
|
||||
or any(count <= 0 for count in prop["covers"])
|
||||
or not (FIRMWARE / prop["source"]).is_file()
|
||||
):
|
||||
raise ValueError(
|
||||
f"Invalid proof owner or empty harness inventory: {prop['id']}"
|
||||
)
|
||||
miri = contract["miri"]
|
||||
if (
|
||||
not miri["packages"]
|
||||
or len(miri["packages"]) != len(set(miri["packages"]))
|
||||
or not set(miri["packages"]) <= set(packages)
|
||||
or not miri["seeds"]
|
||||
or len(miri["seeds"]) != len(set(miri["seeds"]))
|
||||
or any(type(seed) is not int or not 0 <= seed < 2**64 for seed in miri["seeds"])
|
||||
):
|
||||
raise ValueError(
|
||||
"Miri requires known distinct packages and nonempty distinct u64 seeds"
|
||||
)
|
||||
return contract, packages
|
||||
|
||||
|
||||
def reviewed_unreachable(harness, check):
|
||||
addition = "attempt to add with overflow"
|
||||
shift = "attempt to shift left with overflow"
|
||||
reviewed = {
|
||||
(
|
||||
"kani_proofs::clock_regression_requires_successful_inhibited_reset",
|
||||
"kani_proofs::clock_regression_requires_successful_inhibited_reset::{closure#0}",
|
||||
addition,
|
||||
),
|
||||
(
|
||||
"kani_proofs::failed_commit_preserves_error_and_latches_inhibition",
|
||||
"kani_proofs::failed_commit_preserves_error_and_latches_inhibition::{closure#1}",
|
||||
addition,
|
||||
),
|
||||
(
|
||||
"kani_proofs::dshot_symbols_round_trip_and_end_low",
|
||||
"core::fmt::Arguments::<'_>::from_str",
|
||||
shift,
|
||||
),
|
||||
(
|
||||
"kani_proofs::pid_constructor_establishes_invariant",
|
||||
"core::fmt::Arguments::<'_>::from_str",
|
||||
shift,
|
||||
),
|
||||
}
|
||||
return (
|
||||
check["category"] == "assertion"
|
||||
and (harness, check["function"], check["description"]) in reviewed
|
||||
)
|
||||
|
||||
|
||||
def pid_rejected_nan(prop, harness, check):
|
||||
return (
|
||||
prop["id"] == "control.pid-state"
|
||||
and harness == "kani_proofs::pid_update_preserves_bounds_or_state"
|
||||
and check["category"] == "NaN"
|
||||
and check["description"] == "NaN on addition"
|
||||
and check["function"] == "Pid::update_with_valid_time"
|
||||
and check["location"]
|
||||
== {
|
||||
"file": "libs/kaptan-control/src/lib.rs",
|
||||
"line": str(prop["intermediate_nan_line"]),
|
||||
"column": "13",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def validate_kani(path, prop, version, target, exit_code, rustc):
|
||||
report = json.loads(path.read_text())
|
||||
expected = prop["harnesses"]
|
||||
expected_covers = dict(zip(expected, prop["covers"], strict=True))
|
||||
qualified = prop["id"] == "control.pid-state"
|
||||
failures = int(qualified)
|
||||
metadata = report["metadata"]
|
||||
if (
|
||||
metadata["version"] != "1.0"
|
||||
or metadata["kani_version"] != version
|
||||
or metadata["target"] != target
|
||||
or report["tools"]["rustc"] != rustc
|
||||
):
|
||||
raise ValueError(
|
||||
"Kani version or target differs from the verification contract"
|
||||
)
|
||||
owner = prop["package"].replace("-", "_")
|
||||
harness_metadata = report["harness_metadata"]
|
||||
if (
|
||||
len(harness_metadata) != len(expected)
|
||||
or {item["pretty_name"] for item in harness_metadata} != set(expected)
|
||||
or any(
|
||||
item["crate_name"] != owner or item["source"]["file"] != prop["source"]
|
||||
for item in harness_metadata
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Kani harness ownership differs from the verification contract"
|
||||
)
|
||||
results = report["verification_results"]["results"]
|
||||
names = [result["harness_id"] for result in results]
|
||||
summary = report["verification_results"]["summary"]
|
||||
if (
|
||||
not expected
|
||||
or len(names) != len(set(names))
|
||||
or set(names) != set(expected)
|
||||
or summary["status"] != "completed"
|
||||
or summary["total_harnesses"] != len(expected)
|
||||
or summary["executed"] != len(expected)
|
||||
or summary["successful"] != len(expected) - failures
|
||||
or summary["failed"] != failures
|
||||
or exit_code != failures
|
||||
):
|
||||
raise ValueError("Kani did not complete the exact required harness inventory")
|
||||
accepted = []
|
||||
unreachable = []
|
||||
for result in results:
|
||||
is_pid_update = (
|
||||
qualified
|
||||
and result["harness_id"]
|
||||
== "kani_proofs::pid_update_preserves_bounds_or_state"
|
||||
)
|
||||
if result["status"] != ("Failure" if is_pid_update else "Success"):
|
||||
raise ValueError(f"Kani harness failed: {result['harness_id']}")
|
||||
checks = result["checks"]
|
||||
covers = [check for check in checks if check["category"] == "cover"]
|
||||
if len(covers) != expected_covers[result["harness_id"]] or any(
|
||||
check["status"] != "Satisfied" for check in covers
|
||||
):
|
||||
raise ValueError(
|
||||
f"Missing or unsatisfied reachability evidence: {result['harness_id']}"
|
||||
)
|
||||
assertions = [check for check in checks if check["category"] != "cover"]
|
||||
if not assertions:
|
||||
raise ValueError(
|
||||
"A proof needs assertions as well as reachability witnesses"
|
||||
)
|
||||
for check in assertions:
|
||||
if check["status"] == "Success":
|
||||
continue
|
||||
if check["status"] == "Unreachable" and reviewed_unreachable(
|
||||
result["harness_id"], check
|
||||
):
|
||||
unreachable.append(check)
|
||||
elif check["status"] == "Failure" and pid_rejected_nan(
|
||||
prop, result["harness_id"], check
|
||||
):
|
||||
accepted.append(check)
|
||||
else:
|
||||
raise ValueError(f"Unproved assertion: {check}")
|
||||
if len(accepted) != failures:
|
||||
raise ValueError("The reviewed intermediate-NaN diagnostic changed")
|
||||
return {
|
||||
"raw_summary": summary,
|
||||
"raw_verifier_passed": exit_code == 0,
|
||||
"contract_checks_passed": True,
|
||||
"expected_intermediate_nan": accepted,
|
||||
"reviewed_unreachable": unreachable,
|
||||
"tools": report["tools"],
|
||||
}
|
||||
|
||||
|
||||
def validate_miri(log, package_count):
|
||||
results = re.findall(
|
||||
r"test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored; "
|
||||
r"\d+ measured; (\d+) filtered out",
|
||||
log.read_text(),
|
||||
)
|
||||
if len(results) != package_count or any(
|
||||
int(passed) == 0 or int(failed) or int(ignored) or int(filtered)
|
||||
for passed, failed, ignored, filtered in results
|
||||
):
|
||||
raise ValueError(
|
||||
"Miri must execute every selected library suite without ignored tests"
|
||||
)
|
||||
return sum(int(result[0]) for result in results)
|
||||
|
||||
|
||||
def run(argv, log, env, timeout):
|
||||
started = time.time()
|
||||
monotonic_start = time.monotonic()
|
||||
result = {"argv": argv, "cwd": str(FIRMWARE), "started_unix": started}
|
||||
process = None
|
||||
pending = []
|
||||
handlers = {}
|
||||
|
||||
def defer(signum, _frame):
|
||||
pending[:] = [signum]
|
||||
|
||||
with log.open("wb") as output:
|
||||
try:
|
||||
# Keep cancellation deferred until the owned process group is reaped.
|
||||
# Children inherit neither a blocked mask nor Python signal handlers.
|
||||
for signum in (signal.SIGINT, signal.SIGTERM):
|
||||
handlers[signum] = signal.signal(signum, defer)
|
||||
process = subprocess.Popen(
|
||||
argv,
|
||||
cwd=FIRMWARE,
|
||||
env=env,
|
||||
stdout=output,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
)
|
||||
while True:
|
||||
if pending:
|
||||
raise Interrupted(f"Received signal {pending[0]}")
|
||||
remaining = timeout - (time.monotonic() - monotonic_start)
|
||||
if remaining <= 0:
|
||||
raise subprocess.TimeoutExpired(argv, timeout)
|
||||
try:
|
||||
result["exit_code"] = process.wait(timeout=min(remaining, 0.1))
|
||||
result["completed"] = True
|
||||
break
|
||||
except subprocess.TimeoutExpired:
|
||||
continue
|
||||
except (
|
||||
subprocess.TimeoutExpired,
|
||||
Interrupted,
|
||||
KeyboardInterrupt,
|
||||
OSError,
|
||||
) as error:
|
||||
result.update(completed=False, error=str(error), exit_code=None)
|
||||
finally:
|
||||
if process is not None:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
process.wait()
|
||||
for signum, handler in handlers.items():
|
||||
signal.signal(signum, handler)
|
||||
if pending:
|
||||
result.update(
|
||||
completed=False, error=f"Received signal {pending[0]}", exit_code=None
|
||||
)
|
||||
result.update(
|
||||
seconds=time.monotonic() - monotonic_start, log=str(log), log_sha256=sha256(log)
|
||||
)
|
||||
result["passed"] = result["completed"] and result["exit_code"] == 0
|
||||
return result
|
||||
|
||||
|
||||
def tool_paths(contract):
|
||||
install = (
|
||||
FIRMWARE / "target/verification-tools" / f"kani-{contract['kani']['version']}"
|
||||
)
|
||||
return install, install / "home"
|
||||
|
||||
|
||||
def setup(contract):
|
||||
install, home = tool_paths(contract)
|
||||
env = os.environ | {"CARGO_BUILD_JOBS": "2", "KANI_HOME": str(home)}
|
||||
commands = [
|
||||
[
|
||||
"rustup",
|
||||
"toolchain",
|
||||
"install",
|
||||
contract["miri"]["toolchain"],
|
||||
"--profile",
|
||||
"minimal",
|
||||
"--component",
|
||||
"miri,rust-src",
|
||||
],
|
||||
["cargo", f"+{contract['miri']['toolchain']}", "miri", "setup"],
|
||||
[
|
||||
"cargo",
|
||||
"+1.90.0",
|
||||
"install",
|
||||
"kani-verifier",
|
||||
"--locked",
|
||||
"--version",
|
||||
contract["kani"]["version"],
|
||||
"--root",
|
||||
str(install),
|
||||
],
|
||||
[str(install / "bin/cargo-kani"), "kani", "setup"],
|
||||
]
|
||||
logs = FIRMWARE / "target/verification-tools" / f"setup-{time.time_ns()}"
|
||||
logs.mkdir(parents=True)
|
||||
for index, command in enumerate(commands):
|
||||
result = run(command, logs / f"{index}.log", env, 3600)
|
||||
if not result["passed"]:
|
||||
raise ValueError(f"Tool setup failed: {result}")
|
||||
print(f"Tool setup logs: {logs}")
|
||||
|
||||
|
||||
def main():
|
||||
for signum in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(signum, interrupt)
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("gate", choices=(*GATES, "all", "setup", "inventory"))
|
||||
parser.add_argument(
|
||||
"--output", type=Path, help="New evidence directory; must not exist"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.gate in ("setup", "inventory"):
|
||||
contract, packages = load_contract()
|
||||
if args.gate == "setup":
|
||||
setup(contract)
|
||||
else:
|
||||
print(json.dumps({"packages": packages, "contract": contract}, indent=2))
|
||||
return 0
|
||||
selected = GATES if args.gate == "all" else (args.gate,)
|
||||
output = (
|
||||
args.output or FIRMWARE / "target/verification" / str(time.time_ns())
|
||||
).resolve()
|
||||
if output.is_relative_to(ROOT) and not output.is_relative_to(FIRMWARE / "target"):
|
||||
raise ValueError("Evidence inside the repository must be under firmware/target")
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
baseline = None
|
||||
receipt = {
|
||||
"format": 1,
|
||||
"completed": False,
|
||||
"passed": False,
|
||||
"source_stable": False,
|
||||
"source": None,
|
||||
"host": platform.platform(),
|
||||
"python": sys.version,
|
||||
"gates": [],
|
||||
}
|
||||
build = FIRMWARE / "target/verification-build"
|
||||
env = make_environment(os.environ) | {
|
||||
"CARGO_BUILD_JOBS": "2",
|
||||
"RUSTUP_TOOLCHAIN": "1.90.0",
|
||||
"CARGO_TARGET_DIR": str(build / "stable"),
|
||||
"CARGO_NET_OFFLINE": "true",
|
||||
}
|
||||
try:
|
||||
reject_overrides(os.environ)
|
||||
baseline = snapshot()
|
||||
receipt["source"] = baseline
|
||||
contract, packages = load_contract()
|
||||
receipt.update(
|
||||
contract=contract,
|
||||
packages=packages,
|
||||
head=capture(["git", "rev-parse", "HEAD"]),
|
||||
git_status=capture(
|
||||
["git", "status", "--porcelain=v1", "--untracked-files=all"]
|
||||
),
|
||||
)
|
||||
archive = output / "source.tar.gz"
|
||||
with tarfile.open(archive, "w:gz") as source:
|
||||
for entry in baseline["entries"]:
|
||||
if entry["kind"] == "file":
|
||||
source.add(
|
||||
ROOT / entry["path"], arcname=entry["path"], recursive=False
|
||||
)
|
||||
receipt["source_archive_sha256"] = sha256(archive)
|
||||
install, kani_home = tool_paths(contract)
|
||||
receipt["rustc"] = capture(["rustc", "+1.90.0", "-vV"], env=env)
|
||||
receipt["cargo"] = capture(["cargo", "+1.90.0", "-V"], env=env)
|
||||
for gate in selected:
|
||||
if snapshot() != baseline:
|
||||
raise ValueError("Source changed before gate execution")
|
||||
commands = []
|
||||
gate_env = dict(env)
|
||||
if gate in ("host", "targets", "boards", "coverage"):
|
||||
target = {
|
||||
"host": "pre-commit",
|
||||
"targets": "target-checks",
|
||||
"boards": "boards",
|
||||
"coverage": "coverage",
|
||||
}[gate]
|
||||
directory = FIRMWARE if gate == "targets" else ROOT
|
||||
if gate == "coverage":
|
||||
gate_env["CARGO_TARGET_DIR"] = str(build / "coverage")
|
||||
commands.append(
|
||||
(gate, ["make", "-C", str(directory), target], gate_env, None)
|
||||
)
|
||||
elif gate == "miri":
|
||||
miri = contract["miri"]
|
||||
receipt["miri"] = capture(
|
||||
["cargo", f"+{miri['toolchain']}", "miri", "--version"], env=env
|
||||
)
|
||||
receipt["miri_rustc"] = capture(
|
||||
["rustc", f"+{miri['toolchain']}", "-vV"], env=env
|
||||
)
|
||||
sysroot = Path(
|
||||
capture(
|
||||
["rustc", f"+{miri['toolchain']}", "--print", "sysroot"],
|
||||
env=env,
|
||||
)
|
||||
)
|
||||
receipt["miri_binary_sha256"] = sha256(sysroot / "bin/miri")
|
||||
gate_env["CARGO_TARGET_DIR"] = str(build / "miri")
|
||||
for seed in miri["seeds"]:
|
||||
seeded = gate_env | {
|
||||
"MIRIFLAGS": " ".join(miri["flags"] + [f"-Zmiri-seed={seed}"])
|
||||
}
|
||||
command = [
|
||||
"cargo",
|
||||
f"+{miri['toolchain']}",
|
||||
"miri",
|
||||
"test",
|
||||
"--locked",
|
||||
"--offline",
|
||||
"--target",
|
||||
miri["target"],
|
||||
"--lib",
|
||||
]
|
||||
for package in miri["packages"]:
|
||||
command += ["-p", package]
|
||||
commands.append((f"miri-{seed}", command, seeded, None))
|
||||
else:
|
||||
kani = contract["kani"]
|
||||
gate_env["KANI_HOME"] = str(kani_home)
|
||||
binary = str(install / "bin/cargo-kani")
|
||||
receipt["kani"] = capture(
|
||||
[binary, "kani", "--version", "--verbose"], env=gate_env
|
||||
)
|
||||
receipt["kani_launcher_sha256"] = sha256(Path(binary))
|
||||
bundle = kani_home / f"kani-{kani['version']}"
|
||||
receipt["kani_toolchain"] = (
|
||||
(bundle / "rust-toolchain-version").read_text().strip()
|
||||
)
|
||||
receipt["kani_rustc"] = (bundle / "rustc-version").read_text().strip()
|
||||
if receipt["kani_toolchain"] != f"{kani['toolchain']}-{kani['target']}":
|
||||
raise ValueError(
|
||||
"Kani bundled compiler differs from the pinned toolchain"
|
||||
)
|
||||
receipt["kani_binary_hashes"] = {
|
||||
name: sha256(bundle / "bin" / name)
|
||||
for name in ("kani-compiler", "cbmc", "goto-instrument")
|
||||
}
|
||||
for prop in contract["properties"]:
|
||||
export = output / f"{prop['id']}.json"
|
||||
command = [
|
||||
binary,
|
||||
"kani",
|
||||
"--lib",
|
||||
"-p",
|
||||
prop["package"],
|
||||
"--target-dir",
|
||||
str(build / "kani"),
|
||||
"--jobs",
|
||||
"1",
|
||||
"--solver",
|
||||
kani["solver"],
|
||||
"-Z",
|
||||
"unstable-options",
|
||||
"--harness-timeout",
|
||||
str(kani["harness_timeout_seconds"]),
|
||||
"--export-json",
|
||||
str(export),
|
||||
]
|
||||
commands.append((prop["id"], command, gate_env, prop))
|
||||
for name, command, command_env, prop in commands:
|
||||
print(f"Verifying {name}; evidence: {output}", flush=True)
|
||||
result = run(command, output / f"{name}.log", command_env, 3600)
|
||||
result.update(
|
||||
gate=gate,
|
||||
name=name,
|
||||
environment={
|
||||
key: value
|
||||
for key, value in command_env.items()
|
||||
if env.get(key) != value
|
||||
or key
|
||||
in (
|
||||
"CARGO_BUILD_JOBS",
|
||||
"CARGO_TARGET_DIR",
|
||||
"RUSTUP_TOOLCHAIN",
|
||||
"CARGO_NET_OFFLINE",
|
||||
)
|
||||
},
|
||||
)
|
||||
receipt["gates"].append(result)
|
||||
if result["completed"] and (result["passed"] or gate == "kani"):
|
||||
try:
|
||||
if gate == "miri":
|
||||
result["tests_passed"] = validate_miri(
|
||||
Path(result["log"]), len(contract["miri"]["packages"])
|
||||
)
|
||||
elif gate == "kani":
|
||||
export = output / f"{prop['id']}.json"
|
||||
result["proof_summary"] = validate_kani(
|
||||
export,
|
||||
prop,
|
||||
kani["version"],
|
||||
kani["target"],
|
||||
result["exit_code"],
|
||||
receipt["kani_rustc"],
|
||||
)
|
||||
result["export_sha256"] = sha256(export)
|
||||
result["passed"] = True
|
||||
if not result["proof_summary"]["raw_verifier_passed"]:
|
||||
print(
|
||||
"PID contract checks passed; expected intermediate-NaN diagnostic. Raw Kani verdict remains Failure.",
|
||||
flush=True,
|
||||
)
|
||||
elif gate == "boards":
|
||||
artifacts = []
|
||||
for board in ("geprc-f722-v2", "kakute-f7-mini-v3"):
|
||||
for suffix in ("", ".bin"):
|
||||
path = (
|
||||
build
|
||||
/ "stable/thumbv7em-none-eabihf/release"
|
||||
/ f"kaptan-{board}{suffix}"
|
||||
)
|
||||
retained = output / "artifacts" / path.name
|
||||
retained.parent.mkdir(exist_ok=True)
|
||||
shutil.copy2(path, retained)
|
||||
artifacts.append(
|
||||
{
|
||||
"path": str(retained),
|
||||
"sha256": sha256(retained),
|
||||
}
|
||||
)
|
||||
result["artifacts"] = artifacts
|
||||
elif gate == "coverage":
|
||||
reports = build / "coverage"
|
||||
names = ["metadata.json", "packages.txt"]
|
||||
names += [
|
||||
f"{line.split()[0]}.lcov"
|
||||
for line in (reports / "packages.txt")
|
||||
.read_text()
|
||||
.splitlines()
|
||||
]
|
||||
names += [
|
||||
f"{package}.json"
|
||||
for package in (
|
||||
"kaptan-math",
|
||||
"kaptan-navigation",
|
||||
"kaptan-control",
|
||||
"kaptan-safety",
|
||||
)
|
||||
]
|
||||
retained = output / "coverage"
|
||||
retained.mkdir()
|
||||
result["reports"] = []
|
||||
for name in names:
|
||||
destination = retained / name
|
||||
shutil.copy2(reports / name, destination)
|
||||
result["reports"].append(
|
||||
{
|
||||
"path": str(destination),
|
||||
"sha256": sha256(destination),
|
||||
}
|
||||
)
|
||||
except (ValueError, KeyError, OSError) as error:
|
||||
result.update(passed=False, validation_error=str(error))
|
||||
if snapshot() != baseline:
|
||||
raise ValueError("Source changed during gate execution")
|
||||
(output / "receipt.json").write_text(
|
||||
json.dumps(receipt, indent=2) + "\n"
|
||||
)
|
||||
if not result["passed"]:
|
||||
raise ValueError(f"Gate failed: {name}; see {result['log']}")
|
||||
receipt["completed"] = True
|
||||
receipt["passed"] = True
|
||||
except (
|
||||
ValueError,
|
||||
OSError,
|
||||
subprocess.CalledProcessError,
|
||||
Interrupted,
|
||||
KeyboardInterrupt,
|
||||
) as error:
|
||||
receipt["error"] = str(error)
|
||||
print(error, file=sys.stderr)
|
||||
finally:
|
||||
# Finish the failure receipt even if cancellation arrives a second time.
|
||||
for signum in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(signum, signal.SIG_IGN)
|
||||
try:
|
||||
receipt["source_stable"] = baseline is not None and snapshot() == baseline
|
||||
except (ValueError, OSError, subprocess.CalledProcessError) as error:
|
||||
receipt.update(source_stable=False, finalization_error=str(error))
|
||||
receipt["passed"] = receipt["passed"] and receipt["source_stable"]
|
||||
(output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n")
|
||||
print(f"Verification receipt: {output / 'receipt.json'}")
|
||||
return 0 if receipt["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user