From f30ad24d8a32159ccc8c64d182df7d8de6f5cf0a Mon Sep 17 00:00:00 2001 From: Mike Kell Date: Fri, 22 May 2026 10:22:05 -0400 Subject: [PATCH] feat(ci): add test coverage visibility to CI pipeline (Stage 4D) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enhance flutter-test.yml to run tests with --coverage and parse lcov.info files, producing aggregate summary table with per-package line coverage. Changes: - flutter-test.yml: add --coverage flag, lcov.info parsing, coverage % - collect_coverage.sh: new local coverage helper with summary table - tools/README.md: document collect_coverage.sh script - .gitignore: add coverage/ directories - master_development_brief.md: mark Stage 4D complete, document baseline coverage table, update next branch to Stage 5A, resolve improvement #5 Baseline coverage (2026-05-22): - core: 85.7%% (42/49 lines, 20 tests) - design_system: 100.0%% (88/88 lines, 41 tests) - feature_wordpress: 84.7%% (857/1012 lines, 294 tests) - kell_web: 54.1%% (191/353 lines, 24 tests) - Overall: 78.4%% (1178/1502 lines, 379 tests) No minimum thresholds enforced — visibility first. --- .forgejo/workflows/flutter-test.yml | 71 ++++++--- .gitignore | 3 + docs/development/master_development_brief.md | 49 +++--- kell_creations_apps/tools/README.md | 39 +++++ kell_creations_apps/tools/collect_coverage.sh | 149 ++++++++++++++++++ 5 files changed, 265 insertions(+), 46 deletions(-) create mode 100644 kell_creations_apps/tools/collect_coverage.sh diff --git a/.forgejo/workflows/flutter-test.yml b/.forgejo/workflows/flutter-test.yml index ab31137..8c79b19 100644 --- a/.forgejo/workflows/flutter-test.yml +++ b/.forgejo/workflows/flutter-test.yml @@ -36,10 +36,10 @@ jobs: - name: Install dependencies — kell_web run: cd apps/kell_web && flutter pub get - - name: Test — core + - name: Test with coverage — core run: | cd packages/core - flutter test --reporter expanded 2>&1 | tee test_output.txt + flutter test --coverage --reporter expanded 2>&1 | tee test_output.txt echo "" echo "=== core test summary ===" TOTAL=$(grep -cE '^\s*✓' test_output.txt || echo "0") @@ -49,10 +49,10 @@ jobs: # Fail the step if any tests failed if [ "$FAILED" -gt 0 ]; then exit 1; fi - - name: Test — design_system + - name: Test with coverage — design_system run: | cd packages/design_system - flutter test --reporter expanded 2>&1 | tee test_output.txt + flutter test --coverage --reporter expanded 2>&1 | tee test_output.txt echo "" echo "=== design_system test summary ===" TOTAL=$(grep -cE '^\s*✓' test_output.txt || echo "0") @@ -61,10 +61,10 @@ jobs: echo " Failed: $FAILED" if [ "$FAILED" -gt 0 ]; then exit 1; fi - - name: Test — feature_wordpress + - name: Test with coverage — feature_wordpress run: | cd packages/feature_wordpress - flutter test --reporter expanded 2>&1 | tee test_output.txt + flutter test --coverage --reporter expanded 2>&1 | tee test_output.txt echo "" echo "=== feature_wordpress test summary ===" TOTAL=$(grep -cE '^\s*✓' test_output.txt || echo "0") @@ -73,10 +73,10 @@ jobs: echo " Failed: $FAILED" if [ "$FAILED" -gt 0 ]; then exit 1; fi - - name: Test — kell_web + - name: Test with coverage — kell_web run: | cd apps/kell_web - flutter test --reporter expanded 2>&1 | tee test_output.txt + flutter test --coverage --reporter expanded 2>&1 | tee test_output.txt echo "" echo "=== kell_web test summary ===" TOTAL=$(grep -cE '^\s*✓' test_output.txt || echo "0") @@ -85,22 +85,27 @@ jobs: echo " Failed: $FAILED" if [ "$FAILED" -gt 0 ]; then exit 1; fi - - name: Aggregate test report + - name: Aggregate test and coverage report if: always() run: | echo "" - echo "╔══════════════════════════════════════╗" - echo "║ Flutter Test Results Summary ║" - echo "╠══════════════════════════════════════╣" - echo "║ Package Pass Fail ║" - echo "╠══════════════════════════════════════╣" + echo "╔═══════════════════════════════════════════════════════════╗" + echo "║ Flutter Test & Coverage Summary ║" + echo "╠═══════════════════════════════════════════════════════════╣" + echo "║ Package Pass Fail Lines Coverage ║" + echo "╠═══════════════════════════════════════════════════════════╣" TOTAL_PASS=0 TOTAL_FAIL=0 + TOTAL_HIT=0 + TOTAL_FOUND=0 for pkg in packages/core packages/design_system packages/feature_wordpress apps/kell_web; do NAME=$(basename "$pkg") OUTPUT="$pkg/test_output.txt" + LCOV="$pkg/coverage/lcov.info" + + # Test counts if [ -f "$OUTPUT" ]; then PASS=$(grep -cE '^\s*✓' "$OUTPUT" || echo "0") FAIL=$(grep -cE '^\s*✗' "$OUTPUT" || echo "0") @@ -108,14 +113,43 @@ jobs: PASS="—" FAIL="—" fi - printf "║ %-20s %-7s %-7s ║\n" "$NAME" "$PASS" "$FAIL" + + # Coverage from lcov.info + if [ -f "$LCOV" ]; then + LF=$(grep -oP '(?<=LF:)\d+' "$LCOV" | awk '{s+=$1} END {print s+0}') + LH=$(grep -oP '(?<=LH:)\d+' "$LCOV" | awk '{s+=$1} END {print s+0}') + if [ "$LF" -gt 0 ]; then + PCT=$(awk "BEGIN {printf \"%.1f%%\", ($LH/$LF)*100}") + else + PCT="N/A" + fi + LINES="$LH/$LF" + else + LINES="—" + PCT="—" + LF=0 + LH=0 + fi + + printf "║ %-20s %-7s %-7s %-8s %-10s ║\n" "$NAME" "$PASS" "$FAIL" "$LINES" "$PCT" + if [ "$PASS" != "—" ]; then TOTAL_PASS=$((TOTAL_PASS + PASS)); fi if [ "$FAIL" != "—" ]; then TOTAL_FAIL=$((TOTAL_FAIL + FAIL)); fi + TOTAL_HIT=$((TOTAL_HIT + LH)) + TOTAL_FOUND=$((TOTAL_FOUND + LF)) done - echo "╠══════════════════════════════════════╣" - printf "║ %-20s %-7s %-7s ║\n" "TOTAL" "$TOTAL_PASS" "$TOTAL_FAIL" - echo "╚══════════════════════════════════════╝" + if [ "$TOTAL_FOUND" -gt 0 ]; then + TOTAL_PCT=$(awk "BEGIN {printf \"%.1f%%\", ($TOTAL_HIT/$TOTAL_FOUND)*100}") + TOTAL_LINES="$TOTAL_HIT/$TOTAL_FOUND" + else + TOTAL_PCT="—" + TOTAL_LINES="—" + fi + + echo "╠═══════════════════════════════════════════════════════════╣" + printf "║ %-20s %-7s %-7s %-8s %-10s ║\n" "TOTAL" "$TOTAL_PASS" "$TOTAL_FAIL" "$TOTAL_LINES" "$TOTAL_PCT" + echo "╚═══════════════════════════════════════════════════════════╝" if [ "$TOTAL_FAIL" -gt 0 ]; then echo "" @@ -124,4 +158,5 @@ jobs: else echo "" echo "✅ All $TOTAL_PASS tests passed across all packages." + echo "📊 Overall line coverage: $TOTAL_PCT ($TOTAL_LINES lines)" fi diff --git a/.gitignore b/.gitignore index 2beb5af..7fa942b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ __pycache__/ .venv/ .DS_Store Thumbs.db + +# Flutter test coverage output +coverage/ diff --git a/docs/development/master_development_brief.md b/docs/development/master_development_brief.md index 339abd6..52c4795 100644 --- a/docs/development/master_development_brief.md +++ b/docs/development/master_development_brief.md @@ -95,6 +95,7 @@ Rules: - ✅ Design system expansion and shared widget migration landed (Stage 4A complete — merged `feat/design-system-shared-widgets` → `main`, 2026-05-22). - ✅ Cross-platform shell composition strategy landed (Stage 4B complete — merged `feat/shared-composition-pattern` → `main`, 2026-05-22). - ✅ Flutter CI/CD pipeline landed (Stage 4C complete — merged `feat/flutter-cicd` → `main`, 2026-05-22). +- ✅ Test coverage visibility landed (Stage 4D complete — merged `feat/test-coverage-visibility` → `main`, 2026-05-22). ### Current narrow edit capabilities on `main` @@ -115,12 +116,24 @@ Rules: - latest reported count for `design_system`: `41/41 passed` - latest reported count for `feature_wordpress`: `294/294 passed` - latest reported count for `kell_web`: `24/24 passed` -- baseline commit: merge of `feat/flutter-cicd` (2026-05-22) +- baseline commit: merge of `feat/test-coverage-visibility` (2026-05-22) + +#### Baseline test coverage (established 2026-05-22) + +| Package | Tests | Lines Hit | Lines Found | Coverage | +| ------------------- | ------- | --------- | ----------- | --------- | +| `core` | 20 | 42 | 49 | 85.7% | +| `design_system` | 41 | 88 | 88 | 100.0% | +| `feature_wordpress` | 294 | 857 | 1012 | 84.7% | +| `kell_web` | 24 | 191 | 353 | 54.1% | +| **Total** | **379** | **1178** | **1502** | **78.4%** | + +No minimum thresholds are enforced — this is visibility-only tracking. Coverage is measured via `flutter test --coverage` (generates `lcov.info`) and reported in the CI workflow summary table. ### Next recommended branch -**`feat/test-coverage-visibility`** — Stage 4D: Test coverage visibility. -Branch from latest `main`. Stage 4C (Flutter CI/CD pipeline) is complete. +**`feat/android-app-shell`** — Stage 5A: Android app shell and bootstrap. +Branch from latest `main`. Stage 4 (Platform foundations and cross-platform readiness) is complete. --- @@ -260,23 +273,10 @@ Extract or document a shared app composition pattern so `kell_mobile` can mirror > Merged `feat/flutter-cicd` → `main` (2026-05-22). > Added Forgejo Actions workflows: `flutter-analyze.yml` (runs `dart analyze --fatal-infos` on all 8 packages/apps) and `flutter-test.yml` (runs `flutter test` per package with per-package pass/fail count and aggregate summary table). Workflows trigger on PRs to main and all non-main branch pushes using `ghcr.io/cirruslabs/flutter:stable` container. Populated `tools/` directory with `run_all_tests.sh` (local test runner with optional `--analyze` flag) and `README.md` documenting scripts and CI workflow inventory. All existing tests passing (20 core, 41 design_system, 294 feature_wordpress, 24 kell_web — 379 total). Analyze clean. -#### Stage 4D — Test coverage visibility +#### ~~Stage 4D — Test coverage visibility~~ ✅ COMPLETE -##### Goal - -Introduce lightweight test coverage tracking across packages. - -##### Requirements - -- add coverage measurement to CI pipeline (at minimum: total tests and pass rate per package) -- document current baseline coverage in this brief or a companion document -- do not enforce minimum thresholds yet — visibility first - -##### Definition of done - -- coverage data is generated and visible in CI output -- baseline coverage documented -- no regressions in existing tests +> Merged `feat/test-coverage-visibility` → `main` (2026-05-22). +> Enhanced `flutter-test.yml` CI workflow to run `flutter test --coverage` and parse `lcov.info` files, producing an aggregate summary table with per-package pass/fail counts and line coverage percentages. Added `collect_coverage.sh` local coverage helper script and updated `tools/README.md`. Established baseline coverage: core 85.7% (42/49), design_system 100.0% (88/88), feature_wordpress 84.7% (857/1012), kell_web 54.1% (191/353) — overall 78.4% (1178/1502). No minimum thresholds enforced — visibility first. All 379 tests passing. Analyze clean. Stage 4 complete. --- @@ -462,16 +462,9 @@ Currently, CI/CD exists only for MkDocs documentation publishing. There is no au **Recommendation:** Now addressed in **Stage 4C** (Flutter CI/CD pipeline). Establish Flutter CI/CD in Forgejo Actions before the Android expansion in Stage 5 adds more surfaces to validate. -### 5. Test coverage visibility and quality gates +### 5. ~~Test coverage visibility and quality gates~~ ✅ PARTIALLY RESOLVED -Test counts are tracked manually in this brief (e.g., `294/294 passed`). There is no: - -- Automated test count reporting -- Coverage measurement -- Minimum coverage threshold enforcement -- Test trend tracking across slices - -**Recommendation:** Introduce coverage tooling and reporting as part of CI/CD establishment. Even lightweight coverage tracking (total tests, pass rate per package) would improve confidence during the Android expansion. +> Automated test count reporting and line coverage measurement now addressed in Stage 4D (test coverage visibility). CI workflow produces per-package pass/fail counts and coverage percentages. Baseline documented. Minimum threshold enforcement and trend tracking remain future enhancements. ### 6. Build execution tracker synchronization diff --git a/kell_creations_apps/tools/README.md b/kell_creations_apps/tools/README.md index f8a216d..b1b5a57 100644 --- a/kell_creations_apps/tools/README.md +++ b/kell_creations_apps/tools/README.md @@ -32,6 +32,45 @@ When a new package gains tests, add its path to the `TESTABLE` array in the scri - `0` — all tests passed (and analyze clean, if `--analyze` was used) - `1` — one or more failures detected +### `collect_coverage.sh` + +Runs `flutter test --coverage` across all testable packages and apps, parses the generated `lcov.info` files, and produces a combined summary table with test counts and line coverage percentages. + +**Usage:** + +```bash +# From kell_creations_apps/ directory +./tools/collect_coverage.sh +``` + +**What it does:** + +1. Installs dependencies (`flutter pub get`) for testable packages. +2. Runs `flutter test --coverage --reporter expanded` for each package. +3. Parses `coverage/lcov.info` to extract lines hit / lines found per package. +4. Prints an aggregate summary table with pass/fail counts and coverage percentages. + +**Output example:** + +``` +╔═══════════════════════════════════════════════════════════╗ +║ Flutter Test & Coverage Summary ║ +╠═══════════════════════════════════════════════════════════╣ +║ Package Pass Fail Lines Coverage ║ +╠═══════════════════════════════════════════════════════════╣ +║ core 20 0 120/150 80.0% ║ +║ design_system 41 0 95/110 86.4% ║ +║ ... ║ +╠═══════════════════════════════════════════════════════════╣ +║ TOTAL 379 0 500/600 83.3% ║ +╚═══════════════════════════════════════════════════════════╝ +``` + +**Exit codes:** + +- `0` — all tests passed +- `1` — one or more failures detected + ## CI Workflows The corresponding Forgejo Actions workflows live in `.forgejo/workflows/`: diff --git a/kell_creations_apps/tools/collect_coverage.sh b/kell_creations_apps/tools/collect_coverage.sh new file mode 100644 index 0000000..c935cdd --- /dev/null +++ b/kell_creations_apps/tools/collect_coverage.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────── +# collect_coverage.sh — Run flutter test --coverage and report results +# +# Usage: +# ./tools/collect_coverage.sh # Run from kell_creations_apps/ +# +# Generates coverage/lcov.info per package, then prints a summary table +# showing test pass/fail counts and line coverage percentage. +# +# Exit codes: +# 0 — all tests passed +# 1 — one or more failures +# ────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Packages/apps with tests (add new ones here as they gain tests) +TESTABLE=( + packages/core + packages/design_system + packages/feature_wordpress + apps/kell_web +) + +OVERALL_EXIT=0 + +# ── Dependency install ─────────────────────────────────────────────── +echo "" +echo "══════════════════════════════════════" +echo " Installing dependencies" +echo "══════════════════════════════════════" + +for pkg in "${TESTABLE[@]}"; do + echo " → $pkg" + (cd "$ROOT_DIR/$pkg" && flutter pub get --no-example) > /dev/null 2>&1 +done + +# ── Tests with coverage ───────────────────────────────────────────── +echo "" +echo "══════════════════════════════════════" +echo " Running flutter test --coverage" +echo "══════════════════════════════════════" + +declare -A RESULTS_PASS +declare -A RESULTS_FAIL +declare -A RESULTS_LH +declare -A RESULTS_LF +declare -A RESULTS_PCT + +for pkg in "${TESTABLE[@]}"; do + NAME=$(basename "$pkg") + echo "" + echo " ── $NAME ──" + + TMPFILE=$(mktemp) + if (cd "$ROOT_DIR/$pkg" && flutter test --coverage --reporter expanded 2>&1) | tee "$TMPFILE"; then + : # tests passed + else + OVERALL_EXIT=1 + fi + + PASS=$(grep -cE '^\s*✓' "$TMPFILE" 2>/dev/null || echo "0") + FAIL=$(grep -cE '^\s*✗' "$TMPFILE" 2>/dev/null || echo "0") + RESULTS_PASS[$NAME]=$PASS + RESULTS_FAIL[$NAME]=$FAIL + rm -f "$TMPFILE" + + # Parse lcov.info for coverage + LCOV="$ROOT_DIR/$pkg/coverage/lcov.info" + if [ -f "$LCOV" ]; then + LF=$(grep -oP '(?<=LF:)\d+' "$LCOV" | awk '{s+=$1} END {print s+0}') + LH=$(grep -oP '(?<=LH:)\d+' "$LCOV" | awk '{s+=$1} END {print s+0}') + if [ "$LF" -gt 0 ]; then + PCT=$(awk "BEGIN {printf \"%.1f\", ($LH/$LF)*100}") + else + PCT="0.0" + fi + else + LF=0 + LH=0 + PCT="—" + fi + RESULTS_LH[$NAME]=$LH + RESULTS_LF[$NAME]=$LF + RESULTS_PCT[$NAME]=$PCT +done + +# ── Summary ────────────────────────────────────────────────────────── +echo "" +echo "╔═══════════════════════════════════════════════════════════╗" +echo "║ Flutter Test & Coverage Summary ║" +echo "╠═══════════════════════════════════════════════════════════╣" +echo "║ Package Pass Fail Lines Coverage ║" +echo "╠═══════════════════════════════════════════════════════════╣" + +TOTAL_PASS=0 +TOTAL_FAIL=0 +TOTAL_HIT=0 +TOTAL_FOUND=0 + +for pkg in "${TESTABLE[@]}"; do + NAME=$(basename "$pkg") + P=${RESULTS_PASS[$NAME]:-0} + F=${RESULTS_FAIL[$NAME]:-0} + LH=${RESULTS_LH[$NAME]:-0} + LF=${RESULTS_LF[$NAME]:-0} + PCT=${RESULTS_PCT[$NAME]:-"—"} + + if [ "$PCT" != "—" ]; then + LINES="$LH/$LF" + PCT_DISPLAY="${PCT}%" + else + LINES="—" + PCT_DISPLAY="—" + fi + + printf "║ %-20s %-7s %-7s %-8s %-10s ║\n" "$NAME" "$P" "$F" "$LINES" "$PCT_DISPLAY" + TOTAL_PASS=$((TOTAL_PASS + P)) + TOTAL_FAIL=$((TOTAL_FAIL + F)) + TOTAL_HIT=$((TOTAL_HIT + LH)) + TOTAL_FOUND=$((TOTAL_FOUND + LF)) +done + +if [ "$TOTAL_FOUND" -gt 0 ]; then + TOTAL_PCT=$(awk "BEGIN {printf \"%.1f\", ($TOTAL_HIT/$TOTAL_FOUND)*100}") + TOTAL_LINES="$TOTAL_HIT/$TOTAL_FOUND" + TOTAL_PCT_DISPLAY="${TOTAL_PCT}%" +else + TOTAL_LINES="—" + TOTAL_PCT_DISPLAY="—" +fi + +echo "╠═══════════════════════════════════════════════════════════╣" +printf "║ %-20s %-7s %-7s %-8s %-10s ║\n" "TOTAL" "$TOTAL_PASS" "$TOTAL_FAIL" "$TOTAL_LINES" "$TOTAL_PCT_DISPLAY" +echo "╚═══════════════════════════════════════════════════════════╝" + +if [ $OVERALL_EXIT -ne 0 ]; then + echo "" + echo "❌ Failures detected. See details above." +else + echo "" + echo "✅ All $TOTAL_PASS tests passed across all packages." + echo "📊 Overall line coverage: $TOTAL_PCT_DISPLAY ($TOTAL_LINES lines)" +fi + +exit $OVERALL_EXIT