Test all modules

.github/workflows/repo-hygiene.yaml

@@ -44,7 +44,7 @@ jobs:
 
       # Check that go.mod files have fully specificed Go versions that match tool-versions.env
       - name: Enforce go.mod Go versions
-        run: tools/bin/check-go-versions.sh
+        run: just check-go-versions
 
       - name: Run repo hygiene checks
-        run: tools/bin/check_repo_clean.sh tidy mock generate shellcheck
+        run: just check-repo-clean

.github/workflows/test-coverage-report.yaml

@@ -47,17 +47,19 @@ jobs:
 
       - name: Run tests
         run: |
-          just test-coverage coverage.out short
+          just test-coverage coverage.out 1
       - name: Coverage on target branch
         if: github.event_name == 'pull_request'
         run: |
           git fetch origin ${{ github.base_ref }}
           git branch -a
           git checkout ${{ github.base_ref }}
-          just test-coverage coverage_target.out short
+          # Allow test failures on the target branch — a flaky test there should not block the PR.
+          # Coverage data is still written for all packages that did not panic.
+          just test-coverage coverage_target.out 1 || echo "Warning: some tests failed on target branch; coverage may be incomplete"
           # switch back to the head ref
           git checkout ${{ github.head_ref }}
-          ./tools/bin/cov_compare.sh coverage_target.out coverage.out --label1="${{ github.base_ref }}" --label2="${{ github.head_ref }}" > table.txt
+          ./tools/bin/cov_compare.sh coverage_target.out coverage.out --label1="${{ github.base_ref }}" --label2="${{ github.head_ref }}" > table.txt || echo "(coverage comparison unavailable)" > table.txt
           cat table.txt
           {
             echo 'coverage_report<<COVERAGE_REPORT_DELIM'

.github/workflows/test-coverage.yaml

@@ -54,7 +54,7 @@ jobs:
 
       - name: Run tests
         run: |
-          just test-coverage coverage.out
+          just test-coverage
           total=$(go tool cover -func=coverage.out | grep total | awk '{print $3}')
           echo "Total coverage: $total"
           echo "$total" | awk -v threshold="$COVERAGE_THRESHOLD" '{ pc=substr($0, 1, length($0)-1); if (pc < threshold) { print "Coverage (" pc "%) is less than " threshold "%"; exit 1 } }'

Justfile

@@ -58,18 +58,31 @@ shellcheck:
     }
     find . -type f -name *.sh -execdir shellcheck {} +
 
+check-go-versions:
+    ./tools/bin/check-go-versions.sh
+
+# Run hygiene targets and fail if any modify the repo. Default: tidy mock generate shellcheck
+check-repo-clean +targets="tidy mock generate shellcheck":
+    ./tools/bin/check_repo_clean.sh {{targets}}
+
+# Render inter-module dependency graph as mermaid markdown
+modgraph: ensure-go
+    ./tools/bin/modgraph.sh
+
 mod-download: ensure-go
     go mod download
 
-test short="": ensure-go
-    gomods -w go test -fullpath -shuffle on {{ if short != "" { "-short" } else { "" } }} -v -race ./...
+test short="" all="": ensure-go
+    {{ if all != "" { "gomods -w" } else { "" } }} go test -fullpath -shuffle on {{ if short != "" { "-short" } else { "" } }} -v -race ./...
+
+# When all="1", generates <module-dir>/<coverage_file> for each module (discoverable via find . -name coverage.out).
+test-coverage coverage_file="coverage.out" short="" all="":
+    {{ if all != "" { "gomods -w" } else { "" } }} go test -v -race -fullpath -shuffle on {{ if short != "" { "-short" } else { "" } }} -coverprofile={{coverage_file}} ./...
+    {{ if all != "" { "gomods -w " } else { "" } }}sh -c '{ head -n1 {{coverage_file}}; tail -n +2 {{coverage_file}} | grep -v -E "{{COVERAGE_EXCLUDE_REGEX}}" || true; } > {{coverage_file}}.filtered && mv {{coverage_file}}.filtered {{coverage_file}}'
 
-test-coverage coverage_file="coverage.out" short="":
-    # coverage_file := env_var_or_default('COVERAGE_FILE', 'coverage.out')
-    go test -v -race -fullpath -shuffle on {{ if short != "" { "-short" } else { "" } }} -v -coverprofile={{coverage_file}} ./...
-    # Filter mockery-generated files (mock_*.go) from coverage profile
-    { head -n1 {{coverage_file}}; tail -n +2 {{coverage_file}} | grep -v -E '{{COVERAGE_EXCLUDE_REGEX}}' || true; } > {{coverage_file}}.filtered
-    mv {{coverage_file}}.filtered {{coverage_file}}
+# Compare per-module coverage between two profiles. Run test-coverage all=1 twice with different coverage_file names first.
+cov-compare base="coverage_base.out" new="coverage.out":
+    gomods -c 'printf "\n### %s\n" "$(basename $PWD)"; {{justfile_directory()}}/tools/bin/cov_compare.sh "{{base}}" "{{new}}" || echo "(skipped — run: just test-coverage all=1)"'
 
 bump-chainlink-ccip sha:
     @echo "Bumping chainlink-ccip dependencies in root..."

aggregator/pkg/handlers/batch_write_commit_verifier_node_result.go

@@ -5,13 +5,13 @@ import (
 	"fmt"
 	"sync"
 
+	rpcstatus "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc/codes"
+	grpcstatus "google.golang.org/grpc/status"
 
 	"github.com/smartcontractkit/chainlink-ccv/aggregator/pkg/scope"
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 
-	grpcstatus "google.golang.org/grpc/status"
-
 	committeepb "github.com/smartcontractkit/chainlink-protos/chainlink-ccv/committee-verifier/v1"
 )
 
@@ -25,6 +25,22 @@ func (h *BatchWriteCommitVerifierNodeResultHandler) logger(ctx context.Context)
 	return scope.AugmentLogger(ctx, h.handler.l)
 }
 
+func (h *BatchWriteCommitVerifierNodeResultHandler) handleBatchItemError(ctx context.Context, errors []*rpcstatus.Status, i int, err error) {
+	statusErr, ok := grpcstatus.FromError(err)
+	if !ok {
+		if ctx.Err() == nil {
+			h.logger(ctx).Errorw("unexpected error type", "error", err)
+		}
+		SetBatchError(errors, i, codes.Unknown, "internal error")
+		return
+	}
+	code := statusErr.Code()
+	if code != codes.Canceled && code != codes.DeadlineExceeded {
+		h.logger(ctx).Errorw("failed to write commit verification node result", "error", statusErr)
+	}
+	errors[i] = statusErr.Proto()
+}
+
 // Handle processes the write request and saves the commit verification record.
 // The parent context includes a timeout from RequestTimeoutMiddleware to prevent goroutine leaks.
 func (h *BatchWriteCommitVerifierNodeResultHandler) Handle(ctx context.Context, req *committeepb.BatchWriteCommitteeVerifierNodeResultRequest) (*committeepb.BatchWriteCommitteeVerifierNodeResultResponse, error) {
@@ -55,18 +71,12 @@ func (h *BatchWriteCommitVerifierNodeResultHandler) Handle(ctx context.Context,
 				return
 			}
 			resp, err := h.handler.Handle(ctx, r)
-			if err != nil {
-				statusErr, ok := grpcstatus.FromError(err)
-				if !ok {
-					h.logger(ctx).Errorw("unexpected error type", "error", err)
-					SetBatchError(errors, i, codes.Unknown, "internal error")
-				} else {
-					h.logger(ctx).Errorw("failed to write commit verification node result", "error", statusErr)
-					errors[i] = statusErr.Proto()
-				}
-			} else {
+			if err == nil {
 				SetBatchSuccess(errors, i)
+				responses[i] = resp
+				return
 			}
+			h.handleBatchItemError(ctx, errors, i, err)
 			responses[i] = resp
 		}(i, r)
 	}

aggregator/pkg/handlers/write_commit_verifier_node_result.go

@@ -126,6 +126,16 @@ func (h *WriteCommitVerifierNodeResultHandler) Handle(ctx context.Context, req *
 			}, status.Error(codes.ResourceExhausted, "service temporarily unavailable: aggregation queue full")
 		}
 
+		if ctx.Err() != nil {
+			code := codes.DeadlineExceeded
+			if ctx.Err() == context.Canceled {
+				code = codes.Canceled
+			}
+			return &committeepb.WriteCommitteeVerifierNodeResultResponse{
+				Status: committeepb.WriteStatus_FAILED,
+			}, status.Error(code, "request cancelled")
+		}
+
 		reqLogger.Errorw("failed to trigger aggregation", "error", err)
 		return &committeepb.WriteCommitteeVerifierNodeResultResponse{
 			Status: committeepb.WriteStatus_FAILED,

build/devenv/evm/event_poller.go

@@ -165,9 +165,14 @@ func (p *eventPoller[T]) poll() {
 		return
 	}
 	lastScanned := p.lastScannedBlock
+	client := p.ethClient
 	p.mu.Unlock()
 
-	latestBlock, err := p.ethClient.BlockNumber(context.Background())
+	if client == nil {
+		return
+	}
+
+	latestBlock, err := client.BlockNumber(context.Background())
 	if err != nil {
 		p.logger.Warn().Err(err).Str("event", p.eventName).Msg("Failed to get latest block number")
 		return

build/devenv/evm/event_poller_test.go

@@ -171,9 +171,8 @@ func TestEventPollerByMessageID(t *testing.T) {
 		messageID := protocol.Bytes32{3, 4, 5, 6}
 		key := eventKey{chainSelector: 5, messageID: messageID}
 
-		ctx := context.Background()
-		ch1 := poller.registerByMessageID(ctx, key)
-		ch2 := poller.registerByMessageID(ctx, key)
+		ch1 := poller.registerByMessageID(t.Context(), key)
+		ch2 := poller.registerByMessageID(t.Context(), key)
 		require.Equal(t, ch1, ch2, "registerByMessageID should return same channel for duplicate key registration")
 	})
 

build/devenv/tests/composable/messaging/main_test.go

@@ -0,0 +1,15 @@
+package messaging
+
+import (
+	"flag"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	if testing.Short() {
+		os.Exit(0)
+	}
+	os.Exit(m.Run())
+}

build/devenv/tests/e2e/main_test.go

@@ -0,0 +1,15 @@
+package e2e
+
+import (
+	"flag"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	if testing.Short() {
+		os.Exit(0)
+	}
+	os.Exit(m.Run())
+}

build/devenv/tests/services/load/main_test.go

@@ -0,0 +1,15 @@
+package load
+
+import (
+	"flag"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	if testing.Short() {
+		os.Exit(0)
+	}
+	os.Exit(m.Run())
+}

build/devenv/tests/services/main_test.go

@@ -1,6 +1,7 @@
 package services_test
 
 import (
+	"flag"
 	"log"
 	"os"
 	"testing"
@@ -9,6 +10,10 @@ import (
 )
 
 func TestMain(m *testing.M) {
+	flag.Parse()
+	if testing.Short() {
+		os.Exit(0)
+	}
 	// to remove containers after the tests automatically
 	_ = os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "false")
 	// to isolate containers the same way we do in e2e environment