[WIP] Smartly select what tests to run on PR iterations

tools/fingerprint/fingerprint.go

@@ -0,0 +1,73 @@
+// Command fingerprint computes a content hash over a set of tracked files at a
+// git commit. Two commits with the same fingerprint for a given input set have
+// byte-identical inputs, so a test target that passed at one can be safely
+// skipped at the other.
+//
+// This is the primitive behind result-reuse across PR iterations: unlike
+// testmask, which diffs the cumulative PR range and so re-runs a target whenever
+// the PR *ever* touched it, a content fingerprint is identical across PR
+// iterations that leave a target's inputs unchanged — and it is force-push /
+// rebase immune because it addresses content, not history. A target that passed
+// at a fingerprint can be skipped at any later commit with the same fingerprint.
+package main
+
+import (
+	"cmp"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"slices"
+	"strings"
+)
+
+// TreeEntry is one tracked file: its path and git blob object id (content hash).
+type TreeEntry struct {
+	Path string
+	Blob string
+}
+
+// Fingerprint returns a deterministic hash over the given tree entries.
+//
+// It is order-independent (entries are sorted by path first) so it does not
+// depend on git's listing order, and it binds each path to its blob id so both
+// a content change (blob changes) and a rename (path changes) alter the result.
+// The path is length-prefixed to make the encoding unambiguous, so no two
+// distinct entry sets can collide by concatenation.
+func Fingerprint(entries []TreeEntry) string {
+	sorted := slices.Clone(entries)
+	slices.SortFunc(sorted, func(a, b TreeEntry) int {
+		return cmp.Compare(a.Path, b.Path)
+	})
+
+	h := sha256.New()
+	for _, e := range sorted {
+		// Length-prefix the path so "ab"+"c" and "a"+"bc" cannot hash alike.
+		fmt.Fprintf(h, "%d\x00%s\x00%s\x00", len(e.Path), e.Path, e.Blob)
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+// ParseLsTree parses the output of
+//
+//	git ls-tree -r --format='%(objectname) %(path)' <commit> [-- <paths>...]
+//
+// into tree entries. It is split from the git invocation so the parsing is
+// unit-testable with literal strings, without a git repo (the same split
+// testmask uses for GetChangedFiles vs GetTargets).
+//
+// Each line is "<blob-sha> <path>". A blank trailing line is ignored. Paths with
+// spaces are preserved because only the first field is the blob id.
+func ParseLsTree(out string) []TreeEntry {
+	var entries []TreeEntry
+	for line := range strings.SplitSeq(out, "\n") {
+		if line == "" {
+			continue
+		}
+		blob, path, ok := strings.Cut(line, " ")
+		if !ok {
+			continue
+		}
+		entries = append(entries, TreeEntry{Path: path, Blob: blob})
+	}
+	return entries
+}

tools/fingerprint/fingerprint_test.go

@@ -0,0 +1,127 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFingerprintDeterministic(t *testing.T) {
+	// Two independently constructed but equal inputs must hash identically, so
+	// the fingerprint depends only on content, not on slice identity or run.
+	first := []TreeEntry{
+		{Path: "a.go", Blob: "aaa"},
+		{Path: "b.go", Blob: "bbb"},
+	}
+	second := []TreeEntry{
+		{Path: "a.go", Blob: "aaa"},
+		{Path: "b.go", Blob: "bbb"},
+	}
+	assert.Equal(t, Fingerprint(first), Fingerprint(second))
+}
+
+func TestFingerprintOrderIndependent(t *testing.T) {
+	forward := []TreeEntry{
+		{Path: "a.go", Blob: "aaa"},
+		{Path: "b.go", Blob: "bbb"},
+	}
+	reversed := []TreeEntry{
+		{Path: "b.go", Blob: "bbb"},
+		{Path: "a.go", Blob: "aaa"},
+	}
+	assert.Equal(t, Fingerprint(forward), Fingerprint(reversed),
+		"listing order must not change the fingerprint")
+}
+
+func TestFingerprintContentSensitive(t *testing.T) {
+	before := []TreeEntry{{Path: "a.go", Blob: "aaa"}}
+	after := []TreeEntry{{Path: "a.go", Blob: "aab"}}
+	assert.NotEqual(t, Fingerprint(before), Fingerprint(after),
+		"a blob change must change the fingerprint")
+}
+
+func TestFingerprintRenameSensitive(t *testing.T) {
+	before := []TreeEntry{{Path: "a.go", Blob: "aaa"}}
+	after := []TreeEntry{{Path: "renamed.go", Blob: "aaa"}}
+	assert.NotEqual(t, Fingerprint(before), Fingerprint(after),
+		"a rename (same content, new path) must change the fingerprint")
+}
+
+func TestFingerprintAddedFileSensitive(t *testing.T) {
+	before := []TreeEntry{{Path: "a.go", Blob: "aaa"}}
+	after := []TreeEntry{
+		{Path: "a.go", Blob: "aaa"},
+		{Path: "b.go", Blob: "bbb"},
+	}
+	assert.NotEqual(t, Fingerprint(before), Fingerprint(after),
+		"adding a file must change the fingerprint")
+}
+
+// TestFingerprintNoConcatenationCollision guards the length-prefixed encoding:
+// path/blob boundaries must be unambiguous so no two distinct entry sets whose
+// naive concatenation matches can collide.
+func TestFingerprintNoConcatenationCollision(t *testing.T) {
+	a := []TreeEntry{{Path: "ab", Blob: "c"}}
+	b := []TreeEntry{{Path: "a", Blob: "bc"}}
+	assert.NotEqual(t, Fingerprint(a), Fingerprint(b))
+}
+
+func TestFingerprintEmptyStable(t *testing.T) {
+	// The empty set has a well-defined hash; callers (not this function) decide
+	// whether "nothing matched" is meaningful. Just assert it is stable.
+	assert.Equal(t, Fingerprint(nil), Fingerprint([]TreeEntry{}))
+}
+
+func TestParseLsTree(t *testing.T) {
+	tests := []struct {
+		name string
+		out  string
+		want []TreeEntry
+	}{
+		{
+			name: "basic",
+			out:  "aaa a.go\nbbb dir/b.go\n",
+			want: []TreeEntry{
+				{Path: "a.go", Blob: "aaa"},
+				{Path: "dir/b.go", Blob: "bbb"},
+			},
+		},
+		{
+			name: "path_with_spaces",
+			out:  "aaa dir/file with spaces.txt\n",
+			want: []TreeEntry{
+				{Path: "dir/file with spaces.txt", Blob: "aaa"},
+			},
+		},
+		{
+			name: "trailing_and_blank_lines_ignored",
+			out:  "aaa a.go\n\nbbb b.go\n\n",
+			want: []TreeEntry{
+				{Path: "a.go", Blob: "aaa"},
+				{Path: "b.go", Blob: "bbb"},
+			},
+		},
+		{
+			name: "empty",
+			out:  "",
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, ParseLsTree(tt.out))
+		})
+	}
+}
+
+// TestParseThenFingerprintMatchesDirect ties the two halves together: parsing
+// ls-tree output and fingerprinting it yields the same hash as fingerprinting
+// the entries directly.
+func TestParseThenFingerprintMatchesDirect(t *testing.T) {
+	entries := []TreeEntry{
+		{Path: "a.go", Blob: "aaa"},
+		{Path: "b.go", Blob: "bbb"},
+	}
+	parsed := ParseLsTree("aaa a.go\nbbb b.go\n")
+	assert.Equal(t, Fingerprint(entries), Fingerprint(parsed))
+}

tools/fingerprint/git.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+)
+
+// TreeEntriesAt lists the tracked files matching pathspecs at the given commit,
+// with their blob ids, via `git ls-tree`.
+//
+// It uses ls-tree (a snapshot of one commit) rather than a diff so the
+// fingerprint is absolute: it depends only on the content at `commit`, not on
+// any base ref. Blob ids come straight from git's index, so no file is read.
+func TreeEntriesAt(commit string, pathspecs []string) ([]TreeEntry, error) {
+	args := []string{"ls-tree", "-r", "--format=%(objectname) %(path)", commit}
+	if len(pathspecs) > 0 {
+		args = append(args, "--")
+		args = append(args, pathspecs...)
+	}
+	out, err := exec.Command("git", args...).Output()
+	if err != nil {
+		return nil, fmt.Errorf("git ls-tree at %s failed: %w", commit, err)
+	}
+	return ParseLsTree(string(out)), nil
+}

tools/fingerprint/main.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Fprintf(os.Stderr, "Usage: %s <commit> [pathspec...]\n", os.Args[0])
+		fmt.Fprintf(os.Stderr, "Prints a content fingerprint over the tracked files matching pathspec at commit.\n")
+		os.Exit(1)
+	}
+
+	commit := os.Args[1]
+	pathspecs := os.Args[2:]
+
+	entries, err := TreeEntriesAt(commit, pathspecs)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error listing tree: %v\n", err)
+		os.Exit(1)
+	}
+	if len(entries) == 0 {
+		// No file matched the pathspecs. Emitting a hash of the empty set would
+		// let an unrelated later commit collide with it; a caller must decide
+		// what "nothing to fingerprint" means, so make it an explicit error
+		// rather than silently returning a reusable hash.
+		fmt.Fprintf(os.Stderr, "Error: no tracked files matched at %s\n", commit)
+		os.Exit(1)
+	}
+
+	fmt.Println(Fingerprint(entries))
+}

tools/testmask/targets.go

@@ -29,6 +29,52 @@ var commonTriggerPatterns = []string{
 	"tools/task/",
 }
 
+// inertFiles are repo-relative paths whose contents cannot affect any test
+// outcome, so a change touching only these (plus inertPrefixes) runs no tests.
+//
+// This list is deliberately narrow and must never include a file that a test
+// reads: NOTICE is verified by internal/build/notice_test.go, and the many
+// README.md files under acceptance/ and libs/template/ are test fixtures.
+// None of those are inert. TestInertPathsExcludeTestInputs guards this.
+var inertFiles = map[string]bool{
+	"README.md":                        true,
+	"SECURITY.md":                      true,
+	"CHANGELOG.md":                     true,
+	"CLAUDE.md":                        true,
+	"AGENTS.md":                        true,
+	"LICENSE":                          true,
+	".github/PULL_REQUEST_TEMPLATE.md": true,
+}
+
+// inertPrefixes are repo-relative directory prefixes that hold only
+// documentation or contributor metadata, never code or test inputs.
+//
+// .nextchanges/ holds per-PR changelog fragments. Their placement is validated
+// by `task check-changelog` in the `check` workflow, which runs on every PR
+// independently of testmask, so skipping the test suite for a fragment-only
+// change is safe.
+var inertPrefixes = []string{
+	"docs/",
+	".github/ISSUE_TEMPLATE/",
+	".nextchanges/",
+}
+
+// isInertPath reports whether a changed file is documentation or metadata that
+// cannot influence a test outcome. Matching is on exact repo-relative paths (or
+// the inertPrefixes directories), never bare basenames, so fixture files such
+// as acceptance/.../README.md are not misclassified as inert.
+func isInertPath(file string) bool {
+	if inertFiles[file] {
+		return true
+	}
+	for _, prefix := range inertPrefixes {
+		if strings.HasPrefix(file, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
 type targetMapping struct {
 	prefixes []string
 	target   string
@@ -103,10 +149,22 @@ func sourceToPrefix(src string) string {
 
 // GetTargets matches files to targets based on patterns and returns the matched targets.
 func GetTargets(files []string, mappings []targetMapping) []string {
+	// No changed files were determined; fall back to the full test suite. The
+	// all-inert skip below requires at least one changed file, all inert.
+	if len(files) == 0 {
+		return []string{"test"}
+	}
+
 	targetSet := make(map[string]bool)
 	unmatchedFiles := []string{}
 
 	for _, file := range files {
+		if isInertPath(file) {
+			// Documentation/metadata: cannot affect any test outcome, so it
+			// neither selects a target nor counts as an unmatched file that
+			// would otherwise trigger the catch-all `test` target.
+			continue
+		}
 		matched := false
 		for _, mapping := range mappings {
 			for _, prefix := range mapping.prefixes {
@@ -126,8 +184,10 @@ func GetTargets(files []string, mappings []targetMapping) []string {
 		targetSet["test"] = true
 	}
 
+	// Every changed file was inert, so no test target is affected. Return an
+	// empty list; CI treats it as "skip all", including the integration tests.
 	if len(targetSet) == 0 {
-		return []string{"test"}
+		return []string{}
 	}
 
 	return slices.Sorted(maps.Keys(targetSet))