diff --git a/verifier_tools/verify/README.md b/verifier_tools/verify/README.md index ea6e847..3b2a171 100644 --- a/verifier_tools/verify/README.md +++ b/verifier_tools/verify/README.md @@ -39,9 +39,11 @@ The verifier uses the associated checkpoint (depending on the target log) and th * `https://www.gstatic.com/android/binary_transparency/mainline/2026/02/tile/` (latest Tessera-backed sharded log) * `https://www.gstatic.com/android/binary_transparency/mainline/2026/02/tile/entries/` (for data leaves) -To run the verifier after you have built it in the previous section: +### Verification Mode + +To verify that a candidate binary is included in a log: ``` -$ ./verifier --payload_path=${PAYLOAD_PATH} --log_type= +$ ./verifier --payload_path=${PAYLOAD_PATH} --log_type= [--cache_dir=] ``` where `log_type` is one of the following: * `pixel` (for Pixel Factory Images) @@ -49,6 +51,28 @@ where `log_type` is one of the following: * `google_1p_apk` (for Google Product Applications) * `mainline_module` (for Android Mainline Modules) +### Pre-fetching & Offline Cache Mode + +To pre-fetch and locally cache all entry tiles or legacy info files up to the current checkpoint (without requiring a payload or running an inclusion proof): +``` +$ ./verifier --log_type= --fetch_entries [--concurrency=16] [--cache_dir=] +``` + +This enables: + * **Fast, zero-network verifications:** Subsequent verification runs read directly from local cache without multiple HTTP round trips. + * **Concurrent tile fetching:** Tessera entry tiles are fetched concurrently using bounded worker threads (configurable via `--concurrency`, defaulting to 16). Full tiles ($W=256$) are immutable and skipped on subsequent runs, making updates fast and incremental. + * **Custom cache storage:** Pass `--cache_dir` (or `--cache-dir`) to specify a custom directory for the cache. Ideal for persistent CI/CD volumes, container pre-baking, or air-gapped/offline verification bundles. + +### Flags + +| Flag | Description | Default | +| --- | --- | --- | +| `--log_type`, `--log-type` | Target transparency log (`pixel`, `google_1p_code`, `google_1p_apk`, `mainline_module`). Required. | `""` | +| `--payload_path`, `--payload-path` | Path to the payload file describing the candidate binary. Required for verification mode. | `""` | +| `--fetch_entries`, `--fetch-entries` | Pre-fetch and cache all log entries locally up to the latest checkpoint. | `false` | +| `--concurrency` | Number of concurrent workers for fetching Tessera entry tiles. | `16` | +| `--cache_dir`, `--cache-dir` | Custom root directory for local cache. If unspecified, defaults to system cache. | OS user cache dir | + ### Input The verifier takes a `payload_path` and a `log_type` as input. diff --git a/verifier_tools/verify/cmd/verifier/verifier.go b/verifier_tools/verify/cmd/verifier/verifier.go index 575355d..124d86f 100644 --- a/verifier_tools/verify/cmd/verifier/verifier.go +++ b/verifier_tools/verify/cmd/verifier/verifier.go @@ -21,9 +21,13 @@ package main import ( "bytes" + "context" "flag" + "fmt" "log/slog" "os" + "os/signal" + "syscall" "github.com/android/android-binary-transparency/verifier_tools/verify/internal/checkpoint" "github.com/android/android-binary-transparency/verifier_tools/verify/internal/tiles" @@ -76,10 +80,38 @@ var googleAPKLogPubKey []byte var mainlineModuleLogPubKey []byte var ( - payloadPath = flag.String("payload_path", "", "Path to the payload describing the binary of interest.") - logType = flag.String("log_type", "", "Which log: 'pixel' or 'google_1p_code' or 'google_1p_apk' or 'mainline_module'.") + payloadPath = flag.String("payload_path", "", "Path to the payload describing the binary of interest.") + logType = flag.String("log_type", "", "Which log: 'pixel' or 'google_1p_code' or 'google_1p_apk' or 'mainline_module'.") + fetchEntries = flag.Bool("fetch_entries", false, "Pre-fetch and cache all entries/tiles locally for the specified --log_type, without performing an inclusion proof.") + concurrency = flag.Int("concurrency", tiles.DefaultTesseraFetchConcurrency, "Number of concurrent workers for fetching Tessera entry tiles.") + cacheDir = flag.String("cache_dir", "", "Custom root directory for local cache. If unspecified, defaults to system cache directory.") ) +func init() { + flag.StringVar(logType, "log-type", "", "Alias for --log_type.") + flag.StringVar(payloadPath, "payload-path", "", "Alias for --payload_path.") + flag.StringVar(cacheDir, "cache-dir", "", "Alias for --cache_dir.") + flag.BoolVar(fetchEntries, "fetch-entries", false, "Alias for --fetch_entries.") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), `Usage of %s: + +Modes: + 1. Verify binary inclusion in transparency log: + %s --log_type= --payload_path= [--cache_dir=] + + 2. Pre-fetch and cache entries locally for offline verification: + %s --log_type= --fetch_entries [--concurrency=16] [--cache_dir=] + +Supported log types: + pixel, google_1p_code, google_1p_apk, mainline_module + +Flags: +`, os.Args[0], os.Args[0], os.Args[0]) + flag.PrintDefaults() + } +} + type logTarget struct { name string baseURL string @@ -90,35 +122,15 @@ type logTarget struct { binaryInfoFilenames []string } -func main() { - flag.Parse() - - if *payloadPath == "" { - slog.Error("must specify the payload_path for the binary payload") - os.Exit(1) - } - b, err := os.ReadFile(*payloadPath) - if err != nil { - slog.Error("unable to open file", "path", *payloadPath, "error", err) - os.Exit(1) - } - // Payload should not contain excessive leading or trailing whitespace. - payloadBytes := bytes.TrimSpace(b) - payloadBytes = append(payloadBytes, '\n') - if string(b) != string(payloadBytes) { - slog.Info("Reformatted payload content", "from", b, "to", payloadBytes) - } - +func resolveTargets(logType string) ([]logTarget, error) { var targets []logTarget - switch *logType { + switch logType { case "": - slog.Error("must specify which log to verify against using '--log_type' flag: {pixel, google_1p_code, google_1p_apk, mainline_module}") - os.Exit(1) + return nil, fmt.Errorf("must specify which log to target using '--log_type' flag: {pixel, google_1p_code, google_1p_apk, mainline_module}") case "pixel": v, err := checkpoint.NewVerifier(pixelLogPubKey, KeyNameForVerifierPixel) if err != nil { - slog.Error("error creating verifier", "log", "pixel", "error", err) - os.Exit(1) + return nil, fmt.Errorf("error creating verifier for pixel log: %w", err) } targets = append(targets, logTarget{ name: "pixel", @@ -132,8 +144,7 @@ func main() { case "google_1p_code": v, err := checkpoint.NewVerifier(googleSystemAppLogPubKey, KeyNameForVerifierG1PJWT) if err != nil { - slog.Error("error creating verifier", "log", "google_1p_code", "error", err) - os.Exit(1) + return nil, fmt.Errorf("error creating verifier for google_1p_code log: %w", err) } targets = append(targets, logTarget{ name: "google_1p_code", @@ -148,8 +159,7 @@ func main() { // Shard 2026/02: Tessera log v2, err := note.NewVerifier(NoteVerifierG1PAPK202602) if err != nil { - slog.Error("error creating verifier for 2026/02 Tessera log", "error", err) - os.Exit(1) + return nil, fmt.Errorf("error creating verifier for 2026/02 Tessera log: %w", err) } targets = append(targets, logTarget{ name: "google_1p_apk (2026/02 Tessera)", @@ -163,8 +173,7 @@ func main() { // Shard 2026/01: Legacy log continuation fallback v1, err := checkpoint.NewVerifier(googleAPKLogPubKey, KeyNameForVerifierG1PAPK) if err != nil { - slog.Error("error creating verifier for 2026/01 log", "error", err) - os.Exit(1) + return nil, fmt.Errorf("error creating verifier for 2026/01 log: %w", err) } targets = append(targets, logTarget{ name: "google_1p_apk (2026/01)", @@ -179,8 +188,7 @@ func main() { // Shard 2026/02: Tessera log v2, err := note.NewVerifier(NoteVerifierMainlineModule202602) if err != nil { - slog.Error("error creating verifier for 2026/02 Tessera log", "error", err) - os.Exit(1) + return nil, fmt.Errorf("error creating verifier for 2026/02 Tessera log: %w", err) } targets = append(targets, logTarget{ name: "mainline_module (2026/02 Tessera)", @@ -194,8 +202,7 @@ func main() { // Shard 2026/01: Legacy log continuation fallback v1, err := checkpoint.NewVerifier(mainlineModuleLogPubKey, KeyNameForVerifierMainlineModule) if err != nil { - slog.Error("error creating verifier for 2026/01 log", "error", err) - os.Exit(1) + return nil, fmt.Errorf("error creating verifier for 2026/01 log: %w", err) } targets = append(targets, logTarget{ name: "mainline_module (2026/01)", @@ -207,10 +214,82 @@ func main() { binaryInfoFilenames: []string{ModuleInfoFilename}, }) default: - slog.Error("unsupported log type") + return nil, fmt.Errorf("unsupported log type %q", logType) + } + return targets, nil +} + +func runFetchEntries(ctx context.Context, targets []logTarget, concurrency int) error { + for _, target := range targets { + slog.Info("Syncing entries for log", "log", target.name, "url", target.baseURL) + root, err := checkpoint.FromURLWithPath(target.baseURL, target.checkpointPath, target.verifier) + if err != nil { + return fmt.Errorf("failed to read checkpoint for %s: %w", target.name, err) + } + + treeSize := int64(root.Size) + slog.Info("Resolved checkpoint tree size", "log", target.name, "treeSize", treeSize) + + if target.isTessera { + slog.Info("Fetching Tessera entry tiles", "log", target.name, "treeSize", treeSize, "concurrency", concurrency) + if err := tiles.FetchAllTesseraEntries(ctx, target.baseURL, treeSize, concurrency); err != nil { + return fmt.Errorf("failed fetching Tessera entry tiles for %s: %w", target.name, err) + } + } else { + slog.Info("Fetching legacy info files", "log", target.name, "files", target.binaryInfoFilenames, "treeSize", treeSize) + if err := tiles.FetchAllLegacyEntries(ctx, target.baseURL, target.binaryInfoFilenames, treeSize); err != nil { + return fmt.Errorf("failed fetching legacy entries for %s: %w", target.name, err) + } + } + } + return nil +} + +func main() { + flag.Parse() + + if *cacheDir != "" { + tiles.SetCacheDir(*cacheDir) + slog.Info("Using custom cache directory", "path", *cacheDir) + } + + targets, err := resolveTargets(*logType) + if err != nil { + slog.Error(err.Error()) + flag.Usage() os.Exit(1) } + if *fetchEntries { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := runFetchEntries(ctx, targets, *concurrency); err != nil { + slog.Error("FAILURE: error fetching entries", "error", err) + os.Exit(1) + } + activeCacheDir, _ := tiles.CacheDir() + slog.Info("SUCCESS: all entries fetched and cached locally.", "cache_dir", activeCacheDir) + return + } + + if *payloadPath == "" { + slog.Error("must specify either '--payload_path' to verify a binary, or '--fetch_entries' to pre-fetch log entries") + flag.Usage() + os.Exit(1) + } + b, err := os.ReadFile(*payloadPath) + if err != nil { + slog.Error("Unable to open file", "path", *payloadPath, "error", err) + os.Exit(1) + } + // Payload should not contain excessive leading or trailing whitespace. + payloadBytes := bytes.TrimSpace(b) + payloadBytes = append(payloadBytes, '\n') + if string(b) != string(payloadBytes) { + slog.Info("Reformatted payload content", "from", b, "to", payloadBytes) + } + var verified bool for _, target := range targets { slog.Info("Checking log", "log", target.name, "url", target.baseURL) diff --git a/verifier_tools/verify/internal/tiles/reader.go b/verifier_tools/verify/internal/tiles/reader.go index 6251081..45d95df 100644 --- a/verifier_tools/verify/internal/tiles/reader.go +++ b/verifier_tools/verify/internal/tiles/reader.go @@ -3,6 +3,7 @@ package tiles import ( "bytes" + "context" "crypto/sha256" "encoding/binary" "fmt" @@ -15,6 +16,8 @@ import ( "path/filepath" "strconv" "strings" + "sync" + "sync/atomic" "time" "golang.org/x/mod/sumdb/tlog" @@ -116,23 +119,91 @@ func BinaryInfosIndex(logBaseURL string, binaryInfoFilename string, treeSize int return parseBinaryInfosIndex(binaryInfos, binaryInfoFilename) } +var httpClient = &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 32, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, +} + +var ( + customCacheDirMu sync.RWMutex + customCacheDir string +) + +// SetCacheDir configures a custom root directory for the local cache. +// If dir is empty, the default user cache directory scheme is used. +func SetCacheDir(dir string) { + customCacheDirMu.Lock() + defer customCacheDirMu.Unlock() + customCacheDir = dir +} + +// CacheDir returns the active root directory used for caching. +// If a custom directory was set via SetCacheDir, it is returned. +// Otherwise, it returns /android-binary-transparency. +func CacheDir() (string, error) { + customCacheDirMu.RLock() + defer customCacheDirMu.RUnlock() + if customCacheDir != "" { + return customCacheDir, nil + } + userCacheDir, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(userCacheDir, "android-binary-transparency"), nil +} + +// LogDirFromURL derives a clean relative directory path for the cache based on the log's base URL. +// It uses the URL path, removing any redundant "android/binary_transparency" prefix. +// If the path is empty (e.g. Pixel root or localhost test servers), it provides a sensible fallback. +func LogDirFromURL(logBaseURL string) string { + u, err := url.Parse(logBaseURL) + if err != nil { + h := sha256.Sum256([]byte(logBaseURL)) + return fmt.Sprintf("%x", h[:8]) + } + p := strings.Trim(u.Path, "/") + p = strings.TrimPrefix(p, "android/binary_transparency") + p = strings.Trim(p, "/") + if p == "" { + if strings.Contains(logBaseURL, "developers.google.com") || strings.Contains(logBaseURL, "binary_transparency") { + return "pixel" + } + if u.Host != "" { + return strings.ReplaceAll(u.Host, ":", "_") + } + return "default" + } + return filepath.FromSlash(p) +} + func readCachedInfoFile(logBaseURL string, binaryInfoFilename string, treeSize int64) ([]byte, error) { - cacheDir, err := os.UserCacheDir() + return readCachedInfoFileContext(context.Background(), logBaseURL, binaryInfoFilename, treeSize) +} + +func readCachedInfoFileContext(ctx context.Context, logBaseURL string, binaryInfoFilename string, treeSize int64) ([]byte, error) { + abtCacheDir, err := CacheDir() if err != nil { - slog.Warn("Failed to get user cache dir, falling back to direct download", "error", err) - return readFromURL(logBaseURL, binaryInfoFilename) + slog.Warn("Failed to get cache dir, falling back to direct download", "error", err) + return readFromURLContext(ctx, logBaseURL, binaryInfoFilename) } - abtCacheDir := filepath.Join(cacheDir, "android-binary-transparency") - if err := os.MkdirAll(abtCacheDir, 0755); err != nil { + logDir := LogDirFromURL(logBaseURL) + targetDir := filepath.Join(abtCacheDir, logDir) + if err := os.MkdirAll(targetDir, 0755); err != nil { slog.Warn("Failed to create cache dir, falling back to direct download", "error", err) - return readFromURL(logBaseURL, binaryInfoFilename) + return readFromURLContext(ctx, logBaseURL, binaryInfoFilename) } - urlHash := sha256.Sum256([]byte(logBaseURL)) - basePrefix := fmt.Sprintf("%x_%s", urlHash[:8], binaryInfoFilename) - cacheFilename := fmt.Sprintf("%s_%d", basePrefix, treeSize) - cachePath := filepath.Join(abtCacheDir, cacheFilename) + cacheFilename := fmt.Sprintf("%s_%d", binaryInfoFilename, treeSize) + cachePath := filepath.Join(targetDir, cacheFilename) // Try reading from cache if b, err := os.ReadFile(cachePath); err == nil { @@ -142,13 +213,13 @@ func readCachedInfoFile(logBaseURL string, binaryInfoFilename string, treeSize i // Cache miss, download from URL slog.Info("Downloading new info file", "url", logBaseURL+"/"+binaryInfoFilename) - b, err := readFromURL(logBaseURL, binaryInfoFilename) + b, err := readFromURLContext(ctx, logBaseURL, binaryInfoFilename) if err != nil { return nil, err } // Save to cache atomically - tmpFile, err := os.CreateTemp(abtCacheDir, cacheFilename+".*.tmp") + tmpFile, err := os.CreateTemp(targetDir, cacheFilename+".*.tmp") if err != nil { slog.Warn("Failed to create cache tmp file", "error", err) return b, nil @@ -177,16 +248,16 @@ func readCachedInfoFile(logBaseURL string, binaryInfoFilename string, treeSize i slog.Debug("Saved info file to local cache", "path", cachePath) - // Cleanup old cache files for this specific log URL and filename safely - slog.Info("Cleaning up old cache files", "prefix", basePrefix) - if entries, err := os.ReadDir(abtCacheDir); err == nil { + // Cleanup old cache files for this specific binaryInfoFilename safely + slog.Info("Cleaning up old cache files", "prefix", binaryInfoFilename) + if entries, err := os.ReadDir(targetDir); err == nil { for _, entry := range entries { if entry.IsDir() { continue } - // Only process files that match our specific basePrefix - if !strings.HasPrefix(entry.Name(), basePrefix+"_") { + // Only process files that match our specific binaryInfoFilename prefix + if !strings.HasPrefix(entry.Name(), binaryInfoFilename+"_") { continue } @@ -248,15 +319,24 @@ func parseBinaryInfosIndex(binaryInfos string, binaryInfoFilename string) (map[s } func readFromURL(base, suffix string) ([]byte, error) { + return readFromURLContext(context.Background(), base, suffix) +} + +func readFromURLContext(ctx context.Context, base, suffix string) ([]byte, error) { u, err := url.Parse(base) if err != nil { return nil, fmt.Errorf("invalid URL %s: %v", base, err) } u.Path = path.Join(u.Path, suffix) - resp, err := http.Get(u.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { - return nil, fmt.Errorf("http.Get(%s): %v", u.String(), err) + return nil, fmt.Errorf("http.NewRequestWithContext(%s): %v", u.String(), err) + } + + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("httpClient.Do(%s): %v", u.String(), err) } defer resp.Body.Close() if code := resp.StatusCode; code != 200 { @@ -306,35 +386,26 @@ func ParseEntryBundle(data []byte) ([][]byte, error) { } func readCachedEntryTile(logBaseURL string, tileN int64, w int) ([]byte, error) { + return readCachedEntryTileContext(context.Background(), logBaseURL, tileN, w) +} + +func readCachedEntryTileContext(ctx context.Context, logBaseURL string, tileN int64, w int) ([]byte, error) { entryPath := EntryTilePath(tileN, w) - cacheDir, err := os.UserCacheDir() + abtCacheDir, err := CacheDir() if err != nil { - slog.Warn("Failed to get user cache dir, falling back to direct download", "error", err) - b, err := readFromURL(logBaseURL, entryPath) + slog.Warn("Failed to get cache dir, falling back to direct download", "error", err) + b, err := readFromURLContext(ctx, logBaseURL, entryPath) if err != nil { // Fallback to tiles/entries/ if tile/entries/ fails altPath := "tiles/entries/" + strings.TrimPrefix(entryPath, "tile/entries/") - return readFromURL(logBaseURL, altPath) - } - return b, nil - } - - abtCacheDir := filepath.Join(cacheDir, "android-binary-transparency") - if err := os.MkdirAll(abtCacheDir, 0755); err != nil { - slog.Warn("Failed to create cache dir, falling back to direct download", "error", err) - b, err := readFromURL(logBaseURL, entryPath) - if err != nil { - altPath := "tiles/entries/" + strings.TrimPrefix(entryPath, "tile/entries/") - return readFromURL(logBaseURL, altPath) + return readFromURLContext(ctx, logBaseURL, altPath) } return b, nil } - urlHash := sha256.Sum256([]byte(logBaseURL)) - // TODO: Consider implementing cache cleanup / TTL eviction for older partial entry tiles (w < 256) as the tree grows. - cacheFilename := fmt.Sprintf("%x_entry_tile_%d_%d", urlHash[:8], tileN, w) - cachePath := filepath.Join(abtCacheDir, cacheFilename) + logDir := LogDirFromURL(logBaseURL) + cachePath := filepath.Join(abtCacheDir, logDir, filepath.FromSlash(entryPath)) // Try reading from cache if b, err := os.ReadFile(cachePath); err == nil { @@ -344,18 +415,24 @@ func readCachedEntryTile(logBaseURL string, tileN int64, w int) ([]byte, error) // Cache miss, download from URL slog.Debug("Downloading entry tile", "url", logBaseURL+"/"+entryPath) - b, err := readFromURL(logBaseURL, entryPath) + b, err := readFromURLContext(ctx, logBaseURL, entryPath) if err != nil { altPath := "tiles/entries/" + strings.TrimPrefix(entryPath, "tile/entries/") slog.Debug("Trying alternative entry tile path", "url", logBaseURL+"/"+altPath) - b, err = readFromURL(logBaseURL, altPath) + b, err = readFromURLContext(ctx, logBaseURL, altPath) if err != nil { return nil, err } } // Save to cache atomically - tmpFile, err := os.CreateTemp(abtCacheDir, cacheFilename+".*.tmp") + cacheDir := filepath.Dir(cachePath) + if err := os.MkdirAll(cacheDir, 0755); err != nil { + slog.Warn("Failed to create cache dir, falling back to direct download", "error", err) + return b, nil + } + + tmpFile, err := os.CreateTemp(cacheDir, filepath.Base(cachePath)+".*.tmp") if err != nil { slog.Warn("Failed to create cache tmp file", "error", err) return b, nil @@ -378,9 +455,145 @@ func readCachedEntryTile(logBaseURL string, tileN int64, w int) ([]byte, error) return b, nil } + // When a full tile is written, clean up any stale partial tile directory for this tile index. + // Partial tiles for index N reside in ".p" (e.g. tile/entries/000.p/15). + // Removing the directory is an O(1) targeted eviction that avoids scanning the parent directory. + if w == 256 { + _ = os.RemoveAll(cachePath + ".p") + } + return b, nil } +func isEntryTileCached(logBaseURL string, tileN int64, w int) bool { + abtCacheDir, err := CacheDir() + if err != nil { + return false + } + logDir := LogDirFromURL(logBaseURL) + cachePath := filepath.Join(abtCacheDir, logDir, filepath.FromSlash(EntryTilePath(tileN, w))) + info, err := os.Stat(cachePath) + return err == nil && !info.IsDir() && info.Size() > 0 +} + +// DefaultTesseraFetchConcurrency is the default number of concurrent workers used by FetchAllTesseraEntries. +const DefaultTesseraFetchConcurrency = 16 + +// FetchAllLegacyEntries downloads and caches all specified legacy binary info files +// (e.g. package_info.txt, package_info2.txt) for the given log URL and tree size. +// Files already present in the local cache are loaded without re-downloading. +func FetchAllLegacyEntries(ctx context.Context, logBaseURL string, filenames []string, treeSize int64) error { + if ctx == nil { + ctx = context.Background() + } + if treeSize <= 0 { + return fmt.Errorf("invalid treeSize %d for legacy entries", treeSize) + } + for _, filename := range filenames { + slog.Info("Fetching legacy info file", "url", logBaseURL, "file", filename, "treeSize", treeSize) + _, err := readCachedInfoFileContext(ctx, logBaseURL, filename, treeSize) + if err != nil { + return fmt.Errorf("failed to fetch legacy info file %s: %w", filename, err) + } + } + return nil +} + +// FetchAllTesseraEntries concurrently downloads all entry tiles up to treeSize into the local cache. +// Tiles already cached locally are skipped, making incremental runs fast and idempotent. +// If concurrency <= 0, DefaultTesseraFetchConcurrency is used. +func FetchAllTesseraEntries(ctx context.Context, logBaseURL string, treeSize int64, concurrency int) error { + if ctx == nil { + ctx = context.Background() + } + if treeSize <= 0 { + return nil + } + if concurrency <= 0 { + concurrency = DefaultTesseraFetchConcurrency + } + + numTiles := (treeSize + 255) / 256 + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + type tileTask struct { + tileN int64 + w int + } + + taskChan := make(chan tileTask, concurrency*2) + var wg sync.WaitGroup + var firstErr error + var errOnce sync.Once + var downloadedCount atomic.Int64 + var cachedCount atomic.Int64 + + for i := 0; i < concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for task := range taskChan { + select { + case <-ctx.Done(): + return + default: + } + + if isEntryTileCached(logBaseURL, task.tileN, task.w) { + cachedCount.Add(1) + continue + } + + _, err := readCachedEntryTileContext(ctx, logBaseURL, task.tileN, task.w) + if err != nil { + errOnce.Do(func() { + firstErr = fmt.Errorf("failed to fetch entry tile %d (width %d): %w", task.tileN, task.w, err) + cancel() + }) + return + } + downloaded := downloadedCount.Add(1) + if numTiles > 50 && downloaded%100 == 0 { + slog.Info("Fetching Tessera entry tiles...", "downloaded", downloaded, "total", numTiles) + } + } + }() + } + + for tileN := int64(0); tileN < numTiles; tileN++ { + if ctx.Err() != nil { + break + } + + w := 256 + if (tileN+1)*256 > treeSize { + w = int(treeSize - tileN*256) + } + + select { + case <-ctx.Done(): + case taskChan <- tileTask{tileN: tileN, w: w}: + } + if ctx.Err() != nil { + break + } + } + close(taskChan) + wg.Wait() + + if firstErr != nil { + return firstErr + } + if err := ctx.Err(); err != nil { + return err + } + + slog.Info("Completed Tessera entry tiles fetch", "totalTiles", numTiles, "downloaded", downloadedCount.Load(), "alreadyCached", cachedCount.Load()) + return nil +} + // TesseraFindPayloadIndex searches the Tessera entry tiles for targetPayload // and returns its 0-based sequence index in the log. // Returns (index, true, nil) if found, (-1, false, nil) if not found. diff --git a/verifier_tools/verify/internal/tiles/reader_test.go b/verifier_tools/verify/internal/tiles/reader_test.go index e63eacd..d9881fd 100644 --- a/verifier_tools/verify/internal/tiles/reader_test.go +++ b/verifier_tools/verify/internal/tiles/reader_test.go @@ -3,11 +3,16 @@ package tiles import ( "bytes" "context" + "encoding/binary" "encoding/hex" + "fmt" "io" "log" "net/http" "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" "testing" "github.com/google/go-cmp/cmp" @@ -333,3 +338,347 @@ func TestTesseraFindPayloadIndex(t *testing.T) { t.Errorf("expected found=false for non-existent payload, got %d", idx) } } + +func createTestEntryBundle(entries [][]byte) []byte { + var buf bytes.Buffer + for _, entry := range entries { + var length uint16 = uint16(len(entry)) + binary.Write(&buf, binary.BigEndian, length) + buf.Write(entry) + } + return buf.Bytes() +} + +func TestFetchAllLegacyEntries(t *testing.T) { + // Isolate user cache directory across OSes (macOS uses $HOME/Library/Caches, while Linux uses + // $XDG_CACHE_HOME). This ensures the test runs against a fresh, hermetic cache and does not + // read from or mutate the developer's actual cache. + tempDir := t.TempDir() + t.Setenv("HOME", tempDir) + t.Setenv("XDG_CACHE_HOME", tempDir) + + file1Content := "0\nhash0\nhash_desc0\npkg0\n1\n\n1\nhash1\nhash_desc1\npkg1\n2\n" + file2Content := "2\nhash2\nhash_desc2\npkg2\n3\n" + + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + switch r.URL.Path { + case "/package_info.txt": + w.Write([]byte(file1Content)) + case "/package_info2.txt": + w.Write([]byte(file2Content)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + ctx := context.Background() + + // Error case: invalid treeSize <= 0 + if err := FetchAllLegacyEntries(ctx, server.URL, []string{"package_info.txt"}, 0); err == nil { + t.Errorf("FetchAllLegacyEntries with treeSize 0 expected error, got nil") + } + + // Error case: file not found (404) + if err := FetchAllLegacyEntries(ctx, server.URL, []string{"missing.txt"}, 3); err == nil { + t.Errorf("FetchAllLegacyEntries with missing file expected error, got nil") + } + + // Success case: fetch two legacy files + filenames := []string{"package_info.txt", "package_info2.txt"} + if err := FetchAllLegacyEntries(ctx, server.URL, filenames, 3); err != nil { + t.Fatalf("FetchAllLegacyEntries failed: %v", err) + } + + if got := requests.Load(); got != 3 { // 1 for missing.txt + 2 for valid files + t.Errorf("expected 3 server requests so far, got %d", got) + } + + // Subsequent BinaryInfosIndex calls should hit local cache with NO additional network requests + m1, err := BinaryInfosIndex(server.URL, "package_info.txt", 3) + if err != nil { + t.Fatalf("BinaryInfosIndex(package_info.txt) failed: %v", err) + } + if len(m1) != 2 || m1["hash0\nhash_desc0\npkg0\n1\n"] != 0 { + t.Errorf("unexpected index map from package_info.txt: %+v", m1) + } + + m2, err := BinaryInfosIndex(server.URL, "package_info2.txt", 3) + if err != nil { + t.Fatalf("BinaryInfosIndex(package_info2.txt) failed: %v", err) + } + if len(m2) != 1 || m2["hash2\nhash_desc2\npkg2\n3\n"] != 2 { + t.Errorf("unexpected index map from package_info2.txt: %+v", m2) + } + + // Request count must still be 3 (zero network calls for cache hits) + if got := requests.Load(); got != 3 { + t.Errorf("expected request count to remain 3 after cached index reads, got %d", got) + } +} + +func TestFetchAllTesseraEntries(t *testing.T) { + // Isolate user cache directory across OSes (macOS uses $HOME/Library/Caches, while Linux uses + // $XDG_CACHE_HOME). This ensures the test runs against a fresh, hermetic cache and does not + // read from or mutate the developer's actual cache. + tempDir := t.TempDir() + t.Setenv("HOME", tempDir) + t.Setenv("XDG_CACHE_HOME", tempDir) + + // Build 2 tiles: + // Tile 0: full tile of 256 entries + // Tile 1: partial tile of 10 entries (treeSize = 266) + var tile0Entries [][]byte + for i := 0; i < 256; i++ { + tile0Entries = append(tile0Entries, []byte(fmt.Sprintf("tile0_entry_%d\n", i))) + } + tile0Data := createTestEntryBundle(tile0Entries) + + var tile1Entries [][]byte + for i := 0; i < 10; i++ { + tile1Entries = append(tile1Entries, []byte(fmt.Sprintf("tile1_entry_%d\n", i))) + } + tile1Data := createTestEntryBundle(tile1Entries) + + var tile0Requests atomic.Int64 + var tile1Requests atomic.Int64 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/tile/entries/000": + tile0Requests.Add(1) + w.Write(tile0Data) + case "/tile/entries/001.p/10": + tile1Requests.Add(1) + w.Write(tile1Data) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + ctx := context.Background() + + // treeSize 0: no-op + if err := FetchAllTesseraEntries(ctx, server.URL, 0, 4); err != nil { + t.Fatalf("FetchAllTesseraEntries with treeSize 0 returned error: %v", err) + } + + // Fetch all tiles for treeSize 266 with concurrency 4 + if err := FetchAllTesseraEntries(ctx, server.URL, 266, 4); err != nil { + t.Fatalf("FetchAllTesseraEntries(266) failed: %v", err) + } + + if tile0Requests.Load() != 1 { + t.Errorf("expected 1 request for tile 0, got %d", tile0Requests.Load()) + } + if tile1Requests.Load() != 1 { + t.Errorf("expected 1 request for tile 1, got %d", tile1Requests.Load()) + } + + // TesseraFindPayloadIndex should find payloads in both tiles using the cached tiles + idx0, found0, err := TesseraFindPayloadIndex(server.URL, 266, []byte("tile0_entry_42\n")) + if err != nil || !found0 || idx0 != 42 { + t.Errorf("TesseraFindPayloadIndex tile 0 = (%d, %v, %v), want (42, true, nil)", idx0, found0, err) + } + + idx1, found1, err := TesseraFindPayloadIndex(server.URL, 266, []byte("tile1_entry_5\n")) + if err != nil || !found1 || idx1 != 256+5 { + t.Errorf("TesseraFindPayloadIndex tile 1 = (%d, %v, %v), want (261, true, nil)", idx1, found1, err) + } + + // Second run of FetchAllTesseraEntries for the same treeSize: + // Tile 0 (w=256) is full and cached, so it MUST be skipped! + if err := FetchAllTesseraEntries(ctx, server.URL, 266, 4); err != nil { + t.Fatalf("second FetchAllTesseraEntries failed: %v", err) + } + + if tile0Requests.Load() != 1 { + t.Errorf("expected tile 0 to be skipped on second sync, but requests increased to %d", tile0Requests.Load()) + } + + // Cancellation test: cancelled context should return context error + cancelCtx, cancel := context.WithCancel(context.Background()) + cancel() + err = FetchAllTesseraEntries(cancelCtx, server.URL, 266, 4) + if err == nil { + t.Errorf("expected error on cancelled context, got nil") + } + + // Server error handling: requesting a non-existent tile + err = FetchAllTesseraEntries(ctx, server.URL, 600, 4) // needs tile 2 which is 404 + if err == nil { + t.Errorf("expected error when tile is missing (404), got nil") + } +} + +func TestSetCacheDir(t *testing.T) { + customDir := t.TempDir() + SetCacheDir(customDir) + defer SetCacheDir("") + + got, err := CacheDir() + if err != nil { + t.Fatalf("CacheDir() failed: %v", err) + } + if got != customDir { + t.Errorf("CacheDir() = %q, want %q", got, customDir) + } + + // Verify that FetchAllLegacyEntries writes directly to the custom directory + fileContent := "0\nhash0\nhash_desc0\npkg0\n1\n" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(fileContent)) + })) + defer server.Close() + + if err := FetchAllLegacyEntries(context.Background(), server.URL, []string{"test_pkg.txt"}, 1); err != nil { + t.Fatalf("FetchAllLegacyEntries failed with custom cache dir: %v", err) + } + + // Check that customDir contains the cached file + entries, err := os.ReadDir(customDir) + if err != nil { + t.Fatalf("failed to read custom cache dir: %v", err) + } + if len(entries) == 0 { + t.Errorf("expected cached file in custom directory %s, found none", customDir) + } +} + +func TestLogDirFromURL(t *testing.T) { + tests := []struct { + url string + want string + }{ + { + url: "https://developers.google.com/android/binary_transparency", + want: "pixel", + }, + { + url: "https://developers.google.com/android/binary_transparency/google1p", + want: "google1p", + }, + { + url: "https://www.gstatic.com/android/binary_transparency/google1p/apk/2026/01", + want: filepath.FromSlash("google1p/apk/2026/01"), + }, + { + url: "https://www.gstatic.com/android/binary_transparency/google1p/apk/2026/02", + want: filepath.FromSlash("google1p/apk/2026/02"), + }, + { + url: "https://www.gstatic.com/android/binary_transparency/mainline/2026/01", + want: filepath.FromSlash("mainline/2026/01"), + }, + { + url: "https://www.gstatic.com/android/binary_transparency/mainline/2026/02", + want: filepath.FromSlash("mainline/2026/02"), + }, + { + url: "http://127.0.0.1:8080", + want: "127.0.0.1_8080", + }, + { + url: "https://example.com/custom/shard", + want: filepath.FromSlash("custom/shard"), + }, + } + + for _, tt := range tests { + got := LogDirFromURL(tt.url) + if got != tt.want { + t.Errorf("LogDirFromURL(%q) = %q, want %q", tt.url, got, tt.want) + } + } +} + +func TestTesseraCacheShardingAndTargetedEviction(t *testing.T) { + customDir := t.TempDir() + SetCacheDir(customDir) + defer SetCacheDir("") + + tile0Data := createTestEntryBundle([][]byte{[]byte("tile0_item\n")}) + tile1PartialData := createTestEntryBundle([][]byte{[]byte("tile1_partial\n")}) + tile1FullData := createTestEntryBundle([][]byte{[]byte("tile1_full\n")}) + deepTileData := createTestEntryBundle([][]byte{[]byte("deep_tile_entry\n")}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/tile/entries/000": + w.Write(tile0Data) + case "/tile/entries/001.p/10": + w.Write(tile1PartialData) + case "/tile/entries/001": + w.Write(tile1FullData) + case "/tile/entries/x001/x234/067": + w.Write(deepTileData) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + ctx := context.Background() + logDir := LogDirFromURL(server.URL) + + // 1. Fetch with treeSize 266: + // Tile 0 is full (w=256), Tile 1 is partial (w=10). + if err := FetchAllTesseraEntries(ctx, server.URL, 266, 2); err != nil { + t.Fatalf("FetchAllTesseraEntries(266) failed: %v", err) + } + + // Verify Tile 0 is saved to sharded path: //tile/entries/000 + tile0Path := filepath.Join(customDir, logDir, "tile", "entries", "000") + if info, err := os.Stat(tile0Path); err != nil || info.IsDir() { + t.Errorf("expected full tile 0 at %s, err: %v", tile0Path, err) + } + + // Verify Tile 1 partial is saved to: //tile/entries/001.p/10 + tile1PartialDir := filepath.Join(customDir, logDir, "tile", "entries", "001.p") + tile1PartialFile := filepath.Join(tile1PartialDir, "10") + if info, err := os.Stat(tile1PartialFile); err != nil || info.IsDir() { + t.Errorf("expected partial tile 1 at %s, err: %v", tile1PartialFile, err) + } + + // Verify Tile 1 full tile does NOT exist yet + tile1FullPath := filepath.Join(customDir, logDir, "tile", "entries", "001") + if _, err := os.Stat(tile1FullPath); !os.IsNotExist(err) { + t.Errorf("expected full tile 1 to not exist yet at %s", tile1FullPath) + } + + // 2. Fetch with treeSize 512: + // Tile 1 is now full (w=256). + if err := FetchAllTesseraEntries(ctx, server.URL, 512, 2); err != nil { + t.Fatalf("FetchAllTesseraEntries(512) failed: %v", err) + } + + // Verify Tile 1 full tile now exists + if info, err := os.Stat(tile1FullPath); err != nil || info.IsDir() { + t.Errorf("expected full tile 1 at %s, err: %v", tile1FullPath, err) + } + + // Verify O(1) targeted eviction: partial tile directory 001.p MUST BE REMOVED! + if _, err := os.Stat(tile1PartialDir); !os.IsNotExist(err) { + t.Errorf("expected partial tile directory %s to be evicted, but it still exists", tile1PartialDir) + } + + // 3. Test multi-level sharding (tileN = 1234067, w = 256): + // Path should be tile/entries/x001/x234/067 + // Fetch just that tile directly + b, err := readCachedEntryTileContext(ctx, server.URL, 1234067, 256) + if err != nil { + t.Fatalf("readCachedEntryTileContext for tile 1234067 failed: %v", err) + } + if len(b) == 0 { + t.Fatalf("expected non-empty bytes for tile 1234067") + } + + deepTilePath := filepath.Join(customDir, logDir, "tile", "entries", "x001", "x234", "067") + if info, err := os.Stat(deepTilePath); err != nil || info.IsDir() { + t.Errorf("expected deep sharded tile at %s, err: %v", deepTilePath, err) + } +} +