From 3786e9d071a093908dfb0251d7b5264e42eb8057 Mon Sep 17 00:00:00 2001 From: Valentin Maerten Date: Sat, 18 Jul 2026 16:05:30 +0200 Subject: [PATCH 1/3] perf: reuse buffer when hashing source files for checksums io.CopyBuffer ignored the pre-allocated buffer because *os.File implements WriterTo: it took the (*os.File).WriteTo path, which allocates a fresh 32KiB buffer per file via os.genericWriteTo. Hashing many small source files thus allocated ~32KiB per file (~640MB for 20,000 files). Wrapping the file in a plain io.Reader forces io.CopyBuffer to reuse the caller's buffer, keeping the loop allocation-free. The checksum value is unchanged. --- internal/fingerprint/sources_checksum.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/fingerprint/sources_checksum.go b/internal/fingerprint/sources_checksum.go index 3d34136989..23a29b51dc 100644 --- a/internal/fingerprint/sources_checksum.go +++ b/internal/fingerprint/sources_checksum.go @@ -88,6 +88,10 @@ func (*ChecksumChecker) Kind() string { return "checksum" } +// readerOnly hides any WriterTo/ReaderFrom implementation of the wrapped +// reader, forcing io.CopyBuffer to use the caller-provided buffer. +type readerOnly struct{ io.Reader } + func (c *ChecksumChecker) checksum(t *ast.Task) (string, error) { sources, err := Globs(t.Dir, t.Sources, t.ShouldUseGitignore()) if err != nil { @@ -101,14 +105,18 @@ func (c *ChecksumChecker) checksum(t *ast.Task) (string, error) { if _, err := io.CopyBuffer(h, strings.NewReader(filepath.Base(f)), buf); err != nil { return "", err } - f, err := os.Open(f) + file, err := os.Open(f) if err != nil { return "", err } - if _, err = io.CopyBuffer(h, f, buf); err != nil { + // Wrap the file in a plain io.Reader so io.CopyBuffer cannot take the + // (*os.File).WriteTo fast path, which ignores buf and allocates a fresh + // 32KiB buffer for every file. Reusing buf keeps this loop allocation-free. + if _, err = io.CopyBuffer(h, readerOnly{file}, buf); err != nil { + file.Close() return "", err } - f.Close() + file.Close() } hash := h.Sum128() From 867e26a89d2f36bbd39495dd3ba07bab8a49cf56 Mon Sep 17 00:00:00 2001 From: Valentin Maerten Date: Mon, 3 Aug 2026 22:25:12 +0200 Subject: [PATCH 2/3] Trigger Build From de60f446d3131f8a606248f35fd740a9f6bc9195 Mon Sep 17 00:00:00 2001 From: Valentin Maerten Date: Mon, 3 Aug 2026 22:26:38 +0200 Subject: [PATCH 3/3] docs: add changelog entry for checksum buffer reuse --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72ec5c474f..75a230703c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ - Added support for `enum.ref` in `--interactive` prompts. Required vars using `enum.ref` now show the selection list like static enums, instead of falling back to free-form input (#2817 by @vmaerten). +- Further improved fingerprinting performance on large repositories: hashing + source files now reuses a single buffer, reducing memory allocations by ~98% + and wall-clock time by ~7% (#2925 by @vmaerten). ## v3.52.0 - 2026-07-02