fix: harden Vary caching and collapsed forwarding

pkg/x/http/request_key.go

@@ -0,0 +1,68 @@
+package http
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	stdhttp "net/http"
+	"sort"
+	"strings"
+)
+
+// HeaderFingerprint returns a deterministic digest of an HTTP header map.
+// It is intended for request collapsing, where requests with different
+// representation-affecting headers must never share an upstream response.
+func HeaderFingerprint(header stdhttp.Header) string {
+	type entry struct {
+		name   string
+		values []string
+	}
+
+	grouped := make(map[string][]entry, len(header))
+	for name, values := range header {
+		lowerName := strings.ToLower(strings.TrimSpace(name))
+		grouped[lowerName] = append(grouped[lowerName], entry{
+			name:   name,
+			values: values,
+		})
+	}
+
+	names := make([]string, 0, len(grouped))
+	for name := range grouped {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	digest := sha256.New()
+	writeSize := func(value uint64) {
+		var size [8]byte
+		binary.BigEndian.PutUint64(size[:], value)
+		_, _ = digest.Write(size[:])
+	}
+	writePart := func(value string) {
+		writeSize(uint64(len(value)))
+		_, _ = digest.Write([]byte(value))
+	}
+
+	writeSize(uint64(len(names)))
+	for _, name := range names {
+		writePart(name)
+
+		entries := grouped[name]
+		sort.Slice(entries, func(i, j int) bool {
+			return entries[i].name < entries[j].name
+		})
+		valueCount := 0
+		for _, item := range entries {
+			valueCount += len(item.values)
+		}
+		writeSize(uint64(valueCount))
+		for _, item := range entries {
+			for _, value := range item.values {
+				writePart(value)
+			}
+		}
+	}
+
+	return hex.EncodeToString(digest.Sum(nil))
+}

pkg/x/http/request_key_test.go

@@ -0,0 +1,36 @@
+package http
+
+import (
+	stdhttp "net/http"
+	"testing"
+)
+
+func TestHeaderFingerprintIsStable(t *testing.T) {
+	first := stdhttp.Header{
+		"User-Agent":      {"test"},
+		"Accept-Encoding": {"gzip"},
+	}
+	second := stdhttp.Header{
+		"accept-encoding": {"gzip"},
+		"user-agent":      {"test"},
+	}
+	if HeaderFingerprint(first) != HeaderFingerprint(second) {
+		t.Fatal("equivalent header maps produced different fingerprints")
+	}
+}
+
+func TestHeaderFingerprintIsolatesRepresentations(t *testing.T) {
+	gzip := stdhttp.Header{"Accept-Encoding": {"gzip"}}
+	brotli := stdhttp.Header{"Accept-Encoding": {"br"}}
+	if HeaderFingerprint(gzip) == HeaderFingerprint(brotli) {
+		t.Fatal("different representation headers produced the same fingerprint")
+	}
+}
+
+func TestHeaderFingerprintEncodesFieldBoundaries(t *testing.T) {
+	first := stdhttp.Header{"A": {"b"}, "C": {"d"}}
+	second := stdhttp.Header{"A": {"b", "c", "d"}}
+	if HeaderFingerprint(first) == HeaderFingerprint(second) {
+		t.Fatal("different header structures produced the same fingerprint")
+	}
+}

pkg/x/http/varycontrol/vary.go

@@ -1,40 +1,82 @@
 package varycontrol
 
 import (
+	"encoding/json"
 	"net/http"
+	"net/textproto"
 	"slices"
 	"sort"
 	"strings"
 )
 
 const (
 	VaryEmptyIdentity = "tr_identity"
+	dataVersion       = "v2:"
 )
 
 type Key []string
 
-func (k *Key) String() string {
-	return strings.Join(*k, ",")
+type selectedHeader struct {
+	Name    string `json:"name"`
+	Present bool   `json:"present"`
+	Value   string `json:"value,omitempty"`
 }
 
-func (k *Key) Append(val string) {
-	keys := canonical(val)
-	if len(keys) > 0 {
-		if slices.Contains(*k, val) {
-			return
+func (k Key) String() string {
+	return strings.Join(k, ",")
+}
+
+// Append adds canonical Vary field names while preserving API compatibility
+// with the original key helper.
+func (k *Key) Append(value string) {
+	for _, key := range Clean(value) {
+		if !slices.Contains(*k, key) {
+			*k = append(*k, key)
+		}
+	}
+}
+
+// HasWildcard reports whether the Vary field contains "*". Such responses
+// must not be reused from a cache for a later request.
+func (k Key) HasWildcard() bool {
+	return slices.Contains(k, "*")
+}
+
+// VaryData serializes the selected request fields without delimiter
+// ambiguity. Header names are case-insensitive, field order is stable, and an
+// absent field remains distinct from a present field with an empty value.
+func (k Key) VaryData(h http.Header) string {
+	if len(k) == 0 {
+		return ""
+	}
+
+	fields := make([]selectedHeader, 0, len(k))
+	for _, key := range k {
+		values, present := headerValues(h, key)
+		for i := range values {
+			values[i] = strings.TrimSpace(values[i])
 		}
-		*k = append(*k, keys...)
+		fields = append(fields, selectedHeader{
+			Name:    strings.ToLower(key),
+			Present: present,
+			Value:   strings.Join(values, ","),
+		})
 	}
+
+	data, _ := json.Marshal(fields)
+	return dataVersion + string(data)
 }
 
-func (k *Key) VaryData(h http.Header) string {
-	l := len(*k)
-	if l <= 0 {
+// LegacyVaryData reproduces the pre-v2 key format. It is retained only so
+// caches written by older Tavern versions remain readable during migration.
+func (k Key) LegacyVaryData(h http.Header) string {
+	l := len(k)
+	if l == 0 {
 		return ""
 	}
 
 	kv := make(map[string]string, l)
-	for _, key := range *k {
+	for _, key := range k {
 		v := sortValues(h.Values(key))
 		kv[key] = v
 	}
@@ -43,71 +85,88 @@ func (k *Key) VaryData(h http.Header) string {
 	for key := range kv {
 		keys = append(keys, key)
 	}
-
 	sort.Strings(keys)
 
 	var buf strings.Builder
 	for _, key := range keys {
-		v := kv[key]
 		if buf.Len() > 0 {
 			buf.WriteByte('&')
 		}
 		buf.WriteString(key)
 		buf.WriteByte('=')
-		buf.WriteString(v)
+		buf.WriteString(kv[key])
 	}
 	return buf.String()
 }
 
-func (k Key) Compare(k2 Key) bool {
-	if len(k) != len(k2) {
-		return false
-	}
-
-	for i, vk1 := range k {
-		if vk1 != k2[i] {
-			return false
-		}
-	}
-	return true
+func (k Key) Compare(other Key) bool {
+	return slices.Equal(k, other)
 }
 
+// Clean parses, canonicalizes, sorts, and de-duplicates Vary field names.
+// Field names are case-insensitive per HTTP semantics.
 func Clean(values ...string) Key {
 	keys := make([]string, 0)
-
-	for _, val := range values {
-		key := canonical(val)
-		if len(key) > 0 {
-			keys = append(keys, key...)
+	for _, value := range values {
+		for _, key := range split(value) {
+			if key == "*" {
+				keys = append(keys, key)
+				continue
+			}
+			keys = append(keys, textproto.CanonicalMIMEHeaderKey(key))
 		}
 	}
 
-	sort.Strings(keys)
+	sort.Slice(keys, func(i, j int) bool {
+		return strings.ToLower(keys[i]) < strings.ToLower(keys[j])
+	})
+	return slices.CompactFunc(keys, strings.EqualFold)
+}
 
+// LegacyClean preserves the original field-name casing used in legacy cache
+// keys while retaining the old sort and de-duplication behavior.
+func LegacyClean(values ...string) Key {
+	keys := make([]string, 0)
+	for _, value := range values {
+		keys = append(keys, split(value)...)
+	}
+	sort.Strings(keys)
 	return slices.Compact(keys)
 }
 
-func canonical(val string) []string {
-	s := strings.TrimSpace(val)
-	if s == "" || s == "," {
+func split(value string) []string {
+	value = strings.TrimSpace(value)
+	if value == "" || value == "," {
 		return nil
 	}
 
-	keys := make([]string, 0)
-
-	vk := strings.Split(s, ",")
-	for _, k := range vk {
-		key := strings.TrimSpace(k)
-		if key != "" {
+	parts := strings.Split(value, ",")
+	keys := make([]string, 0, len(parts))
+	for _, part := range parts {
+		if key := strings.TrimSpace(part); key != "" {
 			keys = append(keys, key)
 		}
 	}
+	return keys
+}
 
-	if len(keys) == 0 {
-		return nil
+func headerValues(h http.Header, name string) ([]string, bool) {
+	matchingNames := make([]string, 0, 1)
+	for key := range h {
+		if strings.EqualFold(key, name) {
+			matchingNames = append(matchingNames, key)
+		}
+	}
+	if len(matchingNames) == 0 {
+		return nil, false
 	}
 
-	return keys
+	sort.Strings(matchingNames)
+	values := make([]string, 0, len(matchingNames))
+	for _, key := range matchingNames {
+		values = append(values, h[key]...)
+	}
+	return values, true
 }
 
 func sortValues(vals []string) string {
@@ -119,9 +178,7 @@ func sortValues(vals []string) string {
 	for _, val := range vals {
 		v = append(v, sortValue(val))
 	}
-
 	sort.Strings(v)
-
 	return strings.Join(v, ",")
 }
 
@@ -131,22 +188,10 @@ func sortValue(val string) string {
 		return ""
 	}
 
-	v := splitTrimSpace(val)
-
-	sort.Strings(v)
-
-	return strings.Join(v, ",")
-}
-
-func splitTrimSpace(s string) []string {
-	if s == "" {
-		return nil
-	}
-
-	v := strings.Split(s, ",")
+	v := strings.Split(val, ",")
 	for i := range v {
 		v[i] = strings.TrimSpace(v[i])
 	}
-
-	return v
+	sort.Strings(v)
+	return strings.Join(v, ",")
 }