feat(cache): add Cloudflare SigV4 S3 signature compatibility fix and compile locally

2026-05-19 21:28:45 -07:00
parent c4227a0eb1
commit 6e45f50874
18 changed files with 2529 additions and 2 deletions
--- a/go-cache-plugin-src/cmd/go-cache-plugin/addca_default.go
+++ b/go-cache-plugin-src/cmd/go-cache-plugin/addca_default.go
@@ -0,0 +1,29 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !linux
+
+package main
+
+import (
+	"errors"
+	"log"
+
+	"github.com/creachadair/atomicfile"
+	"github.com/creachadair/command"
+	"github.com/creachadair/tlsutil"
+)
+
+func installSigningCert(env *command.Env, cert tlsutil.Certificate) error {
+	const certFile = "revproxy-ca.crt"
+	if err := atomicfile.WriteData(certFile, cert.CertPEM(), 0644); err != nil {
+		log.Printf("WARNING: Unable to write cert file: %v", err)
+	} else {
+		log.Printf("Wrote signing cert to %s", certFile)
+	}
+	// TODO(creachadair): Maybe crib some other cases from mkcert, if we need
+	// them, for example:
+	// https://github.com/FiloSottile/mkcert/blob/master/truststore_darwin.go
+
+	return errors.New("unable to install a certificate on this system")
+}
--- a/go-cache-plugin-src/cmd/go-cache-plugin/addca_linux.go
+++ b/go-cache-plugin-src/cmd/go-cache-plugin/addca_linux.go
@@ -0,0 +1,39 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/creachadair/command"
+	"github.com/creachadair/tlsutil"
+	"golang.org/x/sys/unix"
+)
+
+func installSigningCert(env *command.Env, cert tlsutil.Certificate) error {
+	const ubuntuCertFile = "/etc/ssl/certs/ca-certificates.crt"
+	return lockAndAppend(ubuntuCertFile, cert.CertPEM())
+}
+
+// lockAndAppend acquires an exclusive advisory lock on path, if possible, and
+// appends data to the end of it. It reports an error if path does not exist,
+// or if the lock could not be acquired. The lock is automatically released
+// before returning.
+func lockAndAppend(path string, data []byte) error {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0)
+	if err != nil {
+		return err
+	}
+	fd := int(f.Fd())
+	if err := unix.Flock(fd, unix.LOCK_EX); err != nil {
+		f.Close()
+		return fmt.Errorf("lock: %w", err)
+	}
+	defer unix.Flock(fd, unix.LOCK_UN)
+	_, werr := f.Write(data)
+	cerr := f.Close()
+	return errors.Join(werr, cerr)
+}
--- a/go-cache-plugin-src/cmd/go-cache-plugin/commands.go
+++ b/go-cache-plugin-src/cmd/go-cache-plugin/commands.go
@@ -0,0 +1,213 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+
+	"github.com/creachadair/command"
+	"github.com/creachadair/gocache"
+	"github.com/creachadair/taskgroup"
+)
+
+var flags struct {
+	CacheDir      string        `flag:"cache-dir,default=$GOCACHE_DIR,Local cache directory (required)"`
+	S3Bucket      string        `flag:"bucket,default=$GOCACHE_S3_BUCKET,S3 bucket name (required)"`
+	S3Region      string        `flag:"region,default=$GOCACHE_S3_REGION,S3 region"`
+	S3Endpoint    string        `flag:"s3-endpoint-url,default=$GOCACHE_S3_ENDPOINT_URL,S3 custom endpoint URL (if unset, use AWS default)"`
+	S3PathStyle   bool          `flag:"s3-path-style,default=$GOCACHE_S3_PATH_STYLE,S3 path-style URLs (optional)"`
+	KeyPrefix     string        `flag:"prefix,default=$GOCACHE_KEY_PREFIX,S3 key prefix (optional)"`
+	MinUploadSize int64         `flag:"min-upload-size,default=$GOCACHE_MIN_SIZE,Minimum object size to upload to S3 (in bytes)"`
+	Concurrency   int           `flag:"c,default=$GOCACHE_CONCURRENCY,Maximum number of concurrent requests"`
+	S3Concurrency int           `flag:"u,default=$GOCACHE_S3_CONCURRENCY,Maximum concurrency for upload to S3"`
+	PrintMetrics  bool          `flag:"metrics,default=$GOCACHE_METRICS,Print summary metrics to stderr at exit"`
+	Expiration    time.Duration `flag:"expiry,default=$GOCACHE_EXPIRY,Cache expiration period (optional)"`
+	Verbose       bool          `flag:"v,default=$GOCACHE_VERBOSE,Enable verbose logging"`
+	DebugLog      int           `flag:"debug,default=$GOCACHE_DEBUG,Enable detailed per-request debug logging (noisy)"`
+}
+
+const (
+	debugBuildCache = 1 << iota
+	debugModProxy
+	debugRevProxy
+)
+
+// runDirect runs a cache communicating on stdin/stdout, for use as a direct
+// GOCACHEPROG plugin.
+func runDirect(env *command.Env) error {
+	s, _, err := initCacheServer(env)
+	if err != nil {
+		return err
+	}
+	if err := s.Run(env.Context(), os.Stdin, os.Stdout); err != nil {
+		return fmt.Errorf("cache server exited with error: %w", err)
+	}
+	if flags.Verbose || flags.PrintMetrics {
+		fmt.Fprintln(os.Stderr, s.Metrics())
+	}
+	return nil
+}
+
+var serveFlags struct {
+	Plugin   int    `flag:"plugin,default=$GOCACHE_PLUGIN,Plugin service port (required)"`
+	HTTP     string `flag:"http,default=$GOCACHE_HTTP,HTTP service address ([host]:port)"`
+	ModProxy bool   `flag:"modproxy,default=$GOCACHE_MODPROXY,Enable a Go module proxy (requires --http)"`
+	RevProxy string `flag:"revproxy,default=$GOCACHE_REVPROXY,Reverse proxy these hosts (comma-separated; requires --http)"`
+	SumDB    string `flag:"sumdb,default=$GOCACHE_SUMDB,SumDB servers to proxy for (comma-separated)"`
+}
+
+func noopClose(context.Context) error { return nil }
+
+// runServe runs a cache communicating over a local TCP socket.
+func runServe(env *command.Env) error {
+	if serveFlags.Plugin <= 0 {
+		return env.Usagef("you must provide a --plugin port")
+	}
+
+	// Initialize the cache server. Unlike a direct server, only close down and
+	// wait for cache cleanup when the whole process exits.
+	s, s3c, err := initCacheServer(env)
+	if err != nil {
+		return err
+	}
+	closeHook := s.Close
+	s.Close = noopClose
+
+	// Listen for connections from the Go toolchain on the specified socket.
+	lst, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", serveFlags.Plugin))
+	if err != nil {
+		return fmt.Errorf("listen: %w", err)
+	}
+	log.Printf("plugin listening at %q", lst.Addr())
+
+	ctx, cancel := signal.NotifyContext(env.Context(), syscall.SIGINT, syscall.SIGTERM)
+	defer cancel()
+
+	var g taskgroup.Group
+	g.Run(func() {
+		<-ctx.Done()
+		log.Printf("closing plugin listener")
+		lst.Close()
+	})
+
+	// If a module proxy is enabled, start it.
+	modProxy, modCleanup, err := initModProxy(env.SetContext(ctx), s3c)
+	if err != nil {
+		lst.Close()
+		return fmt.Errorf("module proxy: %w", err)
+	}
+	defer modCleanup()
+
+	// If a reverse proxy is enabled, start it.
+	revProxy, err := initRevProxy(env.SetContext(ctx), s3c, &g)
+	if err != nil {
+		lst.Close()
+		return fmt.Errorf("reverse proxy: %w", err)
+	}
+
+	// If an HTTP server is enabled, start it up with debug routes
+	// and whatever other services were requested.
+	if serveFlags.HTTP != "" {
+		srv := &http.Server{
+			Addr:    serveFlags.HTTP,
+			Handler: makeHandler(modProxy, revProxy),
+		}
+		g.Go(srv.ListenAndServe)
+		vprintf("HTTP server listening at %q", serveFlags.HTTP)
+		g.Run(func() {
+			<-ctx.Done()
+			vprintf("stopping HTTP service")
+			srv.Shutdown(context.Background())
+		})
+	}
+
+	for {
+		conn, err := lst.Accept()
+		if err != nil {
+			if !errors.Is(err, net.ErrClosed) {
+				log.Printf("accept failed: %v, exiting server loop", err)
+			}
+			break
+		}
+		log.Printf("new client connection")
+		g.Go(func() error {
+			defer func() {
+				log.Printf("client connection closed")
+				conn.Close()
+			}()
+			return s.Run(ctx, conn, conn)
+		})
+	}
+	log.Printf("server loop exited, waiting for client exit")
+	g.Wait()
+	if closeHook != nil {
+		ctx := gocache.WithLogf(context.Background(), log.Printf)
+		if err := closeHook(ctx); err != nil {
+			log.Printf("server close: %v (ignored)", err)
+		}
+	}
+	return nil
+}
+
+// runConnect implements a direct cache proxy by connecting to a remote server.
+func runConnect(env *command.Env, plugin string) error {
+	port, err := strconv.Atoi(plugin)
+	if err != nil {
+		return fmt.Errorf("invalid plugin port: %w", err)
+	}
+
+	conn, err := net.Dial("tcp", fmt.Sprintf(":%d", port))
+	if err != nil {
+		return fmt.Errorf("dial: %w", err)
+	}
+	start := time.Now()
+	vprintf("connected to %q", conn.RemoteAddr())
+
+	out := taskgroup.Go(func() error {
+		defer conn.(*net.TCPConn).CloseWrite() // let the server finish
+		return copy(conn, os.Stdin)
+	})
+	if rerr := copy(os.Stdout, conn); rerr != nil {
+		vprintf("read responses: %v", err)
+	}
+	out.Wait()
+	conn.Close()
+	vprintf("connection closed (%v elapsed)", time.Since(start))
+	return nil
+}
+
+// copy emulates the base case of io.Copy, but does not attempt to use the
+// io.ReaderFrom or io.WriterTo implementations.
+//
+// TODO(creachadair): For some reason io.Copy does not work correctly when r is
+// a pipe (e.g., stdin) and w is a TCP socket. Figure out why.
+func copy(w io.Writer, r io.Reader) error {
+	var buf [4096]byte
+	for {
+		nr, err := r.Read(buf[:])
+		if nr > 0 {
+			if nw, err := w.Write(buf[:nr]); err != nil {
+				return fmt.Errorf("copy to: %w", err)
+			} else if nw < nr {
+				return fmt.Errorf("wrote %d < %d bytes: %w", nw, nr, io.ErrShortWrite)
+			}
+		}
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return fmt.Errorf("copy from: %w", err)
+		}
+	}
+}
--- a/go-cache-plugin-src/cmd/go-cache-plugin/go-cache-plugin.go
+++ b/go-cache-plugin-src/cmd/go-cache-plugin/go-cache-plugin.go
@@ -0,0 +1,101 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Program gocache implements the experimental GOCACHEPROG protocol over an S3
+// bucket, for use in builder and CI workers.
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+
+	"github.com/creachadair/command"
+	"github.com/creachadair/flax"
+	"github.com/tailscale/go-cache-plugin/lib/s3util"
+)
+
+func main() {
+	log.SetFlags(log.Ltime | log.Lmicroseconds)
+	root := &command.C{
+		Name:  command.ProgramName(),
+		Usage: "--cache-dir d --bucket b [options]\nhelp",
+		Help: `Run a cache service for the Go toolchain backed by an S3 bucket.
+
+This program serves the Go toolchain cache protocol on stdin/stdout.
+It is meant to be run by the "go" tool as a GOCACHEPROG plugin.
+For example:
+
+    GOCACHEPROG=` + command.ProgramName() + ` go build ./...
+
+Note that Go toolchains prior to 1.24 must be built with GOEXPERIMENT=cacheprog
+to enable this integration. Go 1.24 and later have it enabled by default.
+
+You must provide --cache-dir, --bucket, and --region, or the corresponding
+environment variables (see "help environment").  Entries in the cache are
+stored in the specified S3 bucket, and staged in a local directory specified by
+the --cache-dir flag or GOCACHE_DIR environment.`,
+
+		SetFlags: command.Flags(flax.MustBind, &flags),
+		Run:      command.Adapt(runDirect),
+
+		Commands: []*command.C{
+			{
+				Name:  "serve",
+				Usage: "--plugin <port>",
+				Help: `Run a cache server.
+
+In this mode, the cache server listens for connections on a socket instead of
+serving directly over stdin/stdout. The "connect" command adapts the direct
+interface to this one.
+
+By default, only the build cache is exported via the --plugin port.
+
+If --http is set, the server also exports an HTTP server at that address.
+By default, this exports only /debug endpoints, including metrics.
+When --http is enabled, the following options are available:
+
+- When --modcache is true, the server also exports a caching module proxy at
+  http://<host>:<port>/mod/.
+
+- When --revproxy is set, the server also hosts a caching reverse proxy for the
+  specified hosts at http://<host>:<port>. The reverse proxy handles both HTTP
+  and HTTPS requests, and caches immutable successful responses.`,
+
+				SetFlags: command.Flags(flax.MustBind, &serveFlags),
+				Run:      command.Adapt(runServe),
+			},
+			{
+				Name:  "connect",
+				Usage: "<port>",
+				Help: `Connect to a remote cache server.
+
+This mode bridges stdin/stdout to a cache server (see the "serve" command)
+listening on the specified port.`,
+
+				Run: command.Adapt(runConnect),
+			},
+			command.HelpCommand(helpTopics),
+			command.VersionCommand(),
+		},
+	}
+	command.RunOrFail(root.NewEnv(nil), os.Args[1:])
+}
+
+// getBucketRegion reports the specified region for the given bucket.
+// if the --region flag was set, that value is returned without error.
+// Otherwise, it queries the GetBucketLocation API.
+func getBucketRegion(ctx context.Context, bucket string) (string, error) {
+	if flags.S3Region != "" {
+		return flags.S3Region, nil
+	}
+	return s3util.BucketRegion(ctx, bucket)
+}
+
+// vprintf acts as log.Printf if the --verbose flag is set; otherwise it
+// discards its input.
+func vprintf(msg string, args ...any) {
+	if flags.Verbose || flags.DebugLog != 0 {
+		log.Printf(msg, args...)
+	}
+}
--- a/go-cache-plugin-src/cmd/go-cache-plugin/help.go
+++ b/go-cache-plugin-src/cmd/go-cache-plugin/help.go
@@ -0,0 +1,153 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import "github.com/creachadair/command"
+
+var helpTopics = []command.HelpTopic{
+	{
+		Name: "configure",
+		Help: `How to configure the plugin.
+
+To run the plugin, install the program somewhere on your system and set the
+GOCACHEPROG environment variable to the command line of the plugin. You can
+either specify the full path to the program, or install it in your $PATH.
+
+Parameters can be passed either as flags or via environment variables.
+See also "help environment".
+
+The plugin requires credentials to access S3. If you are running in AWS, it can
+get credentials from the instance metadata service; otherwise you will need to
+plumb AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
+AWS_ENDPOINT_URL) or set up a configuration file.
+
+See also: "help environment".
+Related:  "direct-mode", "serve-mode", "module-proxy", "reverse-proxy".`,
+	},
+	{
+		Name: "environment",
+		Help: `Environment variables understood by this program.
+
+To make it easier to configure this tool for multiple workflows, most of the
+settings can be set via environment variables as well as flags.
+
+   --------------------------------------------------------------------
+   Flag (global)        Variable                 Format      Default
+   --------------------------------------------------------------------
+    --cache-dir         GOCACHE_DIR              path        (required)
+    --bucket            GOCACHE_S3_BUCKET        string      (required)
+    --region            GOCACHE_S3_REGION        string      based on bucket
+    --s3-path-style     GOCACHE_S3_PATH_STYLE    bool        false
+    --s3-endpoint-url   GOCACHE_S3_ENDPOINT_URL  string      ""
+    --prefix            GOCACHE_KEY_PREFIX       string      ""
+    --min-upload-size   GOCACHE_MIN_SIZE         int64       0
+    --metrics           GOCACHE_METRICS          bool        false
+    --expiry            GOCACHE_EXPIRY           duration    0
+    -c                  GOCACHE_CONCURRENCY      int         runtime.NumCPU
+    -u                  GOCACHE_S3_CONCURRENCY   duration    runtime.NumCPU
+    -v                  GOCACHE_VERBOSE          bool        false
+    --debug             GOCACHE_DEBUG            int         0 (see "help debug")
+
+   --------------------------------------------------------------------
+   Flag (serve)         Variable                 Format      Default
+   --------------------------------------------------------------------
+    --plugin            GOCACHE_PLUGIN           port        (required)
+    --http              GOCACHE_HTTP             [host]:port ""
+    --modproxy          GOCACHE_MODPROXY         bool        false
+    --revproxy          GOCACHE_REVPROXY         host,...    ""
+    --sumdb             GOCACHE_SUMDB            host,...    ""
+
+See also: "help configure".`,
+	},
+	{
+		Name: "direct-mode",
+		Help: `Run the plugin directly as a subprocess of the toolchain.
+
+  export GOCACHEPROG="go-cache-plugin --cache-dir=/tmp/gocache --bucket ..."
+  go build ...
+
+Alternatively:
+
+  export GOCACHE_DIR=/tmp/gocache
+  export GOCACHE_S3_BUCKET=cache-bucket-name
+  export GOCACHEPROG=go-cache-plugin
+  go build ...
+
+In this mode, you must specify the --cache-dir and --bucket settings.`,
+	},
+	{
+		Name: "serve-mode",
+		Help: `Run the plugin as a standalone service.
+
+Use the "serve" subcommand:
+
+  go-cache-plugin serve \
+     --cache-dir=/tmp/gocache --bucket=$B \
+     --plugin $PORT
+
+You can then use the "connect" subcommand to wire up the toolchain:
+
+  export GOCACHEPROG="go-cache-plugin connect $PORT"
+
+In this mode, the server must have credentials to access to S3, but the
+toolchain process does not need AWS credentials.`,
+	},
+	{
+		Name: "module-proxy",
+		Help: `Run a Go module and sum database proxy.
+
+With the --modproxy flag, the server will also export an HTTP proxy for the
+public Go module proxy (proxy.golang.org) and sum DB (sum.golang.org) at the
+given address:
+
+   go-cache-plugin serve ... --http=localhost:5970 --modproxy
+
+To use the module proxy, set the standard GOPROXY environment variable.
+The module proxy serves under the path "/mod/":
+
+   export GOPROXY=http://localhost:5970/mod
+   export GOCACHEPROG="go-cache-plugin connect $PORT"
+   go build ...
+
+To use the sum DB proxy, set the GOSUMDB environment variable.
+The proxy path for a given sumdb (e.g., sum.golang.org) has this format:
+
+   export GOSUMDB="sum.golang.org http://localhost:5970/mod/sumdb/sum.golang.org"
+
+See also: https://proxy.golang.org/`,
+	},
+	{
+		Name: "reverse-proxy",
+		Help: `Run a caching reverse proxy.
+
+With the --revproxy flag, the server will also export a caching reverse
+proxy for the specified hosts, given as a comma-separated list:
+
+   go-cache-plugin serve ... \
+      --http=localhost:5970 \
+      --revproxy='api.example.com,www.example.com'
+
+When this is enabled, you can configure this address as an HTTP proxy:
+
+   HTTPS_PROXY=localhost:5970 curl https://api.example.com/foo
+
+The proxy supports both HTTP and HTTPS backends. For HTTPS proxy targets, the
+server generates its own TLS certificate, and tries to install a custom signing
+cert so that other tools will validate it. The ability to do this varies by
+system and configuration, however.`,
+	},
+	{
+		Name: "debug",
+		Help: `Enable detailed debug logging.
+
+The --debug flag enables (very) verbose debug logging for the components of the
+cache plugin. The value of the flag is a bit mask of:
+
+   1:  Go build cache
+   2:  Go module proxy and sum database
+   4:  HTTP reverse proxy
+
+The default is 0 (no debug logging).`,
+	},
+}
--- a/go-cache-plugin-src/cmd/go-cache-plugin/setup.go
+++ b/go-cache-plugin-src/cmd/go-cache-plugin/setup.go
@@ -0,0 +1,319 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"errors"
+	"expvar"
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go/middleware"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+	"github.com/creachadair/command"
+	"github.com/creachadair/gocache"
+	"github.com/creachadair/gocache/cachedir"
+	"github.com/creachadair/mhttp/proxyconn"
+	"github.com/creachadair/taskgroup"
+	"github.com/creachadair/tlsutil"
+	"github.com/goproxy/goproxy"
+	"github.com/tailscale/go-cache-plugin/lib/gobuild"
+	"github.com/tailscale/go-cache-plugin/lib/modproxy"
+	"github.com/tailscale/go-cache-plugin/lib/revproxy"
+	"github.com/tailscale/go-cache-plugin/lib/s3util"
+	"tailscale.com/tsweb"
+)
+
+func initCacheServer(env *command.Env) (*gocache.Server, *s3util.Client, error) {
+	switch {
+	case flags.CacheDir == "":
+		return nil, nil, env.Usagef("you must provide a --cache-dir")
+	case flags.S3Bucket == "":
+		return nil, nil, env.Usagef("you must provide an S3 --bucket name")
+	}
+	region, err := getBucketRegion(env.Context(), flags.S3Bucket)
+	if err != nil {
+		return nil, nil, env.Usagef("you must provide an S3 --region name")
+	}
+
+	dir, err := cachedir.New(flags.CacheDir)
+	if err != nil {
+		return nil, nil, fmt.Errorf("create local cache: %w", err)
+	}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(region),
+		config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired),
+	}
+	if flags.S3Endpoint != "" {
+		vprintf("S3 endpoint URL: %s", flags.S3Endpoint)
+		opts = append(opts, config.WithBaseEndpoint(flags.S3Endpoint))
+	}
+	cfg, err := config.LoadDefaultConfig(env.Context(), opts...)
+	if err != nil {
+		return nil, nil, fmt.Errorf("load AWS config: %w", err)
+	}
+
+	vprintf("local cache directory: %s", flags.CacheDir)
+	vprintf("S3 cache bucket %q (%s)", flags.S3Bucket, region)
+	client := &s3util.Client{
+		Client: s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.UsePathStyle = flags.S3PathStyle
+			
+			// Cloudflare SigV4 Compatibility Fix
+			o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+			o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
+			o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error {
+				return stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc("RemoveAcceptEncoding", 
+					func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (
+						middleware.FinalizeOutput, middleware.Metadata, error,
+					) {
+						if req, ok := in.Request.(*smithyhttp.Request); ok {
+							req.Header.Del("Accept-Encoding")
+							req.Header.Del("Amz-Sdk-Invocation-Id")
+							req.Header.Del("Amz-Sdk-Request")
+						}
+						return next.HandleFinalize(ctx, in)
+					}), "Signing", middleware.Before)
+			})
+		}),
+		Bucket: flags.S3Bucket,
+	}
+	cache := &gobuild.S3Cache{
+		Local:             dir,
+		S3Client:          client,
+		KeyPrefix:         flags.KeyPrefix,
+		MinUploadSize:     flags.MinUploadSize,
+		UploadConcurrency: flags.S3Concurrency,
+	}
+	cache.SetMetrics(env.Context(), expvar.NewMap("gocache_host"))
+
+	close := cache.Close
+	if flags.Expiration > 0 {
+		dirClose := dir.Cleanup(flags.Expiration)
+		close = func(ctx context.Context) error {
+			return errors.Join(cache.Close(ctx), dirClose(ctx))
+		}
+	}
+	s := &gocache.Server{
+		Get:         cache.Get,
+		Put:         cache.Put,
+		Close:       close,
+		SetMetrics:  cache.SetMetrics,
+		MaxRequests: flags.Concurrency,
+		Logf:        vprintf,
+		LogRequests: flags.DebugLog&debugBuildCache != 0,
+	}
+	expvar.Publish("gocache_server", s.Metrics().Get("server"))
+	return s, client, nil
+}
+
+// initModProxy initializes a Go module proxy if one is enabled. If not, it
+// returns a nil handler without error. The caller must defer a call to the
+// cleanup function unless an error is reported.
+func initModProxy(env *command.Env, s3c *s3util.Client) (_ http.Handler, cleanup func(), _ error) {
+	if !serveFlags.ModProxy {
+		return nil, noop, nil // OK, proxy is disabled
+	} else if serveFlags.HTTP == "" {
+		return nil, nil, env.Usagef("you must set --http to enable --modproxy")
+	}
+
+	modCachePath := filepath.Join(flags.CacheDir, "module")
+	if err := os.MkdirAll(modCachePath, 0755); err != nil {
+		return nil, nil, fmt.Errorf("create module cache: %w", err)
+	}
+	cacher := &modproxy.S3Cacher{
+		Local:       modCachePath,
+		S3Client:    s3c,
+		KeyPrefix:   path.Join(flags.KeyPrefix, "module"),
+		MaxTasks:    flags.S3Concurrency,
+		Logf:        vprintf,
+		LogRequests: flags.DebugLog&debugModProxy != 0,
+	}
+	cleanup = func() { vprintf("close cacher (err=%v)", cacher.Close()) }
+	proxy := &goproxy.Goproxy{
+		Fetcher: &goproxy.GoFetcher{
+			// As configured, the fetcher should never shell out to the go
+			// tool. Specifically, because we set GOPROXY and do not set any
+			// bypass via GONOPROXY, GOPRIVATE, etc., we will only attempt to
+			// proxy for the specific server(s) listed in Env.
+			GoBin: "/bin/false",
+			Env:   []string{"GOPROXY=https://proxy.golang.org"},
+		},
+		Cacher:        cacher,
+		ProxiedSumDBs: []string{"sum.golang.org"}, // default, see below
+	}
+	vprintf("enabling Go module proxy")
+	if serveFlags.SumDB != "" {
+		proxy.ProxiedSumDBs = strings.Split(serveFlags.SumDB, ",")
+		vprintf("enabling sum DB proxy for %s", strings.Join(proxy.ProxiedSumDBs, ", "))
+	}
+	expvar.Publish("modcache", cacher.Metrics())
+	return http.StripPrefix("/mod", proxy), cleanup, nil
+}
+
+// initRevProxy initializes a reverse proxy if one is enabled.  If not, it
+// returns nil, nil to indicate a proxy was not requested. Otherwise, it
+// returns a [http.Handler] to dispatch reverse proxy requests.
+//
+// The reverse proxy runs two collaborating HTTP servers:
+//
+//   - The "inner" server is the proxy itself, which checks for cached values,
+//     forwards client requests to the remote origin (if necessary), and
+//     updates the cache with responses. The [revproxy.Server] is a lightweight
+//     wrapper around [net/http/httputil.ReverseProxy].
+//
+//   - The "outer" server is a bridge, that intercepts client requests.  The
+//     bridge forwards plain HTTP requests directly to the inner server.  For
+//     HTTPS CONNECT requests, the bridge hijacks the client connection and
+//     terminates TLS using a locally-signed certificate, and forwards the
+//     decrypted client requests to the inner caching proxy.
+//
+// The outer bridge is what receives requests routed by the main HTTP endpoint;
+// the inner server gets all its input via the bridge:
+//
+//	                          +------------+    +--------+
+//	client --[proxy-request]->|HTTP handler+--->| bridge +--CONNECT--+
+//	                          +------------+    +---+----+           |
+//	                                                |                |
+//	                                               HTTP              v
+//	                          +-------------+       |        +---------------+
+//	            [response]<---| cache proxy |<------+--------+ terminate TLS |
+//	                          +-------------+                +---------------+
+//
+// To the main HTTP listener, the bridge is an [http.Handler] that serves
+// requests routed to it. To the inner server, the bridge is a [net.Listener],
+// a source of client connections (with TLS terminated).
+func initRevProxy(env *command.Env, s3c *s3util.Client, g *taskgroup.Group) (http.Handler, error) {
+	if serveFlags.RevProxy == "" {
+		return nil, nil // OK, proxy is disabled
+	} else if serveFlags.HTTP == "" {
+		return nil, env.Usagef("you must set --http to enable --revproxy")
+	}
+
+	revCachePath := filepath.Join(flags.CacheDir, "revproxy")
+	if err := os.MkdirAll(revCachePath, 0755); err != nil {
+		return nil, fmt.Errorf("create revproxy cache: %w", err)
+	}
+	hosts := strings.Split(serveFlags.RevProxy, ",")
+
+	// Issue a server certificate so we can proxy HTTPS requests.
+	cert, err := initServerCert(env, hosts)
+	if err != nil {
+		return nil, err
+	}
+
+	proxy := &revproxy.Server{
+		Targets:     hosts,
+		Local:       revCachePath,
+		S3Client:    s3c,
+		KeyPrefix:   path.Join(flags.KeyPrefix, "revproxy"),
+		Logf:        vprintf,
+		LogRequests: flags.DebugLog&debugRevProxy != 0,
+	}
+	bridge := &proxyconn.Bridge{
+		Addrs:   hosts,
+		Handler: proxy, // forward HTTP requests unencrypted to the proxy
+		Logf:    vprintf,
+
+		// Forward connections not matching Addrs directly to their targets.
+		ForwardConnect: true,
+	}
+	expvar.Publish("proxyconn", bridge.Metrics())
+
+	// Run the proxy on its own separate server with TLS support.  This server
+	// does not listen on a real network; it receives connections forwarded by
+	// the bridge internally from successful CONNECT requests.
+	psrv := &http.Server{
+		TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}},
+
+		// Ordinarly HTTP proxy requests are delegated directly.
+		Handler: proxy,
+	}
+	g.Go(func() error { return psrv.ServeTLS(bridge, "", "") })
+
+	g.Run(func() {
+		<-env.Context().Done()
+		vprintf("stopping proxy bridge")
+		psrv.Shutdown(context.Background())
+	})
+
+	expvar.Publish("revcache", proxy.Metrics())
+	vprintf("enabling reverse proxy for %s", strings.Join(proxy.Targets, ", "))
+	return bridge, nil
+}
+
+// initServerCert creates a signed certificate advertising the specified host
+// names, for use in creating a TLS server.
+func initServerCert(env *command.Env, hosts []string) (tls.Certificate, error) {
+	ca, err := tlsutil.NewSigningCert(24*time.Hour, &x509.Certificate{
+		Subject: pkix.Name{Organization: []string{"Tailscale build automation"}},
+	})
+	if err != nil {
+		return tls.Certificate{}, fmt.Errorf("generate signing cert: %w", err)
+	}
+	if err := installSigningCert(env, ca); err != nil {
+		vprintf("WARNING: %v", err)
+	} else {
+		vprintf("installed signing cert in system store")
+
+		// TODO(creachadair): We should probably clean up old expired certs.
+		// This is OK for ephemeral build/CI workers, though.
+	}
+
+	sc, err := tlsutil.NewServerCert(24*time.Hour, ca, &x509.Certificate{
+		Subject:  pkix.Name{Organization: []string{"Go cache plugin reverse proxy"}},
+		DNSNames: hosts,
+	})
+	if err != nil {
+		return tls.Certificate{}, fmt.Errorf("generate server cert: %w", err)
+	}
+
+	return sc.TLSCertificate()
+}
+
+// makeHandler returns an HTTP handler that dispatches requests to debug
+// handlers or to the specified proxies, if they are defined.
+func makeHandler(modProxy, revProxy http.Handler) http.HandlerFunc {
+	mux := http.NewServeMux()
+	tsweb.Debugger(mux)
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Host != "" && r.URL.Host == r.Host {
+			// The caller wants us to proxy for them.
+			if revProxy != nil {
+				revProxy.ServeHTTP(w, r)
+				return
+			}
+			// We don't allow proxying in this configuration, bug off.
+			http.Error(w, http.StatusText(http.StatusBadGateway), http.StatusBadGateway)
+			return
+		}
+
+		path := r.URL.Path
+		if strings.HasPrefix(path, "/debug/") {
+			mux.ServeHTTP(w, r)
+			return
+		}
+		if modProxy != nil && r.Method == http.MethodGet && strings.HasPrefix(path, "/mod/") {
+			modProxy.ServeHTTP(w, r)
+			return
+		}
+		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+	}
+}
+
+// noop is a cleanup function that does nothing, used as a default.
+func noop() {}