From 6e45f508743af898f9bbed96fcbf9068d70860da Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Tue, 19 May 2026 21:28:45 -0700 Subject: [PATCH] feat(cache): add Cloudflare SigV4 S3 signature compatibility fix and compile locally --- .gitea/actions/go-cache/action.yml | 4 +- .../.github/workflows/go-presubmit.yml | 28 ++ go-cache-plugin-src/LICENSE | 28 ++ go-cache-plugin-src/README.md | 94 ++++ .../cmd/go-cache-plugin/addca_default.go | 29 ++ .../cmd/go-cache-plugin/addca_linux.go | 39 ++ .../cmd/go-cache-plugin/commands.go | 213 +++++++++ .../cmd/go-cache-plugin/go-cache-plugin.go | 101 +++++ .../cmd/go-cache-plugin/help.go | 153 +++++++ .../cmd/go-cache-plugin/setup.go | 319 ++++++++++++++ go-cache-plugin-src/go.mod | 58 +++ go-cache-plugin-src/go.sum | 98 +++++ go-cache-plugin-src/lib/gobuild/gobuild.go | 267 ++++++++++++ go-cache-plugin-src/lib/modproxy/modproxy.go | 323 ++++++++++++++ go-cache-plugin-src/lib/revproxy/cache.go | 159 +++++++ go-cache-plugin-src/lib/revproxy/revproxy.go | 407 ++++++++++++++++++ go-cache-plugin-src/lib/s3util/s3util.go | 169 ++++++++ go-cache-plugin-src/lib/s3util/s3util_test.go | 42 ++ 18 files changed, 2529 insertions(+), 2 deletions(-) create mode 100644 go-cache-plugin-src/.github/workflows/go-presubmit.yml create mode 100644 go-cache-plugin-src/LICENSE create mode 100644 go-cache-plugin-src/README.md create mode 100644 go-cache-plugin-src/cmd/go-cache-plugin/addca_default.go create mode 100644 go-cache-plugin-src/cmd/go-cache-plugin/addca_linux.go create mode 100644 go-cache-plugin-src/cmd/go-cache-plugin/commands.go create mode 100644 go-cache-plugin-src/cmd/go-cache-plugin/go-cache-plugin.go create mode 100644 go-cache-plugin-src/cmd/go-cache-plugin/help.go create mode 100644 go-cache-plugin-src/cmd/go-cache-plugin/setup.go create mode 100644 go-cache-plugin-src/go.mod create mode 100644 go-cache-plugin-src/go.sum create mode 100644 go-cache-plugin-src/lib/gobuild/gobuild.go create mode 100644 go-cache-plugin-src/lib/modproxy/modproxy.go create mode 100644 go-cache-plugin-src/lib/revproxy/cache.go create mode 100644 go-cache-plugin-src/lib/revproxy/revproxy.go create mode 100644 go-cache-plugin-src/lib/s3util/s3util.go create mode 100644 go-cache-plugin-src/lib/s3util/s3util_test.go diff --git a/.gitea/actions/go-cache/action.yml b/.gitea/actions/go-cache/action.yml index b067aaa..c5a8e4a 100644 --- a/.gitea/actions/go-cache/action.yml +++ b/.gitea/actions/go-cache/action.yml @@ -50,8 +50,8 @@ runs: if: ${{ inputs.cache == 'true' && inputs.s3-bucket != '' }} shell: bash run: | - echo "Installing go-cache-plugin..." - go install github.com/tailscale/go-cache-plugin/cmd/go-cache-plugin@${{ inputs.go-cache-plugin-version }} + echo "Installing custom Cloudflare-compatible go-cache-plugin..." + go build -o $HOME/go/bin/go-cache-plugin ${{ github.action_path }}/go-cache-plugin-src/cmd/go-cache-plugin - name: Configure Go Cache if: ${{ inputs.cache == 'true' && inputs.s3-bucket != '' }} diff --git a/go-cache-plugin-src/.github/workflows/go-presubmit.yml b/go-cache-plugin-src/.github/workflows/go-presubmit.yml new file mode 100644 index 0000000..1930960 --- /dev/null +++ b/go-cache-plugin-src/.github/workflows/go-presubmit.yml @@ -0,0 +1,28 @@ +name: Go presubmit + +on: + push: + branches: + - main + pull_request: + types: [opened, reopened, synchronize] + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: Go presubmit + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: ['stable'] + os: ['ubuntu-latest'] + steps: + - uses: actions/checkout@v4 + - name: Install Go ${{ matrix.go-version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + - uses: creachadair/go-presubmit-action@v2 diff --git a/go-cache-plugin-src/LICENSE b/go-cache-plugin-src/LICENSE new file mode 100644 index 0000000..800a73e --- /dev/null +++ b/go-cache-plugin-src/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2024 Tailscale Inc & AUTHORS. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/go-cache-plugin-src/README.md b/go-cache-plugin-src/README.md new file mode 100644 index 0000000..2eed595 --- /dev/null +++ b/go-cache-plugin-src/README.md @@ -0,0 +1,94 @@ +# go-cache-plugin + +[![GoDoc](https://img.shields.io/static/v1?label=godoc&message=reference&color=lightgrey)](https://pkg.go.dev/github.com/tailscale/go-cache-plugin) +[![CI](https://github.com/tailscale/go-cache-plugin/actions/workflows/go-presubmit.yml/badge.svg?event=push&branch=main)](https://github.com/tailscale/go-cache-plugin/actions/workflows/go-presubmit.yml) + +This repository defines a tool implementing a `GOCACHEPROG` plugin backed by Amazon S3. + +## Installation + +```shell +go install github.com/tailscale/go-cache-plugin/cmd/go-cache-plugin@latest +``` + +## Usage Outline + +```shell +export GOCACHEPROG="go-cache-plugin --cache-dir=/tmp/gocache --bucket=some-s3-bucket" +go test ./... +``` + +Using the plugin requires a Go toolchain built with `GOEXPERIMENT=cacheprog` enabled. +However, you do not need the experiment enabled to build the plugin itself. + +## Discussion + +The `go-cache-plugin` program supports two modes of operation: + +1. **Direct mode**: The program is invoked directly by the Go toolchain as a + subprocess, and exits when the toolchain execution ends. + + This is the default mode of operation, and requires no additional setup. + +2. **Server mode**: The program runs as a separate process and the Go toolchain + communicates with it over a local socket. + + This mode requires the server to be started up ahead of time, but makes the + configuration for the toolchain simpler. This mode also permits running an + in-process module and sum database proxy. + +### Server Mode + +To run in server mode, use the `serve` subcommand: + +```sh +# N.B.: The --plugin flag is required. +go-cache-plugin serve \ + --plugin=5930 \ + --cache-dir=/tmp/gocache \ + --bucket=some-s3-bucket +``` + +To connect to a server running in this mode, use the `connect` subcommand: + +```sh +# Use the same port given to the server's --plugin flag. +# Mnemonic: 5930 == (Go) (C)ache (P)lugin +export GOCACHEPROG="go-cache-plugin connect 5930" +go build ./... +``` + +The `connect` command just bridges the socket to stdin/stdout, which is how the +Go toolchain expects to talk to the plugin. + +### Running a Module Proxy + +To enable a caching module proxy, use the `--modproxy` flag to `serve`. The +module proxy uses HTTP, not the plugin interface, use `--http` to set the address: + +```sh +go-cache-plugin serve \ + --plugin=5930 \ + --http=localhost:5970 --modproxy \ + --cache-dir=/tmp/gocache \ + # ... other flags +``` + +To tell the Go toolchain about the proxy, set: + +```sh +# Mnemonic: 5970 == (Go) (M)odule (P)roxy +export GOPROXY=http://localhost:5970/mod # use the --http address +``` + +If you want to also proxy queries to `sum.golang.org`, also add: + +```sh +export GOSUMDB='sum.golang.org http://locahost:5970/mod/sumdb/sum.golang.org' +``` + +## References + +- [Cache plugin protocol (proposal)](https://github.com/golang/go/issues/59719) +- [Cache plugin library](https://github.com/creachadair/gocache) +- [Go module proxy documentation](https://proxy.golang.org) diff --git a/go-cache-plugin-src/cmd/go-cache-plugin/addca_default.go b/go-cache-plugin-src/cmd/go-cache-plugin/addca_default.go new file mode 100644 index 0000000..01c63f0 --- /dev/null +++ b/go-cache-plugin-src/cmd/go-cache-plugin/addca_default.go @@ -0,0 +1,29 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !linux + +package main + +import ( + "errors" + "log" + + "github.com/creachadair/atomicfile" + "github.com/creachadair/command" + "github.com/creachadair/tlsutil" +) + +func installSigningCert(env *command.Env, cert tlsutil.Certificate) error { + const certFile = "revproxy-ca.crt" + if err := atomicfile.WriteData(certFile, cert.CertPEM(), 0644); err != nil { + log.Printf("WARNING: Unable to write cert file: %v", err) + } else { + log.Printf("Wrote signing cert to %s", certFile) + } + // TODO(creachadair): Maybe crib some other cases from mkcert, if we need + // them, for example: + // https://github.com/FiloSottile/mkcert/blob/master/truststore_darwin.go + + return errors.New("unable to install a certificate on this system") +} diff --git a/go-cache-plugin-src/cmd/go-cache-plugin/addca_linux.go b/go-cache-plugin-src/cmd/go-cache-plugin/addca_linux.go new file mode 100644 index 0000000..8afe726 --- /dev/null +++ b/go-cache-plugin-src/cmd/go-cache-plugin/addca_linux.go @@ -0,0 +1,39 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "errors" + "fmt" + "os" + + "github.com/creachadair/command" + "github.com/creachadair/tlsutil" + "golang.org/x/sys/unix" +) + +func installSigningCert(env *command.Env, cert tlsutil.Certificate) error { + const ubuntuCertFile = "/etc/ssl/certs/ca-certificates.crt" + return lockAndAppend(ubuntuCertFile, cert.CertPEM()) +} + +// lockAndAppend acquires an exclusive advisory lock on path, if possible, and +// appends data to the end of it. It reports an error if path does not exist, +// or if the lock could not be acquired. The lock is automatically released +// before returning. +func lockAndAppend(path string, data []byte) error { + f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0) + if err != nil { + return err + } + fd := int(f.Fd()) + if err := unix.Flock(fd, unix.LOCK_EX); err != nil { + f.Close() + return fmt.Errorf("lock: %w", err) + } + defer unix.Flock(fd, unix.LOCK_UN) + _, werr := f.Write(data) + cerr := f.Close() + return errors.Join(werr, cerr) +} diff --git a/go-cache-plugin-src/cmd/go-cache-plugin/commands.go b/go-cache-plugin-src/cmd/go-cache-plugin/commands.go new file mode 100644 index 0000000..0578f82 --- /dev/null +++ b/go-cache-plugin-src/cmd/go-cache-plugin/commands.go @@ -0,0 +1,213 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "github.com/creachadair/command" + "github.com/creachadair/gocache" + "github.com/creachadair/taskgroup" +) + +var flags struct { + CacheDir string `flag:"cache-dir,default=$GOCACHE_DIR,Local cache directory (required)"` + S3Bucket string `flag:"bucket,default=$GOCACHE_S3_BUCKET,S3 bucket name (required)"` + S3Region string `flag:"region,default=$GOCACHE_S3_REGION,S3 region"` + S3Endpoint string `flag:"s3-endpoint-url,default=$GOCACHE_S3_ENDPOINT_URL,S3 custom endpoint URL (if unset, use AWS default)"` + S3PathStyle bool `flag:"s3-path-style,default=$GOCACHE_S3_PATH_STYLE,S3 path-style URLs (optional)"` + KeyPrefix string `flag:"prefix,default=$GOCACHE_KEY_PREFIX,S3 key prefix (optional)"` + MinUploadSize int64 `flag:"min-upload-size,default=$GOCACHE_MIN_SIZE,Minimum object size to upload to S3 (in bytes)"` + Concurrency int `flag:"c,default=$GOCACHE_CONCURRENCY,Maximum number of concurrent requests"` + S3Concurrency int `flag:"u,default=$GOCACHE_S3_CONCURRENCY,Maximum concurrency for upload to S3"` + PrintMetrics bool `flag:"metrics,default=$GOCACHE_METRICS,Print summary metrics to stderr at exit"` + Expiration time.Duration `flag:"expiry,default=$GOCACHE_EXPIRY,Cache expiration period (optional)"` + Verbose bool `flag:"v,default=$GOCACHE_VERBOSE,Enable verbose logging"` + DebugLog int `flag:"debug,default=$GOCACHE_DEBUG,Enable detailed per-request debug logging (noisy)"` +} + +const ( + debugBuildCache = 1 << iota + debugModProxy + debugRevProxy +) + +// runDirect runs a cache communicating on stdin/stdout, for use as a direct +// GOCACHEPROG plugin. +func runDirect(env *command.Env) error { + s, _, err := initCacheServer(env) + if err != nil { + return err + } + if err := s.Run(env.Context(), os.Stdin, os.Stdout); err != nil { + return fmt.Errorf("cache server exited with error: %w", err) + } + if flags.Verbose || flags.PrintMetrics { + fmt.Fprintln(os.Stderr, s.Metrics()) + } + return nil +} + +var serveFlags struct { + Plugin int `flag:"plugin,default=$GOCACHE_PLUGIN,Plugin service port (required)"` + HTTP string `flag:"http,default=$GOCACHE_HTTP,HTTP service address ([host]:port)"` + ModProxy bool `flag:"modproxy,default=$GOCACHE_MODPROXY,Enable a Go module proxy (requires --http)"` + RevProxy string `flag:"revproxy,default=$GOCACHE_REVPROXY,Reverse proxy these hosts (comma-separated; requires --http)"` + SumDB string `flag:"sumdb,default=$GOCACHE_SUMDB,SumDB servers to proxy for (comma-separated)"` +} + +func noopClose(context.Context) error { return nil } + +// runServe runs a cache communicating over a local TCP socket. +func runServe(env *command.Env) error { + if serveFlags.Plugin <= 0 { + return env.Usagef("you must provide a --plugin port") + } + + // Initialize the cache server. Unlike a direct server, only close down and + // wait for cache cleanup when the whole process exits. + s, s3c, err := initCacheServer(env) + if err != nil { + return err + } + closeHook := s.Close + s.Close = noopClose + + // Listen for connections from the Go toolchain on the specified socket. + lst, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", serveFlags.Plugin)) + if err != nil { + return fmt.Errorf("listen: %w", err) + } + log.Printf("plugin listening at %q", lst.Addr()) + + ctx, cancel := signal.NotifyContext(env.Context(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + var g taskgroup.Group + g.Run(func() { + <-ctx.Done() + log.Printf("closing plugin listener") + lst.Close() + }) + + // If a module proxy is enabled, start it. + modProxy, modCleanup, err := initModProxy(env.SetContext(ctx), s3c) + if err != nil { + lst.Close() + return fmt.Errorf("module proxy: %w", err) + } + defer modCleanup() + + // If a reverse proxy is enabled, start it. + revProxy, err := initRevProxy(env.SetContext(ctx), s3c, &g) + if err != nil { + lst.Close() + return fmt.Errorf("reverse proxy: %w", err) + } + + // If an HTTP server is enabled, start it up with debug routes + // and whatever other services were requested. + if serveFlags.HTTP != "" { + srv := &http.Server{ + Addr: serveFlags.HTTP, + Handler: makeHandler(modProxy, revProxy), + } + g.Go(srv.ListenAndServe) + vprintf("HTTP server listening at %q", serveFlags.HTTP) + g.Run(func() { + <-ctx.Done() + vprintf("stopping HTTP service") + srv.Shutdown(context.Background()) + }) + } + + for { + conn, err := lst.Accept() + if err != nil { + if !errors.Is(err, net.ErrClosed) { + log.Printf("accept failed: %v, exiting server loop", err) + } + break + } + log.Printf("new client connection") + g.Go(func() error { + defer func() { + log.Printf("client connection closed") + conn.Close() + }() + return s.Run(ctx, conn, conn) + }) + } + log.Printf("server loop exited, waiting for client exit") + g.Wait() + if closeHook != nil { + ctx := gocache.WithLogf(context.Background(), log.Printf) + if err := closeHook(ctx); err != nil { + log.Printf("server close: %v (ignored)", err) + } + } + return nil +} + +// runConnect implements a direct cache proxy by connecting to a remote server. +func runConnect(env *command.Env, plugin string) error { + port, err := strconv.Atoi(plugin) + if err != nil { + return fmt.Errorf("invalid plugin port: %w", err) + } + + conn, err := net.Dial("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + return fmt.Errorf("dial: %w", err) + } + start := time.Now() + vprintf("connected to %q", conn.RemoteAddr()) + + out := taskgroup.Go(func() error { + defer conn.(*net.TCPConn).CloseWrite() // let the server finish + return copy(conn, os.Stdin) + }) + if rerr := copy(os.Stdout, conn); rerr != nil { + vprintf("read responses: %v", err) + } + out.Wait() + conn.Close() + vprintf("connection closed (%v elapsed)", time.Since(start)) + return nil +} + +// copy emulates the base case of io.Copy, but does not attempt to use the +// io.ReaderFrom or io.WriterTo implementations. +// +// TODO(creachadair): For some reason io.Copy does not work correctly when r is +// a pipe (e.g., stdin) and w is a TCP socket. Figure out why. +func copy(w io.Writer, r io.Reader) error { + var buf [4096]byte + for { + nr, err := r.Read(buf[:]) + if nr > 0 { + if nw, err := w.Write(buf[:nr]); err != nil { + return fmt.Errorf("copy to: %w", err) + } else if nw < nr { + return fmt.Errorf("wrote %d < %d bytes: %w", nw, nr, io.ErrShortWrite) + } + } + if err == io.EOF { + return nil + } else if err != nil { + return fmt.Errorf("copy from: %w", err) + } + } +} diff --git a/go-cache-plugin-src/cmd/go-cache-plugin/go-cache-plugin.go b/go-cache-plugin-src/cmd/go-cache-plugin/go-cache-plugin.go new file mode 100644 index 0000000..4712d96 --- /dev/null +++ b/go-cache-plugin-src/cmd/go-cache-plugin/go-cache-plugin.go @@ -0,0 +1,101 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Program gocache implements the experimental GOCACHEPROG protocol over an S3 +// bucket, for use in builder and CI workers. +package main + +import ( + "context" + "log" + "os" + + "github.com/creachadair/command" + "github.com/creachadair/flax" + "github.com/tailscale/go-cache-plugin/lib/s3util" +) + +func main() { + log.SetFlags(log.Ltime | log.Lmicroseconds) + root := &command.C{ + Name: command.ProgramName(), + Usage: "--cache-dir d --bucket b [options]\nhelp", + Help: `Run a cache service for the Go toolchain backed by an S3 bucket. + +This program serves the Go toolchain cache protocol on stdin/stdout. +It is meant to be run by the "go" tool as a GOCACHEPROG plugin. +For example: + + GOCACHEPROG=` + command.ProgramName() + ` go build ./... + +Note that Go toolchains prior to 1.24 must be built with GOEXPERIMENT=cacheprog +to enable this integration. Go 1.24 and later have it enabled by default. + +You must provide --cache-dir, --bucket, and --region, or the corresponding +environment variables (see "help environment"). Entries in the cache are +stored in the specified S3 bucket, and staged in a local directory specified by +the --cache-dir flag or GOCACHE_DIR environment.`, + + SetFlags: command.Flags(flax.MustBind, &flags), + Run: command.Adapt(runDirect), + + Commands: []*command.C{ + { + Name: "serve", + Usage: "--plugin ", + Help: `Run a cache server. + +In this mode, the cache server listens for connections on a socket instead of +serving directly over stdin/stdout. The "connect" command adapts the direct +interface to this one. + +By default, only the build cache is exported via the --plugin port. + +If --http is set, the server also exports an HTTP server at that address. +By default, this exports only /debug endpoints, including metrics. +When --http is enabled, the following options are available: + +- When --modcache is true, the server also exports a caching module proxy at + http://:/mod/. + +- When --revproxy is set, the server also hosts a caching reverse proxy for the + specified hosts at http://:. The reverse proxy handles both HTTP + and HTTPS requests, and caches immutable successful responses.`, + + SetFlags: command.Flags(flax.MustBind, &serveFlags), + Run: command.Adapt(runServe), + }, + { + Name: "connect", + Usage: "", + Help: `Connect to a remote cache server. + +This mode bridges stdin/stdout to a cache server (see the "serve" command) +listening on the specified port.`, + + Run: command.Adapt(runConnect), + }, + command.HelpCommand(helpTopics), + command.VersionCommand(), + }, + } + command.RunOrFail(root.NewEnv(nil), os.Args[1:]) +} + +// getBucketRegion reports the specified region for the given bucket. +// if the --region flag was set, that value is returned without error. +// Otherwise, it queries the GetBucketLocation API. +func getBucketRegion(ctx context.Context, bucket string) (string, error) { + if flags.S3Region != "" { + return flags.S3Region, nil + } + return s3util.BucketRegion(ctx, bucket) +} + +// vprintf acts as log.Printf if the --verbose flag is set; otherwise it +// discards its input. +func vprintf(msg string, args ...any) { + if flags.Verbose || flags.DebugLog != 0 { + log.Printf(msg, args...) + } +} diff --git a/go-cache-plugin-src/cmd/go-cache-plugin/help.go b/go-cache-plugin-src/cmd/go-cache-plugin/help.go new file mode 100644 index 0000000..3164cce --- /dev/null +++ b/go-cache-plugin-src/cmd/go-cache-plugin/help.go @@ -0,0 +1,153 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import "github.com/creachadair/command" + +var helpTopics = []command.HelpTopic{ + { + Name: "configure", + Help: `How to configure the plugin. + +To run the plugin, install the program somewhere on your system and set the +GOCACHEPROG environment variable to the command line of the plugin. You can +either specify the full path to the program, or install it in your $PATH. + +Parameters can be passed either as flags or via environment variables. +See also "help environment". + +The plugin requires credentials to access S3. If you are running in AWS, it can +get credentials from the instance metadata service; otherwise you will need to +plumb AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, +AWS_ENDPOINT_URL) or set up a configuration file. + +See also: "help environment". +Related: "direct-mode", "serve-mode", "module-proxy", "reverse-proxy".`, + }, + { + Name: "environment", + Help: `Environment variables understood by this program. + +To make it easier to configure this tool for multiple workflows, most of the +settings can be set via environment variables as well as flags. + + -------------------------------------------------------------------- + Flag (global) Variable Format Default + -------------------------------------------------------------------- + --cache-dir GOCACHE_DIR path (required) + --bucket GOCACHE_S3_BUCKET string (required) + --region GOCACHE_S3_REGION string based on bucket + --s3-path-style GOCACHE_S3_PATH_STYLE bool false + --s3-endpoint-url GOCACHE_S3_ENDPOINT_URL string "" + --prefix GOCACHE_KEY_PREFIX string "" + --min-upload-size GOCACHE_MIN_SIZE int64 0 + --metrics GOCACHE_METRICS bool false + --expiry GOCACHE_EXPIRY duration 0 + -c GOCACHE_CONCURRENCY int runtime.NumCPU + -u GOCACHE_S3_CONCURRENCY duration runtime.NumCPU + -v GOCACHE_VERBOSE bool false + --debug GOCACHE_DEBUG int 0 (see "help debug") + + -------------------------------------------------------------------- + Flag (serve) Variable Format Default + -------------------------------------------------------------------- + --plugin GOCACHE_PLUGIN port (required) + --http GOCACHE_HTTP [host]:port "" + --modproxy GOCACHE_MODPROXY bool false + --revproxy GOCACHE_REVPROXY host,... "" + --sumdb GOCACHE_SUMDB host,... "" + +See also: "help configure".`, + }, + { + Name: "direct-mode", + Help: `Run the plugin directly as a subprocess of the toolchain. + + export GOCACHEPROG="go-cache-plugin --cache-dir=/tmp/gocache --bucket ..." + go build ... + +Alternatively: + + export GOCACHE_DIR=/tmp/gocache + export GOCACHE_S3_BUCKET=cache-bucket-name + export GOCACHEPROG=go-cache-plugin + go build ... + +In this mode, you must specify the --cache-dir and --bucket settings.`, + }, + { + Name: "serve-mode", + Help: `Run the plugin as a standalone service. + +Use the "serve" subcommand: + + go-cache-plugin serve \ + --cache-dir=/tmp/gocache --bucket=$B \ + --plugin $PORT + +You can then use the "connect" subcommand to wire up the toolchain: + + export GOCACHEPROG="go-cache-plugin connect $PORT" + +In this mode, the server must have credentials to access to S3, but the +toolchain process does not need AWS credentials.`, + }, + { + Name: "module-proxy", + Help: `Run a Go module and sum database proxy. + +With the --modproxy flag, the server will also export an HTTP proxy for the +public Go module proxy (proxy.golang.org) and sum DB (sum.golang.org) at the +given address: + + go-cache-plugin serve ... --http=localhost:5970 --modproxy + +To use the module proxy, set the standard GOPROXY environment variable. +The module proxy serves under the path "/mod/": + + export GOPROXY=http://localhost:5970/mod + export GOCACHEPROG="go-cache-plugin connect $PORT" + go build ... + +To use the sum DB proxy, set the GOSUMDB environment variable. +The proxy path for a given sumdb (e.g., sum.golang.org) has this format: + + export GOSUMDB="sum.golang.org http://localhost:5970/mod/sumdb/sum.golang.org" + +See also: https://proxy.golang.org/`, + }, + { + Name: "reverse-proxy", + Help: `Run a caching reverse proxy. + +With the --revproxy flag, the server will also export a caching reverse +proxy for the specified hosts, given as a comma-separated list: + + go-cache-plugin serve ... \ + --http=localhost:5970 \ + --revproxy='api.example.com,www.example.com' + +When this is enabled, you can configure this address as an HTTP proxy: + + HTTPS_PROXY=localhost:5970 curl https://api.example.com/foo + +The proxy supports both HTTP and HTTPS backends. For HTTPS proxy targets, the +server generates its own TLS certificate, and tries to install a custom signing +cert so that other tools will validate it. The ability to do this varies by +system and configuration, however.`, + }, + { + Name: "debug", + Help: `Enable detailed debug logging. + +The --debug flag enables (very) verbose debug logging for the components of the +cache plugin. The value of the flag is a bit mask of: + + 1: Go build cache + 2: Go module proxy and sum database + 4: HTTP reverse proxy + +The default is 0 (no debug logging).`, + }, +} diff --git a/go-cache-plugin-src/cmd/go-cache-plugin/setup.go b/go-cache-plugin-src/cmd/go-cache-plugin/setup.go new file mode 100644 index 0000000..435743c --- /dev/null +++ b/go-cache-plugin-src/cmd/go-cache-plugin/setup.go @@ -0,0 +1,319 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "expvar" + "fmt" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" + "github.com/creachadair/command" + "github.com/creachadair/gocache" + "github.com/creachadair/gocache/cachedir" + "github.com/creachadair/mhttp/proxyconn" + "github.com/creachadair/taskgroup" + "github.com/creachadair/tlsutil" + "github.com/goproxy/goproxy" + "github.com/tailscale/go-cache-plugin/lib/gobuild" + "github.com/tailscale/go-cache-plugin/lib/modproxy" + "github.com/tailscale/go-cache-plugin/lib/revproxy" + "github.com/tailscale/go-cache-plugin/lib/s3util" + "tailscale.com/tsweb" +) + +func initCacheServer(env *command.Env) (*gocache.Server, *s3util.Client, error) { + switch { + case flags.CacheDir == "": + return nil, nil, env.Usagef("you must provide a --cache-dir") + case flags.S3Bucket == "": + return nil, nil, env.Usagef("you must provide an S3 --bucket name") + } + region, err := getBucketRegion(env.Context(), flags.S3Bucket) + if err != nil { + return nil, nil, env.Usagef("you must provide an S3 --region name") + } + + dir, err := cachedir.New(flags.CacheDir) + if err != nil { + return nil, nil, fmt.Errorf("create local cache: %w", err) + } + + opts := []func(*config.LoadOptions) error{ + config.WithRegion(region), + config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired), + } + if flags.S3Endpoint != "" { + vprintf("S3 endpoint URL: %s", flags.S3Endpoint) + opts = append(opts, config.WithBaseEndpoint(flags.S3Endpoint)) + } + cfg, err := config.LoadDefaultConfig(env.Context(), opts...) + if err != nil { + return nil, nil, fmt.Errorf("load AWS config: %w", err) + } + + vprintf("local cache directory: %s", flags.CacheDir) + vprintf("S3 cache bucket %q (%s)", flags.S3Bucket, region) + client := &s3util.Client{ + Client: s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = flags.S3PathStyle + + // Cloudflare SigV4 Compatibility Fix + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { + return stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc("RemoveAcceptEncoding", + func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) ( + middleware.FinalizeOutput, middleware.Metadata, error, + ) { + if req, ok := in.Request.(*smithyhttp.Request); ok { + req.Header.Del("Accept-Encoding") + req.Header.Del("Amz-Sdk-Invocation-Id") + req.Header.Del("Amz-Sdk-Request") + } + return next.HandleFinalize(ctx, in) + }), "Signing", middleware.Before) + }) + }), + Bucket: flags.S3Bucket, + } + cache := &gobuild.S3Cache{ + Local: dir, + S3Client: client, + KeyPrefix: flags.KeyPrefix, + MinUploadSize: flags.MinUploadSize, + UploadConcurrency: flags.S3Concurrency, + } + cache.SetMetrics(env.Context(), expvar.NewMap("gocache_host")) + + close := cache.Close + if flags.Expiration > 0 { + dirClose := dir.Cleanup(flags.Expiration) + close = func(ctx context.Context) error { + return errors.Join(cache.Close(ctx), dirClose(ctx)) + } + } + s := &gocache.Server{ + Get: cache.Get, + Put: cache.Put, + Close: close, + SetMetrics: cache.SetMetrics, + MaxRequests: flags.Concurrency, + Logf: vprintf, + LogRequests: flags.DebugLog&debugBuildCache != 0, + } + expvar.Publish("gocache_server", s.Metrics().Get("server")) + return s, client, nil +} + +// initModProxy initializes a Go module proxy if one is enabled. If not, it +// returns a nil handler without error. The caller must defer a call to the +// cleanup function unless an error is reported. +func initModProxy(env *command.Env, s3c *s3util.Client) (_ http.Handler, cleanup func(), _ error) { + if !serveFlags.ModProxy { + return nil, noop, nil // OK, proxy is disabled + } else if serveFlags.HTTP == "" { + return nil, nil, env.Usagef("you must set --http to enable --modproxy") + } + + modCachePath := filepath.Join(flags.CacheDir, "module") + if err := os.MkdirAll(modCachePath, 0755); err != nil { + return nil, nil, fmt.Errorf("create module cache: %w", err) + } + cacher := &modproxy.S3Cacher{ + Local: modCachePath, + S3Client: s3c, + KeyPrefix: path.Join(flags.KeyPrefix, "module"), + MaxTasks: flags.S3Concurrency, + Logf: vprintf, + LogRequests: flags.DebugLog&debugModProxy != 0, + } + cleanup = func() { vprintf("close cacher (err=%v)", cacher.Close()) } + proxy := &goproxy.Goproxy{ + Fetcher: &goproxy.GoFetcher{ + // As configured, the fetcher should never shell out to the go + // tool. Specifically, because we set GOPROXY and do not set any + // bypass via GONOPROXY, GOPRIVATE, etc., we will only attempt to + // proxy for the specific server(s) listed in Env. + GoBin: "/bin/false", + Env: []string{"GOPROXY=https://proxy.golang.org"}, + }, + Cacher: cacher, + ProxiedSumDBs: []string{"sum.golang.org"}, // default, see below + } + vprintf("enabling Go module proxy") + if serveFlags.SumDB != "" { + proxy.ProxiedSumDBs = strings.Split(serveFlags.SumDB, ",") + vprintf("enabling sum DB proxy for %s", strings.Join(proxy.ProxiedSumDBs, ", ")) + } + expvar.Publish("modcache", cacher.Metrics()) + return http.StripPrefix("/mod", proxy), cleanup, nil +} + +// initRevProxy initializes a reverse proxy if one is enabled. If not, it +// returns nil, nil to indicate a proxy was not requested. Otherwise, it +// returns a [http.Handler] to dispatch reverse proxy requests. +// +// The reverse proxy runs two collaborating HTTP servers: +// +// - The "inner" server is the proxy itself, which checks for cached values, +// forwards client requests to the remote origin (if necessary), and +// updates the cache with responses. The [revproxy.Server] is a lightweight +// wrapper around [net/http/httputil.ReverseProxy]. +// +// - The "outer" server is a bridge, that intercepts client requests. The +// bridge forwards plain HTTP requests directly to the inner server. For +// HTTPS CONNECT requests, the bridge hijacks the client connection and +// terminates TLS using a locally-signed certificate, and forwards the +// decrypted client requests to the inner caching proxy. +// +// The outer bridge is what receives requests routed by the main HTTP endpoint; +// the inner server gets all its input via the bridge: +// +// +------------+ +--------+ +// client --[proxy-request]->|HTTP handler+--->| bridge +--CONNECT--+ +// +------------+ +---+----+ | +// | | +// HTTP v +// +-------------+ | +---------------+ +// [response]<---| cache proxy |<------+--------+ terminate TLS | +// +-------------+ +---------------+ +// +// To the main HTTP listener, the bridge is an [http.Handler] that serves +// requests routed to it. To the inner server, the bridge is a [net.Listener], +// a source of client connections (with TLS terminated). +func initRevProxy(env *command.Env, s3c *s3util.Client, g *taskgroup.Group) (http.Handler, error) { + if serveFlags.RevProxy == "" { + return nil, nil // OK, proxy is disabled + } else if serveFlags.HTTP == "" { + return nil, env.Usagef("you must set --http to enable --revproxy") + } + + revCachePath := filepath.Join(flags.CacheDir, "revproxy") + if err := os.MkdirAll(revCachePath, 0755); err != nil { + return nil, fmt.Errorf("create revproxy cache: %w", err) + } + hosts := strings.Split(serveFlags.RevProxy, ",") + + // Issue a server certificate so we can proxy HTTPS requests. + cert, err := initServerCert(env, hosts) + if err != nil { + return nil, err + } + + proxy := &revproxy.Server{ + Targets: hosts, + Local: revCachePath, + S3Client: s3c, + KeyPrefix: path.Join(flags.KeyPrefix, "revproxy"), + Logf: vprintf, + LogRequests: flags.DebugLog&debugRevProxy != 0, + } + bridge := &proxyconn.Bridge{ + Addrs: hosts, + Handler: proxy, // forward HTTP requests unencrypted to the proxy + Logf: vprintf, + + // Forward connections not matching Addrs directly to their targets. + ForwardConnect: true, + } + expvar.Publish("proxyconn", bridge.Metrics()) + + // Run the proxy on its own separate server with TLS support. This server + // does not listen on a real network; it receives connections forwarded by + // the bridge internally from successful CONNECT requests. + psrv := &http.Server{ + TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}}, + + // Ordinarly HTTP proxy requests are delegated directly. + Handler: proxy, + } + g.Go(func() error { return psrv.ServeTLS(bridge, "", "") }) + + g.Run(func() { + <-env.Context().Done() + vprintf("stopping proxy bridge") + psrv.Shutdown(context.Background()) + }) + + expvar.Publish("revcache", proxy.Metrics()) + vprintf("enabling reverse proxy for %s", strings.Join(proxy.Targets, ", ")) + return bridge, nil +} + +// initServerCert creates a signed certificate advertising the specified host +// names, for use in creating a TLS server. +func initServerCert(env *command.Env, hosts []string) (tls.Certificate, error) { + ca, err := tlsutil.NewSigningCert(24*time.Hour, &x509.Certificate{ + Subject: pkix.Name{Organization: []string{"Tailscale build automation"}}, + }) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate signing cert: %w", err) + } + if err := installSigningCert(env, ca); err != nil { + vprintf("WARNING: %v", err) + } else { + vprintf("installed signing cert in system store") + + // TODO(creachadair): We should probably clean up old expired certs. + // This is OK for ephemeral build/CI workers, though. + } + + sc, err := tlsutil.NewServerCert(24*time.Hour, ca, &x509.Certificate{ + Subject: pkix.Name{Organization: []string{"Go cache plugin reverse proxy"}}, + DNSNames: hosts, + }) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate server cert: %w", err) + } + + return sc.TLSCertificate() +} + +// makeHandler returns an HTTP handler that dispatches requests to debug +// handlers or to the specified proxies, if they are defined. +func makeHandler(modProxy, revProxy http.Handler) http.HandlerFunc { + mux := http.NewServeMux() + tsweb.Debugger(mux) + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Host != "" && r.URL.Host == r.Host { + // The caller wants us to proxy for them. + if revProxy != nil { + revProxy.ServeHTTP(w, r) + return + } + // We don't allow proxying in this configuration, bug off. + http.Error(w, http.StatusText(http.StatusBadGateway), http.StatusBadGateway) + return + } + + path := r.URL.Path + if strings.HasPrefix(path, "/debug/") { + mux.ServeHTTP(w, r) + return + } + if modProxy != nil && r.Method == http.MethodGet && strings.HasPrefix(path, "/mod/") { + modProxy.ServeHTTP(w, r) + return + } + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + } +} + +// noop is a cleanup function that does nothing, used as a default. +func noop() {} diff --git a/go-cache-plugin-src/go.mod b/go-cache-plugin-src/go.mod new file mode 100644 index 0000000..0d515fa --- /dev/null +++ b/go-cache-plugin-src/go.mod @@ -0,0 +1,58 @@ +module github.com/tailscale/go-cache-plugin + +go 1.26.1 + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.6 + github.com/aws/aws-sdk-go-v2/config v1.32.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.99.1 + github.com/creachadair/atomicfile v0.4.1 + github.com/creachadair/command v0.2.2 + github.com/creachadair/flax v0.0.5 + github.com/creachadair/gocache v0.0.0-20260418161958-99bafa82eafe + github.com/creachadair/mds v0.27.1 + github.com/creachadair/mhttp v0.0.0-20260407161248-6866af4c556b + github.com/creachadair/scheddle v0.0.0-20260418161627-87a4a0c853c4 + github.com/creachadair/taskgroup v0.14.2 + github.com/creachadair/tlsutil v0.0.0-20260218173745-49b0059fedaf + github.com/goproxy/goproxy v0.21.0 + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.40.0 + tailscale.com v1.96.5 +) + +require ( + github.com/BurntSushi/toml v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.15 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.16 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.0 // indirect + github.com/aws/smithy-go v1.25.0 // indirect + github.com/creachadair/msync v0.8.1 // indirect + github.com/go-json-experiment/json v0.0.0-20250813024750-ebf49471dced // indirect + go4.org/mem v0.0.0-20240501181205-ae6ca9944745 // indirect + go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect + golang.org/x/mod v0.31.0 // indirect + golang.org/x/tools v0.40.1-0.20260108161641-ca281cf95054 // indirect + honnef.co/go/tools v0.7.0 // indirect +) + +retract ( + v0.0.19 + v0.0.16 +) + +tool honnef.co/go/tools/cmd/staticcheck diff --git a/go-cache-plugin-src/go.sum b/go-cache-plugin-src/go.sum new file mode 100644 index 0000000..706e6ec --- /dev/null +++ b/go-cache-plugin-src/go.sum @@ -0,0 +1,98 @@ +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/aws/aws-sdk-go-v2 v1.41.6 h1:1AX0AthnBQzMx1vbmir3Y4WsnJgiydmnJjiLu+LvXOg= +github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 h1:adBsCIIpLbLmYnkQU+nAChU5yhVTvu5PerROm+/Kq2A= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9/go.mod h1:uOYhgfgThm/ZyAuJGNQ5YgNyOlYfqnGpTHXvk3cpykg= +github.com/aws/aws-sdk-go-v2/config v1.32.16 h1:Q0iQ7quUgJP0F/SCRTieScnaMdXr9h/2+wze1u3cNeM= +github.com/aws/aws-sdk-go-v2/config v1.32.16/go.mod h1:duCCnJEFqpt2RC6no1iK6q+8HpwOAkiUua0pY507dQc= +github.com/aws/aws-sdk-go-v2/credentials v1.19.15 h1:fyvgWTszojq8hEnMi8PPBTvZdTtEVmAVyo+NFLHBhH4= +github.com/aws/aws-sdk-go-v2/credentials v1.19.15/go.mod h1:gJiYyMOjNg8OEdRWOf3CrFQxM2a98qmrtjx1zuiQfB8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22 h1:IOGsJ1xVWhsi+ZO7/NW8OuZZBtMJLZbk4P5HDjJO0jQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22/go.mod h1:b+hYdbU+jGKfXE8kKM6g1+h+L/Go3vMvzlxBsiuGsxg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22 h1:GmLa5Kw1ESqtFpXsx5MmC84QWa/ZrLZvlJGa2y+4kcQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22/go.mod h1:6sW9iWm9DK9YRpRGga/qzrzNLgKpT2cIxb7Vo2eNOp0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22 h1:dY4kWZiSaXIzxnKlj17nHnBcXXBfac6UlsAx2qL6XrU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.22/go.mod h1:KIpEUx0JuRZLO7U6cbV204cWAEco2iC3l061IxlwLtI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23 h1:FPXsW9+gMuIeKmz7j6ENWcWtBGTe1kH8r9thNt5Uxx4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.23/go.mod h1:7J8iGMdRKk6lw2C+cMIphgAnT8uTwBwNOsGkyOCm80U= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8 h1:HtOTYcbVcGABLOVuPYaIihj6IlkqubBwFj10K5fxRek= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.8/go.mod h1:VsK9abqQeGlzPgUr+isNWzPlK2vKe9INMLWnY65f5Xs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14 h1:xnvDEnw+pnj5mctWiYuFbigrEzSm35x7k4KS/ZkCANg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.14/go.mod h1:yS5rNogD8e0Wu9+l3MUwr6eENBzEeGejvINpN5PAYfY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22 h1:PUmZeJU6Y1Lbvt9WFuJ0ugUK2xn6hIWUBBbKuOWF30s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.22/go.mod h1:nO6egFBoAaoXze24a2C0NjQCvdpk8OueRoYimvEB9jo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22 h1:SE+aQ4DEqG53RRCAIHlCf//B2ycxGH7jFkpnAh/kKPM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.22/go.mod h1:ES3ynECd7fYeJIL6+oax+uIEljmfps0S70BaQzbMd/o= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.1 h1:kU/eBN5+MWNo/LcbNa4hWDdN76hdcd7hocU5kvu7IsU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.1/go.mod h1:Fw9aqhJicIVee1VytBBjH+l+5ov6/PhbtIK/u3rt/ls= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.10 h1:a1Fq/KXn75wSzoJaPQTgZO0wHGqE9mjFnylnqEPTchA= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.10/go.mod h1:p6+MXNxW7IA6dMgHfTAzljuwSKD0NCm/4lbS4t6+7vI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.16 h1:x6bKbmDhsgSZwv6q19wY/u3rLk/3FGjJWyqKcIRufpE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.16/go.mod h1:CudnEVKRtLn0+3uMV0yEXZ+YZOKnAtUJ5DmDhilVnIw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20 h1:oK/njaL8GtyEihkWMD4k3VgHCT64RQKkZwh0DG5j8ak= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20/go.mod h1:JHs8/y1f3zY7U5WcuzoJ/yAYGYtNIVPKLIbp61euvmg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.0 h1:ks8KBcZPh3PYISr5dAiXCM5/Thcuxk8l+PG4+A0exds= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.0/go.mod h1:pFw33T0WLvXU3rw1WBkpMlkgIn54eCB5FYLhjDc9Foo= +github.com/aws/smithy-go v1.25.0 h1:Sz/XJ64rwuiKtB6j98nDIPyYrV1nVNJ4YU74gttcl5U= +github.com/aws/smithy-go v1.25.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/creachadair/atomicfile v0.4.1 h1:72SopAy00u42/iL0p3CZILv51oUEem8uDbxUfnzmsXU= +github.com/creachadair/atomicfile v0.4.1/go.mod h1:+PsFcWa9SZuK+xnykzupXErqESId5eCGsuqFWkazMQI= +github.com/creachadair/command v0.2.2 h1:4RGsUhqFf1imFC+vMWOOCiQdncThCdcdMJp0JNCjxxc= +github.com/creachadair/command v0.2.2/go.mod h1:Z6Zp6CSJcnaWWR4wHgdqzODnFdxFJAaa/DrcVkeUu3E= +github.com/creachadair/flax v0.0.5 h1:zt+CRuXQASxwQ68e9GHAOnEgAU29nF0zYMHOCrL5wzE= +github.com/creachadair/flax v0.0.5/go.mod h1:F1PML0JZLXSNDMNiRGK2yjm5f+L9QCHchyHBldFymj8= +github.com/creachadair/gocache v0.0.0-20260418161958-99bafa82eafe h1:u136zrzzDUBqNQKPLiclaxza2spo+xtz06Mho09Vb/M= +github.com/creachadair/gocache v0.0.0-20260418161958-99bafa82eafe/go.mod h1:wmtczGXKsjsSCKcMzEDDPofBdwXKfOM5fX0TMg1nsvU= +github.com/creachadair/mds v0.27.1 h1:GlO1tPbrsaoafkF6mz7dFutkGXAtIfQLI450u0ypqwA= +github.com/creachadair/mds v0.27.1/go.mod h1:dMBTCSy3iS3dwh4Rb1zxeZz2d7K8+N24GCTsayWtQRI= +github.com/creachadair/mhttp v0.0.0-20260407161248-6866af4c556b h1:bIvrALyE2ixkCvz9LADfOIuhkUMYWMBsvQ29xe70nI0= +github.com/creachadair/mhttp v0.0.0-20260407161248-6866af4c556b/go.mod h1:eV+igunEfCBbgpM0gkDK/25/ukdeNYXH9PZuuEWZTDo= +github.com/creachadair/msync v0.8.1 h1:QRd8si3qZ2Q4TaDL7tS/MG/lFE3YND7U7J9fy42eAFM= +github.com/creachadair/msync v0.8.1/go.mod h1:dt0bscS09J8Ie3AdccK9JpCb7LfStaDGlAmDLukOlY4= +github.com/creachadair/scheddle v0.0.0-20260418161627-87a4a0c853c4 h1:uYu6k39F6Sz4Y5Zb+R2ZnhE1S5ehCw4lknwrt4bm7NI= +github.com/creachadair/scheddle v0.0.0-20260418161627-87a4a0c853c4/go.mod h1:PE4owEX+08lTR1VNomiSFFh6DzkmmzlBgysEOq8zwdI= +github.com/creachadair/taskgroup v0.14.2 h1:vPHw50VFJ98Eb+h3S1RYRejK3DE0xpGCXb2ZFtHfFeA= +github.com/creachadair/taskgroup v0.14.2/go.mod h1:FV5K1BzHUCnTVc04O1WLO+iLmvjfOTnYLTcZaC9ZxbM= +github.com/creachadair/tlsutil v0.0.0-20260218173745-49b0059fedaf h1:6ZqacBCoIdOSlwh03zr3TMqA5VyuFpzUS7E2yzfjq7w= +github.com/creachadair/tlsutil v0.0.0-20260218173745-49b0059fedaf/go.mod h1:YUng4wUBh60nfswe0FwLeBOpoIsrW6ReXt9Zr9k55Sg= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/go-json-experiment/json v0.0.0-20250813024750-ebf49471dced h1:Q311OHjMh/u5E2TITc++WlTP5We0xNseRMkHDyvhW7I= +github.com/go-json-experiment/json v0.0.0-20250813024750-ebf49471dced/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/goproxy/goproxy v0.21.0 h1:S3JPCp/WLqLl7X40eo5FFiAbBZwD8S6TzGPBITwfHlo= +github.com/goproxy/goproxy v0.21.0/go.mod h1:4j3iRV76B133PK4sOzVYWSBs/SsE5ovbZsdceoC5f+w= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +go4.org/mem v0.0.0-20240501181205-ae6ca9944745 h1:Tl++JLUCe4sxGu8cTpDzRLd3tN7US4hOxG5YpKCzkek= +go4.org/mem v0.0.0-20240501181205-ae6ca9944745/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9A8KkmRtY9WvOFIxN8wgfvy6Zm1DV8= +golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/tools v0.40.1-0.20260108161641-ca281cf95054 h1:CHVDrNHx9ZoOrNN9kKWYIbT5Rj+WF2rlwPkhbQQ5V4U= +golang.org/x/tools v0.40.1-0.20260108161641-ca281cf95054/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= +honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= +tailscale.com v1.96.5 h1:gNkfA/KSZAl6jCH9cj8urq00HRWItDDTtGsyATI89jA= +tailscale.com v1.96.5/go.mod h1:/3lnZBYb2UEwnN0MNu2SDXUtT06AGd5k0s+OWx3WmcY= diff --git a/go-cache-plugin-src/lib/gobuild/gobuild.go b/go-cache-plugin-src/lib/gobuild/gobuild.go new file mode 100644 index 0000000..5592019 --- /dev/null +++ b/go-cache-plugin-src/lib/gobuild/gobuild.go @@ -0,0 +1,267 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package gobuild implements callbacks for a gocache.Server that store data +// into an S3 bucket through a local directory. +package gobuild + +import ( + "context" + "errors" + "expvar" + "fmt" + "io/fs" + "os" + "path" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "github.com/creachadair/gocache" + "github.com/creachadair/gocache/cachedir" + "github.com/creachadair/taskgroup" + "github.com/tailscale/go-cache-plugin/lib/s3util" +) + +// S3Cache implements callbacks for a gocache.Server using an S3 bucket for +// backing store with a local directory for staging. +// +// # Remote Cache Layout +// +// Within the designated S3 bucket, keys are organized into two groups. Each +// action is stored in a file named: +// +// [/]action// +// +// Each output object is stored in a file named: +// +// [/]output// +// +// The object and action IDs are encoded as lower-case hexadecimal strings, +// with "" denoting the first two bytes of the ID to partition the space. +// +// The contents of each action file have the format: +// +// +// +// where the object ID is hex encoded and the timestamp is Unix nanoseconds. +// The object file contains just the binary data of the object. +type S3Cache struct { + // Local is the local cache directory where actions and objects are staged. + // It must be non-nil. A local stage is required because the Go toolchain + // needs direct access to read the files reported by the cache. + // It is safe to use a tmpfs directory. + Local *cachedir.Dir + + // S3Client is the S3 client used to read and write cache entries to the + // backing store. It must be non-nil. + S3Client *s3util.Client + + // KeyPrefix, if non-empty, is prepended to each key stored into S3, with an + // intervening slash. + KeyPrefix string + + // MinUploadSize, if positive, defines a minimum object size in bytes below + // which the cache will not write the object to S3. + MinUploadSize int64 + + // UploadConcurrency, if positive, defines the maximum number of concurrent + // tasks for writing cache entries to S3. If zero or negative, it uses + // runtime.NumCPU. + UploadConcurrency int + + // Tracks tasks pushing cache writes to S3. + initOnce sync.Once + push *taskgroup.Group + start func(taskgroup.Task) + + getLocalHit expvar.Int // count of Get hits in the local cache + getFaultHit expvar.Int // count of Get hits faulted in from S3 + getFaultMiss expvar.Int // count of Get faults that were misses + putSkipSmall expvar.Int // count of "small" objects not written to S3 + putS3Found expvar.Int // count of objects not written to S3 because they were already present + putS3Action expvar.Int // count of actions written to S3 + putS3Object expvar.Int // count of objects written to S3 + putS3Error expvar.Int // count of errors writing to S3 +} + +func (s *S3Cache) init() { + s.initOnce.Do(func() { + s.push, s.start = taskgroup.New(nil).Limit(s.uploadConcurrency()) + }) +} + +// Get implements the corresponding callback of the cache protocol. +func (s *S3Cache) Get(ctx context.Context, actionID string) (outputID, diskPath string, _ error) { + s.init() + + objID, diskPath, err := s.Local.Get(ctx, actionID) + if err == nil && objID != "" && diskPath != "" { + s.getLocalHit.Add(1) + return objID, diskPath, nil // cache hit, OK + } + + // Reaching here, either we got a cache miss or an error reading from local. + // Try reading the action from S3. + action, err := s.S3Client.GetData(ctx, s.actionKey(actionID)) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + s.getFaultMiss.Add(1) + return "", "", nil // cache miss, OK + } + return "", "", fmt.Errorf("[s3] read action %s: %w", actionID, err) + } + + // We got an action hit remotely, try to update the local copy. + outputID, mtime, err := parseAction(action) + if err != nil { + return "", "", err + } + + object, size, err := s.S3Client.Get(ctx, s.outputKey(outputID)) + if err != nil { + // At this point we know the action exists, so if we can't read the + // object report it as an error rather than a cache miss. + return "", "", fmt.Errorf("[s3] read object %s: %w", outputID, err) + } + defer object.Close() + s.getFaultHit.Add(1) + + // Now we should have the body; poke it into the local cache. Preserve the + // modification timestamp recorded with the original action. + diskPath, err = s.Local.Put(ctx, gocache.Object{ + ActionID: actionID, + OutputID: outputID, + Size: size, + Body: object, + ModTime: mtime, + }) + return outputID, diskPath, err +} + +// Put implements the corresponding callback of the cache protocol. +func (s *S3Cache) Put(ctx context.Context, obj gocache.Object) (diskPath string, _ error) { + s.init() + + // Compute an etag so we can do a conditional put on the object data. + // We do not rely on it as a secure checksum. The toolchain verifies the + // content address against the bits we actually store. + etr := s3util.NewETagReader(obj.Body) + obj.Body = etr + + diskPath, err := s.Local.Put(ctx, obj) + if err != nil { + return "", err // don't bother trying to forward it to the remote + } + if obj.Size < s.MinUploadSize { + s.putSkipSmall.Add(1) + return diskPath, nil // don't bother uploading this, it's too small + } + + // Try to push the record to S3 in the background. + s.start(func() error { + // Override the context with a separate timeout in case S3 is farkakte. + sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 1*time.Minute) + defer cancel() + + // Stage 1: Maybe write the object. Do this before writing the action + // record so we are less likely to get a spurious miss later. + mtime, err := s.maybePutObject(sctx, obj.OutputID, diskPath, etr.ETag()) + if err != nil { + return err + } + + // Stage 2: Write the action record. + if err := s.S3Client.Put(ctx, s.actionKey(obj.ActionID), + strings.NewReader(fmt.Sprintf("%s %d", obj.OutputID, mtime.UnixNano()))); err != nil { + gocache.Logf(ctx, "write action %s: %v", obj.ActionID, err) + return err + } + s.putS3Action.Add(1) + return nil + }) + + return diskPath, nil +} + +// Close implements the corresponding callback of the cache protocol. +func (s *S3Cache) Close(ctx context.Context) error { + if s.push != nil { + gocache.Logf(ctx, "waiting for uploads...") + wstart := time.Now() + s.push.Wait() + gocache.Logf(ctx, "uploads complete (%v elapsed)", time.Since(wstart).Round(10*time.Microsecond)) + } + return nil +} + +// SetMetrics implements the corresponding server callback. +func (s *S3Cache) SetMetrics(_ context.Context, m *expvar.Map) { + m.Set("get_local_hit", &s.getLocalHit) + m.Set("get_fault_hit", &s.getFaultHit) + m.Set("get_fault_miss", &s.getFaultMiss) + m.Set("put_skip_small", &s.putSkipSmall) + m.Set("put_s3_found", &s.putS3Found) + m.Set("put_s3_action", &s.putS3Action) + m.Set("put_s3_object", &s.putS3Object) + m.Set("put_s3_error", &s.putS3Error) +} + +// maybePutObject writes the specified object contents to S3 if there is not +// already a matching key with the same etag. It returns the modified time of +// the object file, whether or not it was sent to S3. +func (s *S3Cache) maybePutObject(ctx context.Context, outputID, diskPath, etag string) (time.Time, error) { + f, err := os.Open(diskPath) + if err != nil { + gocache.Logf(ctx, "[s3] open local object %s: %v", outputID, err) + return time.Time{}, err + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + return time.Time{}, err + } + + written, err := s.S3Client.PutCond(ctx, s.outputKey(outputID), etag, f) + if err != nil { + s.putS3Error.Add(1) + gocache.Logf(ctx, "[s3] put object %s: %v", outputID, err) + return fi.ModTime(), err + } + if written { + s.putS3Found.Add(1) + return fi.ModTime(), nil // already present and matching + } + s.putS3Object.Add(1) + return fi.ModTime(), nil +} + +// makeKey assembles a complete key from the specified parts, including the key +// prefix if one is defined. +func (s *S3Cache) makeKey(parts ...string) string { + return path.Join(s.KeyPrefix, path.Join(parts...)) +} + +func (s *S3Cache) actionKey(id string) string { return s.makeKey("action", id[:2], id) } +func (s *S3Cache) outputKey(id string) string { return s.makeKey("output", id[:2], id) } + +func (s *S3Cache) uploadConcurrency() int { + if s.UploadConcurrency <= 0 { + return runtime.NumCPU() + } + return s.UploadConcurrency +} + +func parseAction(data []byte) (outputID string, mtime time.Time, _ error) { + fs := strings.Fields(string(data)) + if len(fs) != 2 { + return "", time.Time{}, errors.New("invalid action record") + } + ts, err := strconv.ParseInt(fs[1], 10, 64) + if err != nil { + return "", time.Time{}, fmt.Errorf("invalid timestamp: %w", err) + } + return fs[0], time.Unix(ts/1e9, ts%1e9), nil +} diff --git a/go-cache-plugin-src/lib/modproxy/modproxy.go b/go-cache-plugin-src/lib/modproxy/modproxy.go new file mode 100644 index 0000000..889292c --- /dev/null +++ b/go-cache-plugin-src/lib/modproxy/modproxy.go @@ -0,0 +1,323 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package modproxy implements components of a Go module proxy that caches +// files locally on disk, backed by objects in an S3 bucket. +package modproxy + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "expvar" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "runtime" + "sync" + "time" + + "github.com/creachadair/atomicfile" + "github.com/creachadair/taskgroup" + "github.com/goproxy/goproxy" + "github.com/tailscale/go-cache-plugin/lib/s3util" + "golang.org/x/sync/semaphore" +) + +var _ goproxy.Cacher = (*S3Cacher)(nil) + +// S3Cacher implements the [github.com/goproxy/goproxy.Cacher] interface using +// a local disk cache backed by an S3 bucket. +// +// # Cache Layout +// +// Module cache files are stored under a SHA256 digest of the filename +// presented to the cache, encoded as hex and partitioned by the first two +// bytes of the digest: +// +// For example: +// +// SHA256("fizzlepug") → 160db4d719252162c87a9169e26deda33d2340770d0d540fd4c580c55008b2d6 +// /module/16/160db4d719252162c87a9169e26deda33d2340770d0d540fd4c580c55008b2d6 +// +// When files are stored in S3, the same naming convention is used, but with +// the specified key prefix instead: +// +// /module/16/0db4d719252162c87a9169e26deda33d2340770d0d540fd4c580c55008b2d6 +type S3Cacher struct { + // Local is the path of a local cache directory where modules are cached. + // It must be non-empty. + Local string + + // S3Client is the S3 client used to read and write cache entries to the + // backing store. It must be non-nil. + S3Client *s3util.Client + + // KeyPrefix, if non-empty, is prepended to each key stored into S3, with an + // intervening slash. + KeyPrefix string + + // MaxTasks, if positive, limits the number of concurrent tasks that may be + // interacting with S3. If zero or negative, the default is + // [runtime.NumCPU]. + MaxTasks int + + // Logf, if non-nil, is used to write log messages. If nil, logs are + // discarded. + Logf func(string, ...any) + + // LogRequests, if true, enables detailed (but noisy) debug logging of all + // requests handled by the cache. Logs are written to Logf. + // + // Each result is presented in the format: + // + // B "" () + // E "", err=,