1// Copyright 2019 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// Go checksum database lookup
6
7//go:build !cmd_go_bootstrap
8
9package modfetch
10
11import (
12	"bytes"
13	"errors"
14	"fmt"
15	"io"
16	"io/fs"
17	"net/url"
18	"os"
19	"path/filepath"
20	"strings"
21	"sync"
22	"time"
23
24	"cmd/go/internal/base"
25	"cmd/go/internal/cfg"
26	"cmd/go/internal/lockedfile"
27	"cmd/go/internal/web"
28
29	"golang.org/x/mod/module"
30	"golang.org/x/mod/sumdb"
31	"golang.org/x/mod/sumdb/note"
32)
33
34// useSumDB reports whether to use the Go checksum database for the given module.
35func useSumDB(mod module.Version) bool {
36	if mod.Path == "golang.org/toolchain" {
37		must := true
38		// Downloaded toolchains cannot be listed in go.sum,
39		// so we require checksum database lookups even if
40		// GOSUMDB=off or GONOSUMDB matches the pattern.
41		// If GOSUMDB=off, then the eventual lookup will fail
42		// with a good error message.
43
44		// Exception #1: using GOPROXY=file:// to test a distpack.
45		if strings.HasPrefix(cfg.GOPROXY, "file://") && !strings.ContainsAny(cfg.GOPROXY, ",|") {
46			must = false
47		}
48		// Exception #2: the Go proxy+checksum database cannot check itself
49		// while doing the initial download.
50		if strings.Contains(os.Getenv("GIT_HTTP_USER_AGENT"), "proxy.golang.org") {
51			must = false
52		}
53
54		// Another potential exception would be GOPROXY=direct,
55		// but that would make toolchain downloads only as secure
56		// as HTTPS, and in particular they'd be susceptible to MITM
57		// attacks on systems with less-than-trustworthy root certificates.
58		// The checksum database provides a stronger guarantee,
59		// so we don't make that exception.
60
61		// Otherwise, require the checksum database.
62		if must {
63			return true
64		}
65	}
66	return cfg.GOSUMDB != "off" && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path)
67}
68
69// lookupSumDB returns the Go checksum database's go.sum lines for the given module,
70// along with the name of the database.
71func lookupSumDB(mod module.Version) (dbname string, lines []string, err error) {
72	dbOnce.Do(func() {
73		dbName, db, dbErr = dbDial()
74	})
75	if dbErr != nil {
76		return "", nil, dbErr
77	}
78	lines, err = db.Lookup(mod.Path, mod.Version)
79	return dbName, lines, err
80}
81
82var (
83	dbOnce sync.Once
84	dbName string
85	db     *sumdb.Client
86	dbErr  error
87)
88
89func dbDial() (dbName string, db *sumdb.Client, err error) {
90	// $GOSUMDB can be "key" or "key url",
91	// and the key can be a full verifier key
92	// or a host on our list of known keys.
93
94	// Special case: sum.golang.google.cn
95	// is an alias, reachable inside mainland China,
96	// for sum.golang.org. If there are more
97	// of these we should add a map like knownGOSUMDB.
98	gosumdb := cfg.GOSUMDB
99	if gosumdb == "sum.golang.google.cn" {
100		gosumdb = "sum.golang.org https://sum.golang.google.cn"
101	}
102
103	if gosumdb == "off" {
104		return "", nil, fmt.Errorf("checksum database disabled by GOSUMDB=off")
105	}
106
107	key := strings.Fields(gosumdb)
108	if len(key) >= 1 {
109		if k := knownGOSUMDB[key[0]]; k != "" {
110			key[0] = k
111		}
112	}
113	if len(key) == 0 {
114		return "", nil, fmt.Errorf("missing GOSUMDB")
115	}
116	if len(key) > 2 {
117		return "", nil, fmt.Errorf("invalid GOSUMDB: too many fields")
118	}
119	vkey, err := note.NewVerifier(key[0])
120	if err != nil {
121		return "", nil, fmt.Errorf("invalid GOSUMDB: %v", err)
122	}
123	name := vkey.Name()
124
125	// No funny business in the database name.
126	direct, err := url.Parse("https://" + name)
127	if err != nil || strings.HasSuffix(name, "/") || *direct != (url.URL{Scheme: "https", Host: direct.Host, Path: direct.Path, RawPath: direct.RawPath}) || direct.RawPath != "" || direct.Host == "" {
128		return "", nil, fmt.Errorf("invalid sumdb name (must be host[/path]): %s %+v", name, *direct)
129	}
130
131	// Determine how to get to database.
132	var base *url.URL
133	if len(key) >= 2 {
134		// Use explicit alternate URL listed in $GOSUMDB,
135		// bypassing both the default URL derivation and any proxies.
136		u, err := url.Parse(key[1])
137		if err != nil {
138			return "", nil, fmt.Errorf("invalid GOSUMDB URL: %v", err)
139		}
140		base = u
141	}
142
143	return name, sumdb.NewClient(&dbClient{key: key[0], name: name, direct: direct, base: base}), nil
144}
145
146type dbClient struct {
147	key    string
148	name   string
149	direct *url.URL
150
151	once    sync.Once
152	base    *url.URL
153	baseErr error
154}
155
156func (c *dbClient) ReadRemote(path string) ([]byte, error) {
157	c.once.Do(c.initBase)
158	if c.baseErr != nil {
159		return nil, c.baseErr
160	}
161
162	var data []byte
163	start := time.Now()
164	targ := web.Join(c.base, path)
165	data, err := web.GetBytes(targ)
166	if false {
167		fmt.Fprintf(os.Stderr, "%.3fs %s\n", time.Since(start).Seconds(), targ.Redacted())
168	}
169	return data, err
170}
171
172// initBase determines the base URL for connecting to the database.
173// Determining the URL requires sending network traffic to proxies,
174// so this work is delayed until we need to download something from
175// the database. If everything we need is in the local cache and
176// c.ReadRemote is never called, we will never do this work.
177func (c *dbClient) initBase() {
178	if c.base != nil {
179		return
180	}
181
182	// Try proxies in turn until we find out how to connect to this database.
183	//
184	// Before accessing any checksum database URL using a proxy, the proxy
185	// client should first fetch <proxyURL>/sumdb/<sumdb-name>/supported.
186	//
187	// If that request returns a successful (HTTP 200) response, then the proxy
188	// supports proxying checksum database requests. In that case, the client
189	// should use the proxied access method only, never falling back to a direct
190	// connection to the database.
191	//
192	// If the /sumdb/<sumdb-name>/supported check fails with a “not found” (HTTP
193	// 404) or “gone” (HTTP 410) response, or if the proxy is configured to fall
194	// back on errors, the client will try the next proxy. If there are no
195	// proxies left or if the proxy is "direct" or "off", the client should
196	// connect directly to that database.
197	//
198	// Any other response is treated as the database being unavailable.
199	//
200	// See https://golang.org/design/25530-sumdb#proxying-a-checksum-database.
201	err := TryProxies(func(proxy string) error {
202		switch proxy {
203		case "noproxy":
204			return errUseProxy
205		case "direct", "off":
206			return errProxyOff
207		default:
208			proxyURL, err := url.Parse(proxy)
209			if err != nil {
210				return err
211			}
212			if _, err := web.GetBytes(web.Join(proxyURL, "sumdb/"+c.name+"/supported")); err != nil {
213				return err
214			}
215			// Success! This proxy will help us.
216			c.base = web.Join(proxyURL, "sumdb/"+c.name)
217			return nil
218		}
219	})
220	if errors.Is(err, fs.ErrNotExist) {
221		// No proxies, or all proxies failed (with 404, 410, or were allowed
222		// to fall back), or we reached an explicit "direct" or "off".
223		c.base = c.direct
224	} else if err != nil {
225		c.baseErr = err
226	}
227}
228
229// ReadConfig reads the key from c.key
230// and otherwise reads the config (a latest tree head) from GOPATH/pkg/sumdb/<file>.
231func (c *dbClient) ReadConfig(file string) (data []byte, err error) {
232	if file == "key" {
233		return []byte(c.key), nil
234	}
235
236	if cfg.SumdbDir == "" {
237		return nil, fmt.Errorf("could not locate sumdb file: missing $GOPATH: %s",
238			cfg.GoPathError)
239	}
240	targ := filepath.Join(cfg.SumdbDir, file)
241	data, err = lockedfile.Read(targ)
242	if errors.Is(err, fs.ErrNotExist) {
243		// Treat non-existent as empty, to bootstrap the "latest" file
244		// the first time we connect to a given database.
245		return []byte{}, nil
246	}
247	return data, err
248}
249
250// WriteConfig rewrites the latest tree head.
251func (*dbClient) WriteConfig(file string, old, new []byte) error {
252	if file == "key" {
253		// Should not happen.
254		return fmt.Errorf("cannot write key")
255	}
256	if cfg.SumdbDir == "" {
257		return fmt.Errorf("could not locate sumdb file: missing $GOPATH: %s",
258			cfg.GoPathError)
259	}
260	targ := filepath.Join(cfg.SumdbDir, file)
261	os.MkdirAll(filepath.Dir(targ), 0777)
262	f, err := lockedfile.Edit(targ)
263	if err != nil {
264		return err
265	}
266	defer f.Close()
267	data, err := io.ReadAll(f)
268	if err != nil {
269		return err
270	}
271	if len(data) > 0 && !bytes.Equal(data, old) {
272		return sumdb.ErrWriteConflict
273	}
274	if _, err := f.Seek(0, 0); err != nil {
275		return err
276	}
277	if err := f.Truncate(0); err != nil {
278		return err
279	}
280	if _, err := f.Write(new); err != nil {
281		return err
282	}
283	return f.Close()
284}
285
286// ReadCache reads cached lookups or tiles from
287// GOPATH/pkg/mod/cache/download/sumdb,
288// which will be deleted by "go clean -modcache".
289func (*dbClient) ReadCache(file string) ([]byte, error) {
290	targ := filepath.Join(cfg.GOMODCACHE, "cache/download/sumdb", file)
291	data, err := lockedfile.Read(targ)
292	// lockedfile.Write does not atomically create the file with contents.
293	// There is a moment between file creation and locking the file for writing,
294	// during which the empty file can be locked for reading.
295	// Treat observing an empty file as file not found.
296	if err == nil && len(data) == 0 {
297		err = &fs.PathError{Op: "read", Path: targ, Err: fs.ErrNotExist}
298	}
299	return data, err
300}
301
302// WriteCache updates cached lookups or tiles.
303func (*dbClient) WriteCache(file string, data []byte) {
304	targ := filepath.Join(cfg.GOMODCACHE, "cache/download/sumdb", file)
305	os.MkdirAll(filepath.Dir(targ), 0777)
306	lockedfile.Write(targ, bytes.NewReader(data), 0666)
307}
308
309func (*dbClient) Log(msg string) {
310	// nothing for now
311}
312
313func (*dbClient) SecurityError(msg string) {
314	base.Fatalf("%s", msg)
315}
316