-
Notifications
You must be signed in to change notification settings - Fork 17.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
cmd/dist: add -distpack flag to build distribution archives
We want to enable others to reproduce the exact distribution archives we are serving on go.dev/dl. Today the code for building those archives lives in golang.org/x/build, which is fundamentally tied to running on the Go team build infrastructure and not easy for others to run. This CL adds a new flag -distpack to cmd/dist, usually invoked as make.bash -distpack, to build the distribution archives using code in the main repository that anyone can run. Starting in Go 1.21, the Go team build infrastructure will run this instead of its current custom code to build those archives. The current builds are not reproducible even given identical infrastructure, because the archives are stamped with the current time. It is helpful to have a timestamp in the archives indicating when the code is from, but that time needs to be reproducible. To ensure this, the new -distpack flag extends the VERSION file to include a time stamp, which it uses as the modification time for all files in the archive. The new -distpack flag is implemented by a separate program, cmd/distpack, instead of being in cmd/dist, so that it can be compiled by the toolchain being distributed and not the bootstrap toolchain. Otherwise details like the exact compression algorithms might vary from one bootstrap toolchain to another and produce non-reproducible builds. So there is a new 'go tool distpack', but it's omitted from the distributions themselves, just as 'go tool dist' is. make.bash already accepts any flags for cmd/dist, including -distpack. make.bat is less sophisticated and looks for each known flag, so this CL adds an update to look for -distpack. The CL also changes make.bat to accept the idiomatic Go -flagname in addition to the non-idiomatic (for Go) --flagname. Previously it insisted on the --flag form. I have confirmed that using make.bash -distpack produces the identical distribution archives for windows/amd64, linux/amd64, darwin/amd64, and darwin/arm64 whether it is run on windows/amd64, linux/amd64, or darwin/amd64 hosts. For #24904. Change-Id: Ie6d69365ee3d7294d05b4f96ffb9159b41918074 Reviewed-on: https://go-review.googlesource.com/c/go/+/470676 TryBot-Result: Gopher Robot <[email protected]> Reviewed-by: Heschi Kreinick <[email protected]> Run-TryBot: Russ Cox <[email protected]> Reviewed-by: Carlos Amedee <[email protected]>
- Loading branch information
Showing
6 changed files
with
849 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,197 @@ | ||
| // Copyright 2023 The Go Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "io/fs" | ||
| "log" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
| "sort" | ||
| "strings" | ||
| "time" | ||
| ) | ||
|
|
||
| // An Archive describes an archive to write: a collection of files. | ||
| // Directories are implied by the files and not explicitly listed. | ||
| type Archive struct { | ||
| Files []File | ||
| } | ||
|
|
||
| // A File describes a single file to write to an archive. | ||
| type File struct { | ||
| Name string // name in archive | ||
| Time time.Time // modification time | ||
| Mode fs.FileMode | ||
| Size int64 | ||
| Src string // source file in OS file system | ||
| } | ||
|
|
||
| // Info returns a FileInfo about the file, for use with tar.FileInfoHeader | ||
| // and zip.FileInfoHeader. | ||
| func (f *File) Info() fs.FileInfo { | ||
| return fileInfo{f} | ||
| } | ||
|
|
||
| // A fileInfo is an implementation of fs.FileInfo describing a File. | ||
| type fileInfo struct { | ||
| f *File | ||
| } | ||
|
|
||
| func (i fileInfo) Name() string { return path.Base(i.f.Name) } | ||
| func (i fileInfo) ModTime() time.Time { return i.f.Time } | ||
| func (i fileInfo) Mode() fs.FileMode { return i.f.Mode } | ||
| func (i fileInfo) IsDir() bool { return false } | ||
| func (i fileInfo) Size() int64 { return i.f.Size } | ||
| func (i fileInfo) Sys() any { return nil } | ||
|
|
||
| // NewArchive returns a new Archive containing all the files in the directory dir. | ||
| // The archive can be amended afterward using methods like Add and Filter. | ||
| func NewArchive(dir string) (*Archive, error) { | ||
| a := new(Archive) | ||
| err := fs.WalkDir(os.DirFS(dir), ".", func(name string, d fs.DirEntry, err error) error { | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if d.IsDir() { | ||
| return nil | ||
| } | ||
| info, err := d.Info() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| a.Add(name, filepath.Join(dir, name), info) | ||
| return nil | ||
| }) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| a.Sort() | ||
| return a, nil | ||
| } | ||
|
|
||
| // Add adds a file with the given name and info to the archive. | ||
| // The content of the file comes from the operating system file src. | ||
| // After a sequence of one or more calls to Add, | ||
| // the caller should invoke Sort to re-sort the archive's files. | ||
| func (a *Archive) Add(name, src string, info fs.FileInfo) { | ||
| a.Files = append(a.Files, File{ | ||
| Name: name, | ||
| Time: info.ModTime(), | ||
| Mode: info.Mode(), | ||
| Size: info.Size(), | ||
| Src: src, | ||
| }) | ||
| } | ||
|
|
||
| // Sort sorts the files in the archive. | ||
| // It is only necessary to call Sort after calling Add. | ||
| // ArchiveDir returns a sorted archive, and the other methods | ||
| // preserve the sorting of the archive. | ||
| func (a *Archive) Sort() { | ||
| sort.Slice(a.Files, func(i, j int) bool { | ||
| return a.Files[i].Name < a.Files[j].Name | ||
| }) | ||
| } | ||
|
|
||
| // Clone returns a copy of the Archive. | ||
| // Method calls like Add and Filter invoked on the copy do not affect the original, | ||
| // nor do calls on the original affect the copy. | ||
| func (a *Archive) Clone() *Archive { | ||
| b := &Archive{ | ||
| Files: make([]File, len(a.Files)), | ||
| } | ||
| copy(b.Files, a.Files) | ||
| return b | ||
| } | ||
|
|
||
| // AddPrefix adds a prefix to all file names in the archive. | ||
| func (a *Archive) AddPrefix(prefix string) { | ||
| for i := range a.Files { | ||
| a.Files[i].Name = path.Join(prefix, a.Files[i].Name) | ||
| } | ||
| } | ||
|
|
||
| // Filter removes files from the archive for which keep(name) returns false. | ||
| func (a *Archive) Filter(keep func(name string) bool) { | ||
| files := a.Files[:0] | ||
| for _, f := range a.Files { | ||
| if keep(f.Name) { | ||
| files = append(files, f) | ||
| } | ||
| } | ||
| a.Files = files | ||
| } | ||
|
|
||
| // SetMode changes the mode of every file in the archive | ||
| // to be mode(name, m), where m is the file's current mode. | ||
| func (a *Archive) SetMode(mode func(name string, m fs.FileMode) fs.FileMode) { | ||
| for i := range a.Files { | ||
| a.Files[i].Mode = mode(a.Files[i].Name, a.Files[i].Mode) | ||
| } | ||
| } | ||
|
|
||
| // Remove removes files matching any of the patterns from the archive. | ||
| // The patterns use the syntax of path.Match, with an extension of allowing | ||
| // a leading **/ or trailing /**, which match any number of path elements | ||
| // (including no path elements) before or after the main match. | ||
| func (a *Archive) Remove(patterns ...string) { | ||
| a.Filter(func(name string) bool { | ||
| for _, pattern := range patterns { | ||
| match, err := amatch(pattern, name) | ||
| if err != nil { | ||
| log.Fatalf("archive remove: %v", err) | ||
| } | ||
| if match { | ||
| return false | ||
| } | ||
| } | ||
| return true | ||
| }) | ||
| } | ||
|
|
||
| // SetTime sets the modification time of all files in the archive to t. | ||
| func (a *Archive) SetTime(t time.Time) { | ||
| for i := range a.Files { | ||
| a.Files[i].Time = t | ||
| } | ||
| } | ||
|
|
||
| func amatch(pattern, name string) (bool, error) { | ||
| // firstN returns the prefix of name corresponding to the first n path elements. | ||
| // If n <= 0, firstN returns the entire name. | ||
| firstN := func(name string, n int) string { | ||
| for i := 0; i < len(name); i++ { | ||
| if name[i] == '/' { | ||
| if n--; n == 0 { | ||
| return name[:i] | ||
| } | ||
| } | ||
| } | ||
| return name | ||
| } | ||
|
|
||
| // lastN returns the suffix of name corresponding to the last n path elements. | ||
| // If n <= 0, lastN returns the entire name. | ||
| lastN := func(name string, n int) string { | ||
| for i := len(name) - 1; i >= 0; i-- { | ||
| if name[i] == '/' { | ||
| if n--; n == 0 { | ||
| return name[i+1:] | ||
| } | ||
| } | ||
| } | ||
| return name | ||
| } | ||
|
|
||
| if p, ok := strings.CutPrefix(pattern, "**/"); ok { | ||
| return path.Match(p, lastN(name, 1+strings.Count(p, "/"))) | ||
| } | ||
| if p, ok := strings.CutSuffix(pattern, "/**"); ok { | ||
| return path.Match(p, firstN(name, 1+strings.Count(p, "/"))) | ||
| } | ||
| return path.Match(pattern, name) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| // Copyright 2023 The Go Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| package main | ||
|
|
||
| import "testing" | ||
|
|
||
| var amatchTests = []struct { | ||
| pattern string | ||
| name string | ||
| ok bool | ||
| }{ | ||
| {"a", "a", true}, | ||
| {"a", "b", false}, | ||
| {"a/**", "a", true}, | ||
| {"a/**", "b", false}, | ||
| {"a/**", "a/b", true}, | ||
| {"a/**", "b/b", false}, | ||
| {"a/**", "a/b/c/d/e/f", true}, | ||
| {"a/**", "z/a/b/c/d/e/f", false}, | ||
| {"**/a", "a", true}, | ||
| {"**/a", "b", false}, | ||
| {"**/a", "x/a", true}, | ||
| {"**/a", "x/a/b", false}, | ||
| {"**/a", "x/y/z/a", true}, | ||
| {"**/a", "x/y/z/a/b", false}, | ||
|
|
||
| {"go/pkg/tool/*/compile", "go/pkg/tool/darwin_amd64/compile", true}, | ||
| } | ||
|
|
||
| func TestAmatch(t *testing.T) { | ||
| for _, tt := range amatchTests { | ||
| ok, err := amatch(tt.pattern, tt.name) | ||
| if ok != tt.ok || err != nil { | ||
| t.Errorf("amatch(%q, %q) = %v, %v, want %v, nil", tt.pattern, tt.name, ok, err, tt.ok) | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.