buildtool.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go bootstrap version.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go bootstrap toolchains go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"go/version"
    17  	"os"
    18  	"path/filepath"
    19  	"regexp"
    20  	"strings"
    21  )
    22  
    23  // bootstrapDirs is a list of directories holding code that must be
    24  // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
    25  // All directories in this list are relative to and must be below $GOROOT/src.
    26  //
    27  // The list has two kinds of entries: names beginning with cmd/ with
    28  // no other slashes, which are commands, and other paths, which are packages
    29  // supporting the commands. Packages in the standard library can be listed
    30  // if a newer copy needs to be substituted for the Go bootstrap copy when used
    31  // by the command packages. Paths ending with /... automatically
    32  // include all packages within subdirectories as well.
    33  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    34  var bootstrapDirs = []string{
    35  	"cmp",
    36  	"cmd/asm",
    37  	"cmd/asm/internal/...",
    38  	"cmd/cgo",
    39  	"cmd/compile",
    40  	"cmd/compile/internal/...",
    41  	"cmd/internal/archive",
    42  	"cmd/internal/bio",
    43  	"cmd/internal/codesign",
    44  	"cmd/internal/dwarf",
    45  	"cmd/internal/edit",
    46  	"cmd/internal/gcprog",
    47  	"cmd/internal/goobj",
    48  	"cmd/internal/hash",
    49  	"cmd/internal/macho",
    50  	"cmd/internal/obj/...",
    51  	"cmd/internal/objabi",
    52  	"cmd/internal/par",
    53  	"cmd/internal/pgo",
    54  	"cmd/internal/pkgpath",
    55  	"cmd/internal/quoted",
    56  	"cmd/internal/src",
    57  	"cmd/internal/sys",
    58  	"cmd/internal/telemetry",
    59  	"cmd/internal/telemetry/counter",
    60  	"cmd/link",
    61  	"cmd/link/internal/...",
    62  	"compress/flate",
    63  	"compress/zlib",
    64  	"container/heap",
    65  	"debug/dwarf",
    66  	"debug/elf",
    67  	"debug/macho",
    68  	"debug/pe",
    69  	"go/build/constraint",
    70  	"go/constant",
    71  	"go/version",
    72  	"internal/abi",
    73  	"internal/coverage",
    74  	"cmd/internal/cov/covcmd",
    75  	"internal/bisect",
    76  	"internal/buildcfg",
    77  	"internal/exportdata",
    78  	"internal/goarch",
    79  	"internal/godebugs",
    80  	"internal/goexperiment",
    81  	"internal/goroot",
    82  	"internal/gover",
    83  	"internal/goversion",
    84  	// internal/lazyregexp is provided by Go 1.17, which permits it to
    85  	// be imported by other packages in this list, but is not provided
    86  	// by the Go 1.17 version of gccgo. It's on this list only to
    87  	// support gccgo, and can be removed if we require gccgo 14 or later.
    88  	"internal/lazyregexp",
    89  	"internal/pkgbits",
    90  	"internal/platform",
    91  	"internal/profile",
    92  	"internal/race",
    93  	"internal/runtime/gc",
    94  	"internal/saferio",
    95  	"internal/strconv",
    96  	"internal/syscall/unix",
    97  	"internal/types/errors",
    98  	"internal/unsafeheader",
    99  	"internal/xcoff",
   100  	"internal/zstd",
   101  	"math/bits",
   102  	"sort",
   103  }
   104  
   105  // File prefixes that are ignored by go/build anyway, and cause
   106  // problems with editor generated temporary files (#18931).
   107  var ignorePrefixes = []string{
   108  	".",
   109  	"_",
   110  	"#",
   111  }
   112  
   113  // File suffixes that use build tags introduced since Go 1.17.
   114  // These must not be copied into the bootstrap build directory.
   115  // Also ignore test files.
   116  var ignoreSuffixes = []string{
   117  	"_test.s",
   118  	"_test.go",
   119  	// Skip PGO profile. No need to build toolchain1 compiler
   120  	// with PGO. And as it is not a text file the import path
   121  	// rewrite will break it.
   122  	".pgo",
   123  	// Skip editor backup files.
   124  	"~",
   125  }
   126  
   127  const minBootstrap = "go1.24.6"
   128  
   129  var tryDirs = []string{
   130  	"sdk/" + minBootstrap,
   131  	minBootstrap,
   132  }
   133  
   134  func bootstrapBuildTools() {
   135  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   136  	if goroot_bootstrap == "" {
   137  		home := os.Getenv("HOME")
   138  		goroot_bootstrap = pathf("%s/go1.4", home)
   139  		for _, d := range tryDirs {
   140  			if p := pathf("%s/%s", home, d); isdir(p) {
   141  				goroot_bootstrap = p
   142  			}
   143  		}
   144  	}
   145  
   146  	// check bootstrap version.
   147  	ver := run(pathf("%s/bin", goroot_bootstrap), CheckExit, pathf("%s/bin/go", goroot_bootstrap), "env", "GOVERSION")
   148  	// go env GOVERSION output like "go1.22.6\n" or "devel go1.24-ffb3e574 Thu Aug 29 20:16:26 2024 +0000\n".
   149  	ver = ver[:len(ver)-1]
   150  	if version.Compare(ver, version.Lang(minBootstrap)) > 0 && version.Compare(ver, minBootstrap) < 0 {
   151  		fatalf("%s does not meet the minimum bootstrap requirement of %s or later", ver, minBootstrap)
   152  	}
   153  
   154  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   155  
   156  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   157  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   158  
   159  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   160  	// We use a subdirectory of $GOROOT/pkg because that's the
   161  	// space within $GOROOT where we store all generated objects.
   162  	// We could use a temporary directory outside $GOROOT instead,
   163  	// but it is easier to debug on failure if the files are in a known location.
   164  	workspace := pathf("%s/pkg/bootstrap", goroot)
   165  	xremoveall(workspace)
   166  	xatexit(func() { xremoveall(workspace) })
   167  	base := pathf("%s/src/bootstrap", workspace)
   168  	xmkdirall(base)
   169  
   170  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   171  	minBootstrapVers := requiredBootstrapVersion(goModVersion()) // require the minimum required go version to build this go version in the go.mod file
   172  	writefile("module bootstrap\ngo "+minBootstrapVers+"\n", pathf("%s/%s", base, "go.mod"), 0)
   173  	for _, dir := range bootstrapDirs {
   174  		recurse := strings.HasSuffix(dir, "/...")
   175  		dir = strings.TrimSuffix(dir, "/...")
   176  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   177  			if err != nil {
   178  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   179  			}
   180  
   181  			name := filepath.Base(path)
   182  			src := pathf("%s/src/%s", goroot, path)
   183  			dst := pathf("%s/%s", base, path)
   184  
   185  			if info.IsDir() {
   186  				if !recurse && path != dir || name == "testdata" {
   187  					return filepath.SkipDir
   188  				}
   189  
   190  				xmkdirall(dst)
   191  				if path == "cmd/cgo" {
   192  					// Write to src because we need the file both for bootstrap
   193  					// and for later in the main build.
   194  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   195  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   196  				}
   197  				return nil
   198  			}
   199  
   200  			for _, pre := range ignorePrefixes {
   201  				if strings.HasPrefix(name, pre) {
   202  					return nil
   203  				}
   204  			}
   205  			for _, suf := range ignoreSuffixes {
   206  				if strings.HasSuffix(name, suf) {
   207  					return nil
   208  				}
   209  			}
   210  
   211  			text := bootstrapRewriteFile(src)
   212  			writefile(text, dst, 0)
   213  			return nil
   214  		})
   215  	}
   216  
   217  	// Set up environment for invoking Go bootstrap toolchains go command.
   218  	// GOROOT points at Go bootstrap GOROOT,
   219  	// GOPATH points at our bootstrap workspace,
   220  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   221  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   222  	// so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
   223  	// Set GOFIPS140=off to work with the purego build tag.
   224  	// Restore GOROOT, GOPATH, GOBIN, and GOFIPS140 when done.
   225  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   226  	// because setup will take care of those when bootstrapBuildTools returns.
   227  
   228  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   229  	os.Setenv("GOROOT", goroot_bootstrap)
   230  
   231  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   232  	os.Setenv("GOPATH", workspace)
   233  
   234  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   235  	os.Setenv("GOBIN", "")
   236  
   237  	defer os.Setenv("GOFIPS140", os.Getenv("GOFIPS140"))
   238  	os.Setenv("GOFIPS140", "off")
   239  
   240  	os.Setenv("GOOS", "")
   241  	os.Setenv("GOHOSTOS", "")
   242  	os.Setenv("GOARCH", "")
   243  	os.Setenv("GOHOSTARCH", "")
   244  
   245  	// Run Go bootstrap to build binaries.
   246  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   247  	// which may contain unsupported instructions.
   248  	// Use the purego build tag to disable other assembly code.
   249  	cmd := []string{
   250  		pathf("%s/bin/go", goroot_bootstrap),
   251  		"install",
   252  		"-tags=math_big_pure_go compiler_bootstrap purego",
   253  	}
   254  	if vflag > 0 {
   255  		cmd = append(cmd, "-v")
   256  	}
   257  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   258  		cmd = append(cmd, "-toolexec="+tool)
   259  	}
   260  	cmd = append(cmd, "bootstrap/cmd/...")
   261  	run(base, ShowOutput|CheckExit, cmd...)
   262  
   263  	// Copy binaries into tool binary directory.
   264  	for _, name := range bootstrapDirs {
   265  		if !strings.HasPrefix(name, "cmd/") {
   266  			continue
   267  		}
   268  		name = name[len("cmd/"):]
   269  		if !strings.Contains(name, "/") {
   270  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   271  		}
   272  	}
   273  
   274  	if vflag > 0 {
   275  		xprintf("\n")
   276  	}
   277  }
   278  
   279  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   280  
   281  // isUnneededSSARewriteFile reports whether srcFile is a
   282  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   283  // architecture that isn't for the given GOARCH.
   284  //
   285  // When unneeded is true archCaps is the rewrite base filename without
   286  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   287  func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
   288  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   289  		return "", false
   290  	}
   291  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   292  	if fileArch == "" {
   293  		return "", false
   294  	}
   295  	b := fileArch[0]
   296  	if b == '_' || ('a' <= b && b <= 'z') {
   297  		return "", false
   298  	}
   299  	archCaps = fileArch
   300  	fileArch = strings.ToLower(fileArch)
   301  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   302  	fileArch = strings.TrimSuffix(fileArch, "latelower")
   303  	if fileArch == goArch {
   304  		return "", false
   305  	}
   306  	if fileArch == strings.TrimSuffix(goArch, "le") {
   307  		return "", false
   308  	}
   309  	return archCaps, true
   310  }
   311  
   312  func bootstrapRewriteFile(srcFile string) string {
   313  	// During bootstrap, generate dummy rewrite files for
   314  	// irrelevant architectures. We only need to build a bootstrap
   315  	// binary that works for the current gohostarch.
   316  	// This saves 6+ seconds of bootstrap.
   317  	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
   318  		return fmt.Sprintf(`%spackage ssa
   319  
   320  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   321  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   322  `, generatedHeader, archCaps, archCaps)
   323  	}
   324  
   325  	return bootstrapFixImports(srcFile)
   326  }
   327  
   328  var (
   329  	importRE      = regexp.MustCompile(`\Aimport\s+(\.|[A-Za-z0-9_]+)?\s*"([^"]+)"\s*(//.*)?\n\z`)
   330  	importBlockRE = regexp.MustCompile(`\A\s*(?:(\.|[A-Za-z0-9_]+)?\s*"([^"]+)")?\s*(//.*)?\n\z`)
   331  )
   332  
   333  func bootstrapFixImports(srcFile string) string {
   334  	text := readfile(srcFile)
   335  	lines := strings.SplitAfter(text, "\n")
   336  	inBlock := false
   337  	inComment := false
   338  	for i, line := range lines {
   339  		if strings.HasSuffix(line, "*/\n") {
   340  			inComment = false
   341  		}
   342  		if strings.HasSuffix(line, "/*\n") {
   343  			inComment = true
   344  		}
   345  		if inComment {
   346  			continue
   347  		}
   348  		if strings.HasPrefix(line, "import (") {
   349  			inBlock = true
   350  			continue
   351  		}
   352  		if inBlock && strings.HasPrefix(line, ")") {
   353  			inBlock = false
   354  			continue
   355  		}
   356  
   357  		var m []string
   358  		if !inBlock {
   359  			if !strings.HasPrefix(line, "import ") {
   360  				continue
   361  			}
   362  			m = importRE.FindStringSubmatch(line)
   363  			if m == nil {
   364  				fatalf("%s:%d: invalid import declaration: %q", srcFile, i+1, line)
   365  			}
   366  		} else {
   367  			m = importBlockRE.FindStringSubmatch(line)
   368  			if m == nil {
   369  				fatalf("%s:%d: invalid import block line", srcFile, i+1)
   370  			}
   371  			if m[2] == "" {
   372  				continue
   373  			}
   374  		}
   375  
   376  		path := m[2]
   377  		if strings.HasPrefix(path, "cmd/") {
   378  			path = "bootstrap/" + path
   379  		} else {
   380  			for _, dir := range bootstrapDirs {
   381  				if path == dir {
   382  					path = "bootstrap/" + dir
   383  					break
   384  				}
   385  			}
   386  		}
   387  
   388  		// Rewrite use of internal/reflectlite to be plain reflect.
   389  		if path == "internal/reflectlite" {
   390  			lines[i] = strings.ReplaceAll(line, `"reflect"`, `reflectlite "reflect"`)
   391  			continue
   392  		}
   393  
   394  		// Otherwise, reject direct imports of internal packages,
   395  		// since that implies knowledge of internal details that might
   396  		// change from one bootstrap toolchain to the next.
   397  		// There are many internal packages that are listed in
   398  		// bootstrapDirs and made into bootstrap copies based on the
   399  		// current repo's source code. Those are fine; this is catching
   400  		// references to internal packages in the older bootstrap toolchain.
   401  		if strings.HasPrefix(path, "internal/") {
   402  			fatalf("%s:%d: bootstrap-copied source file cannot import %s", srcFile, i+1, path)
   403  		}
   404  		if path != m[2] {
   405  			lines[i] = strings.ReplaceAll(line, `"`+m[2]+`"`, `"`+path+`"`)
   406  		}
   407  	}
   408  
   409  	lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   410  
   411  	return strings.Join(lines, "")
   412  }
   413
View as plain text