build: add -dlgo flag in ci.go (#21824)

This new flag downloads a known version of Go and builds with it. This
is meant for environments where we can't easily upgrade the installed Go
version.

* .travis.yml: remove install step for PR test builders

We added this step originally to avoid re-building everything
for every test. go test has become much smarter in recent go
releases, so we no longer need to install anything here.

.travis.yml

@@ -31,7 +31,6 @@ jobs:
       env:
         - GO111MODULE=on
       script:
-        - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # These are the latest Go versions.
@@ -43,7 +42,6 @@ jobs:
       env:
         - GO111MODULE=on
       script:
-        - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - stage: build
@@ -55,7 +53,6 @@ jobs:
       env:
         - GO111MODULE=on
       script:
-        - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - stage: build
@@ -74,7 +71,6 @@ jobs:
         - ulimit -S -n $NOFILE
         - ulimit -n
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
-        - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # This builder does the Ubuntu PPA upload
@@ -99,7 +95,7 @@ jobs:
             - python-paramiko
       script:
         - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
-        - go run build/ci.go debsrc -goversion 1.15 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
+        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Linux Azure uploads
     - stage: build
@@ -119,22 +115,22 @@ jobs:
             - gcc-multilib
       script:
         # Build for the primary platforms that Trusty can manage
-        - go run build/ci.go install
+        - go run build/ci.go install -dlgo
         - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - go run build/ci.go install -arch 386
+        - go run build/ci.go install -dlgo -arch 386
         - go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
         # Switch over GCC to cross compilation (breaks 386, hence why do it here only)
         - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
         - sudo ln -s /usr/include/asm-generic /usr/include/asm
 
-        - GOARM=5 go run build/ci.go install -arch arm -cc arm-linux-gnueabi-gcc
+        - GOARM=5 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
         - GOARM=5 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - GOARM=6 go run build/ci.go install -arch arm -cc arm-linux-gnueabi-gcc
+        - GOARM=6 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
         - GOARM=6 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - GOARM=7 go run build/ci.go install -arch arm -cc arm-linux-gnueabihf-gcc
+        - GOARM=7 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabihf-gcc
         - GOARM=7 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - go run build/ci.go install -arch arm64 -cc aarch64-linux-gnu-gcc
+        - go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Linux Azure MIPS xgo uploads
@@ -219,7 +215,7 @@ jobs:
       git:
         submodules: false # avoid cloning ethereum/tests
       script:
-        - go run build/ci.go install
+        - go run build/ci.go install -dlgo
         - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds
 
         # Build the iOS framework and upload it to CocoaPods and Azure

appveyor.yml

@@ -30,7 +30,7 @@ install:
   - gcc --version
 
 build_script:
-  - go run build\ci.go install
+  - go run build\ci.go install -dlgo
 
 after_build:
   - go run build\ci.go archive -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds

build/checksums.txt

 # This file contains sha256 checksums of optional build dependencies.
 
-69438f7ed4f532154ffaf878f3dfd83747e7a00b70b3556eddabf7aaee28ac3a  go1.15.src.tar.gz
+063da6a9a4186b8118a0e584532c8c94e65582e2cd951ed078bfd595d27d2367  go1.15.4.src.tar.gz
+aaf8c5323e0557211680960a8f51bedf98ab9a368775a687d6cf1f0079232b1d  go1.15.4.darwin-amd64.tar.gz
+6b2f6d8afddfb198bf0e36044084dc4db4cb0be1107375240b34d215aa5ff6ad  go1.15.4.linux-386.tar.gz
+eb61005f0b932c93b424a3a4eaa67d72196c79129d9a3ea8578047683e2c80d5  go1.15.4.linux-amd64.tar.gz
+6f083b453484fc5f95afb345547a58ccc957cde91348b7a7c68f5b060e488c85  go1.15.4.linux-arm64.tar.gz
+fe449ad3e121472e5db2f70becc0fef9d1a7188616c0605ada63f1e3bbad280e  go1.15.4.linux-armv6l.tar.gz
+3be3cfc08ccc7e7056fdee17b6f5d18e9d7f3d1351dcfec8de34b1c95cb05b50  go1.15.4.windows-386.zip
+3593204e3851be577e4209900ece031b36f1e9ce1671f3f3221c9af7a090a941  go1.15.4.windows-amd64.zip
 
 d998a84eea42f2271aca792a7b027ca5c1edfcba229e8e5a844c9ac3f336df35  golangci-lint-1.27.0-linux-armv7.tar.gz
 bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4  golangci-lint.exe-1.27.0-windows-amd64.zip

internal/build/archive.go

@@ -184,24 +184,35 @@ func (a *TarballArchive) Close() error {
 	return a.file.Close()
 }
 
-func ExtractTarballArchive(archive string, dest string) error {
-	// We're only interested in gzipped archives, wrap the reader now
+// ExtractArchive unpacks a .zip or .tar.gz archive to the destination directory.
+func ExtractArchive(archive string, dest string) error {
 	ar, err := os.Open(archive)
 	if err != nil {
 		return err
 	}
 	defer ar.Close()
 
+	switch {
+	case strings.HasSuffix(archive, ".tar.gz"):
+		return extractTarball(ar, dest)
+	case strings.HasSuffix(archive, ".zip"):
+		return extractZip(ar, dest)
+	default:
+		return fmt.Errorf("unhandled archive type %s", archive)
+	}
+}
+
+// extractTarball unpacks a .tar.gz file.
+func extractTarball(ar io.Reader, dest string) error {
 	gzr, err := gzip.NewReader(ar)
 	if err != nil {
 		return err
 	}
 	defer gzr.Close()
 
-	// Iterate over all the files in the tarball
 	tr := tar.NewReader(gzr)
 	for {
-		// Fetch the next tarball header and abort if needed
+		// Move to the next file header.
 		header, err := tr.Next()
 		if err != nil {
 			if err == io.EOF {
@@ -209,22 +220,69 @@ func ExtractTarballArchive(archive string, dest string) error {
 			}
 			return err
 		}
-		// Figure out the target and create it
-		target := filepath.Join(dest, header.Name)
-
-		switch header.Typeflag {
-		case tar.TypeReg:
-			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
-				return err
-			}
-			file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+		// We only care about regular files, directory modes
+		// and special file types are not supported.
+		if header.Typeflag == tar.TypeReg {
+			armode := header.FileInfo().Mode()
+			err := extractFile(header.Name, armode, tr, dest)
 			if err != nil {
-				return err
-			}
-			if _, err := io.Copy(file, tr); err != nil {
-				return err
+				return fmt.Errorf("extract %s: %v", header.Name, err)
 			}
-			file.Close()
 		}
 	}
 }
+
+// extractZip unpacks the given .zip file.
+func extractZip(ar *os.File, dest string) error {
+	info, err := ar.Stat()
+	if err != nil {
+		return err
+	}
+	zr, err := zip.NewReader(ar, info.Size())
+	if err != nil {
+		return err
+	}
+
+	for _, zf := range zr.File {
+		if !zf.Mode().IsRegular() {
+			continue
+		}
+
+		data, err := zf.Open()
+		if err != nil {
+			return err
+		}
+		err = extractFile(zf.Name, zf.Mode(), data, dest)
+		data.Close()
+		if err != nil {
+			return fmt.Errorf("extract %s: %v", zf.Name, err)
+		}
+	}
+	return nil
+}
+
+// extractFile extracts a single file from an archive.
+func extractFile(arpath string, armode os.FileMode, data io.Reader, dest string) error {
+	// Check that path is inside destination directory.
+	target := filepath.Join(dest, filepath.FromSlash(arpath))
+	if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) {
+		return fmt.Errorf("path %q escapes archive destination", target)
+	}
+
+	// Ensure the destination directory exists.
+	if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
+		return err
+	}
+
+	// Copy file data.
+	file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, armode)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(file, data); err != nil {
+		file.Close()
+		os.Remove(target)
+		return err
+	}
+	return file.Close()
+}

internal/build/util.go

@@ -20,6 +20,8 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
+	"go/parser"
+	"go/token"
 	"io"
 	"io/ioutil"
 	"log"
@@ -152,3 +154,28 @@ func UploadSFTP(identityFile, host, dir string, files []string) error {
 	stdin.Close()
 	return sftp.Wait()
 }
+
+// FindMainPackages finds all 'main' packages in the given directory and returns their
+// package paths.
+func FindMainPackages(dir string) []string {
+	var commands []string
+	cmds, err := ioutil.ReadDir(dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, cmd := range cmds {
+		pkgdir := filepath.Join(dir, cmd.Name())
+		pkgs, err := parser.ParseDir(token.NewFileSet(), pkgdir, nil, parser.PackageClauseOnly)
+		if err != nil {
+			log.Fatal(err)
+		}
+		for name := range pkgs {
+			if name == "main" {
+				path := "./" + filepath.ToSlash(pkgdir)
+				commands = append(commands, path)
+				break
+			}
+		}
+	}
+	return commands
+}