build: improve debian packaging

This commit tweaks the debian packaging tool:

* All build environment metadata can now be overriden on the command
  line. This allows testing the CI build behaviour locally.
* -unstable packages now actually contain the binaries (oops)
* packages use Go 1.7 to build
* archiving is skipped for PR builds

.travis.yml

@@ -27,7 +27,7 @@ matrix:
             - debhelper
             - dput
       script:
-        - go run build/ci.go travis-debsrc
+        - go run build/ci.go debsrc -signer "Felix Lange (Geth CI Testing Key) <fjl@twurst.com>" -upload ppa:lp-fjl/geth-ci-testing
 
 install:
   - go get golang.org/x/tools/cmd/cover

Makefile

@@ -42,12 +42,12 @@ geth-linux: geth-linux-386 geth-linux-amd64 geth-linux-arm geth-linux-mips64 get
 	@ls -ld $(GOBIN)/geth-linux-*
 
 geth-linux-386:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/386 -v ./cmd/geth
 	@echo "Linux 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep 386
 
 geth-linux-amd64:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/amd64 -v ./cmd/geth
 	@echo "Linux amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep amd64
 
@@ -56,32 +56,32 @@ geth-linux-arm: geth-linux-arm-5 geth-linux-arm-6 geth-linux-arm-7 geth-linux-ar
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm
 
 geth-linux-arm-5:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-5 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-5 -v ./cmd/geth
 	@echo "Linux ARMv5 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-5
 
 geth-linux-arm-6:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-6 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-6 -v ./cmd/geth
 	@echo "Linux ARMv6 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-6
 
 geth-linux-arm-7:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-7 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-7 -v ./cmd/geth
 	@echo "Linux ARMv7 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-7
 
 geth-linux-arm64:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/arm64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm64 -v ./cmd/geth
 	@echo "Linux ARM64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm64
 
 geth-linux-mips64:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64 -v ./cmd/geth
 	@echo "Linux MIPS64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep mips64
 
 geth-linux-mips64le:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64le -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64le -v ./cmd/geth
 	@echo "Linux MIPS64le cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep mips64le
 
@@ -90,12 +90,12 @@ geth-darwin: geth-darwin-386 geth-darwin-amd64
 	@ls -ld $(GOBIN)/geth-darwin-*
 
 geth-darwin-386:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=darwin/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=darwin/386 -v ./cmd/geth
 	@echo "Darwin 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-darwin-* | grep 386
 
 geth-darwin-amd64:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=darwin/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=darwin/amd64 -v ./cmd/geth
 	@echo "Darwin amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-darwin-* | grep amd64
 
@@ -104,21 +104,21 @@ geth-windows: geth-windows-386 geth-windows-amd64
 	@ls -ld $(GOBIN)/geth-windows-*
 
 geth-windows-386:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=windows/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=windows/386 -v ./cmd/geth
 	@echo "Windows 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-windows-* | grep 386
 
 geth-windows-amd64:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=windows/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=windows/amd64 -v ./cmd/geth
 	@echo "Windows amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-windows-* | grep amd64
 
 geth-android:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=android-21/aar -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=android-21/aar -v ./cmd/geth
 	@echo "Android cross compilation done:"
 	@ls -ld $(GOBIN)/geth-android-*
 
 geth-ios:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=ios-7.0/framework -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=ios-7.0/framework -v ./cmd/geth
 	@echo "iOS framework cross compilation done:"
 	@ls -ld $(GOBIN)/geth-ios-*

build/ci-notes.md

-Debian Packaging
-----------------
+# Debian Packaging
 
 Tagged releases and develop branch commits are available as installable Debian packages
 for Ubuntu. Packages are built for the all Ubuntu versions which are supported by
@@ -8,6 +7,7 @@ Canonical:
 - Trusty Tahr (14.04 LTS)
 - Wily Werewolf (15.10)
 - Xenial Xerus (16.04 LTS)
+- Yakkety Yak (16.10)
 
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
@@ -21,6 +21,29 @@ variable which Travis CI makes available to certain builds.
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
 our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.6, which is co-installable alongside the regular golang package. PPA dependencies
+golang-1.7, which is co-installable alongside the regular golang package. PPA dependencies
 can be edited at https://launchpad.net/%7Elp-fjl/+archive/ubuntu/geth-ci-testing/+edit-dependencies
 
+## Building Packages Locally (for testing)
+
+You need to run Ubuntu to do test packaging.
+
+Add the gophers PPA and install Go 1.7 and Debian packaging tools:
+
+    $ sudo apt-add-repository ppa:gophers/ubuntu/archive
+    $ sudo apt-get update
+    $ sudo apt-get install build-essential golang-1.7 devscripts debhelper
+
+Create the source packages:
+
+    $ go run build/ci.go debsrc -workdir dist
+
+Then go into the source package directory for your running distribution and build the package:
+
+    $ cd dist/ethereum-unstable-1.5.0+xenial
+    $ dpkg-buildpackage
+
+Built packages are placed in the dist/ directory.
+
+    $ cd ..
+    $ dpkg-deb -c geth-unstable_1.5.0+xenial_amd64.deb

build/deb.changelog

 {{.Name}} ({{.VersionString}}) {{.Distro}}; urgency=low
 
-  * git build of {{.Commit}}
+  * git build of {{.Env.Commit}}
 
  -- {{.Author}}  {{.Time}}

build/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.6
+Build-Depends: debhelper (>= 8.0.0), golang-1.7
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git

build/deb.rules

@@ -5,7 +5,7 @@
 #export DH_VERBOSE=1
 
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.6/bin/go run build/ci.go install -gitcommit {{.Commit}}
+	build/env.sh /usr/lib/go-1.7/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

internal/build/env.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+	"flag"
+	"fmt"
+	"os"
+)
+
+var (
+	// These flags override values in build env.
+	GitCommitFlag   = flag.String("git-commit", "", `Overrides git commit hash embedded into executables`)
+	GitBranchFlag   = flag.String("git-branch", "", `Overrides git branch being built`)
+	GitTagFlag      = flag.String("git-tag", "", `Overrides git tag being built`)
+	BuildnumFlag    = flag.String("buildnum", "", `Overrides CI build number`)
+	PullRequestFlag = flag.Bool("pull-request", false, `Overrides pull request status of the build`)
+)
+
+// Environment contains metadata provided by the build environment.
+type Environment struct {
+	Name                string // name of the environment
+	Repo                string // name of GitHub repo
+	Commit, Branch, Tag string // Git info
+	Buildnum            string
+	IsPullRequest       bool
+}
+
+func (env Environment) String() string {
+	return fmt.Sprintf("%s env (commit:%s branch:%s tag:%s buildnum:%s pr:%t)",
+		env.Name, env.Commit, env.Branch, env.Tag, env.Buildnum, env.IsPullRequest)
+}
+
+// Env returns metadata about the current CI environment, falling back to LocalEnv
+// if not running on CI.
+func Env() Environment {
+	switch {
+	case os.Getenv("CI") == "true" && os.Getenv("TRAVIS") == "true":
+		return Environment{
+			Name:          "travis",
+			Repo:          os.Getenv("TRAVIS_REPO_SLUG"),
+			Commit:        os.Getenv("TRAVIS_COMMIT"),
+			Branch:        os.Getenv("TRAVIS_BRANCH"),
+			Tag:           os.Getenv("TRAVIS_TAG"),
+			Buildnum:      os.Getenv("TRAVIS_BUILD_NUMBER"),
+			IsPullRequest: os.Getenv("TRAVIS_PULL_REQUEST") != "false",
+		}
+	case os.Getenv("CI") == "True" && os.Getenv("APPVEYOR") == "True":
+		return Environment{
+			Name:          "appveyor",
+			Repo:          os.Getenv("APPVEYOR_REPO_NAME"),
+			Commit:        os.Getenv("APPVEYOR_REPO_COMMIT"),
+			Branch:        os.Getenv("APPVEYOR_REPO_BRANCH"),
+			Tag:           os.Getenv("APPVEYOR_REPO_TAG"),
+			Buildnum:      os.Getenv("APPVEYOR_BUILD_NUMBER"),
+			IsPullRequest: os.Getenv("APPVEYOR_PULL_REQUEST_NUMBER") != "",
+		}
+	default:
+		return LocalEnv()
+	}
+}
+
+// LocalEnv returns build environment metadata gathered from git.
+func LocalEnv() Environment {
+	env := applyEnvFlags(Environment{Name: "local", Repo: "ethereum/go-ethereum"})
+	if _, err := os.Stat(".git"); err != nil {
+		return env
+	}
+	if env.Commit == "" {
+		env.Commit = RunGit("rev-parse", "HEAD")
+	}
+	if env.Branch == "" {
+		env.Branch = RunGit("symbolic-ref", "-q", "--short", "HEAD")
+	}
+	// Note that we don't get the current git tag. It would slow down
+	// builds and isn't used by anything.
+	return env
+}
+
+func applyEnvFlags(env Environment) Environment {
+	if !flag.Parsed() {
+		panic("you need to call flag.Parse before Env or LocalEnv")
+	}
+	if *GitCommitFlag != "" {
+		env.Commit = *GitCommitFlag
+	}
+	if *GitBranchFlag != "" {
+		env.Branch = *GitBranchFlag
+	}
+	if *GitTagFlag != "" {
+		env.Tag = *GitTagFlag
+	}
+	if *BuildnumFlag != "" {
+		env.Buildnum = *BuildnumFlag
+	}
+	if *PullRequestFlag {
+		env.IsPullRequest = true
+	}
+	return env
+}

internal/build/util.go

@@ -29,9 +29,7 @@ import (
 	"text/template"
 )
 
-var (
-	DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands")
-)
+var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands")
 
 // MustRun executes the given command and exits the host process for
 // any error.
@@ -69,6 +67,7 @@ func GOPATH() string {
 	return strings.Join(newpath, string(filepath.ListSeparator))
 }
 
+// VERSION returns the content of the VERSION file.
 func VERSION() string {
 	version, err := ioutil.ReadFile("VERSION")
 	if err != nil {
@@ -77,10 +76,8 @@ func VERSION() string {
 	return string(bytes.TrimSpace(version))
 }
 
-func GitCommit() string {
-	return RunGit("rev-parse", "HEAD")
-}
-
+// RunGit runs a git subcommand and returns its output.
+// The command must complete successfully.
 func RunGit(args ...string) string {
 	cmd := exec.Command("git", args...)
 	var stdout, stderr bytes.Buffer
@@ -94,12 +91,13 @@ func RunGit(args ...string) string {
 	return strings.TrimSpace(stdout.String())
 }
 
-// Render renders the given template file.
+// Render renders the given template file into outputFile.
 func Render(templateFile, outputFile string, outputPerm os.FileMode, x interface{}) {
 	tpl := template.Must(template.ParseFiles(templateFile))
 	render(tpl, outputFile, outputPerm, x)
 }
 
+// RenderString renders the given template string into outputFile.
 func RenderString(templateContent, outputFile string, outputPerm os.FileMode, x interface{}) {
 	tpl := template.Must(template.New("").Parse(templateContent))
 	render(tpl, outputFile, outputPerm, x)