build: add ci.go, use it everywhere

The new build script, ci.go, replaces some of the older shell scripts.
ci.go can compile go-ethereum, run the tests, create release archives
and debian source packages.

.gitignore

@@ -23,17 +23,11 @@ Godeps/_workspace/bin
 .project
 .settings
 
-deploy/osx/Mist.app
-deploy/osx/Mist\ Installer.dmg
-cmd/mist/assets/ext/ethereum.js/
-
 # used by the Makefile
 /build/_workspace/
 /build/bin/
+/geth*.zip
 
 # travis
 profile.tmp
 profile.cov
-
-# vagrant
-.vagrant

.gitmodules

-[submodule "cmd/mist/assets/ext/ethereum.js"]
-	path = cmd/mist/assets/ext/ethereum.js
-	url = https://github.com/ethereum/web3.js

.travis.yml

 language: go
-go:
-  - 1.4.2
-  - 1.5.4
-  - 1.6.2
+go_import_path: github.com/ethereum/go-ethereum
+sudo: false
+matrix:
+  include:
+    - os: linux
+      dist: trusty
+      go: 1.4.2
+    - os: linux
+      dist: trusty
+      go: 1.5.4
+    - os: linux
+      dist: trusty
+      go: 1.6.2
+    - os: osx
+      go: 1.6.2
+
+    # This builder does the PPA upload (and nothing else).
+    - os: linux
+      dist: trusty
+      go: 1.6.2
+      env: PPA
+      addons:
+        apt:
+          packages:
+            - devscripts
+            - debhelper
+            - dput
+      script:
+        - go run build/ci.go travis-debsrc
+
 install:
-  # - go get code.google.com/p/go.tools/cmd/goimports
-  # - go get github.com/golang/lint/golint
-  # - go get golang.org/x/tools/cmd/vet
   - go get golang.org/x/tools/cmd/cover
-before_script:
-  # - gofmt -l -w .
-  # - goimports -l -w .
-  # - golint .
-  # - go vet ./...
-  # - go test -race ./...
 script:
-  - make travis-test-with-coverage
+  - go run build/ci.go install
+  - go run build/ci.go test -coverage -vet
 after_success:
-  - bash <(curl -s https://codecov.io/bash)
-env:
-  global:
-    - secure: "U2U1AmkU4NJBgKR/uUAebQY87cNL0+1JHjnLOmmXwxYYyj5ralWb1aSuSH3qSXiT93qLBmtaUkuv9fberHVqrbAeVlztVdUsKAq7JMQH+M99iFkC9UiRMqHmtjWJ0ok4COD1sRYixxi21wb/JrMe3M1iL4QJVS61iltjHhVdM64="
-sudo: false
+  # - go run build/ci.go archive -type tar
+
 notifications:
   webhooks:
     urls:
       - https://webhooks.gitter.im/e/e09ccdce1048c5e03445
     on_success: change
     on_failure: always
-    on_start: false

Makefile

@@ -2,7 +2,7 @@
 # with Go source code. If you know what GOPATH is then you probably
 # don't need to bother with make.
 
-.PHONY: geth geth-cross evm all test travis-test-with-coverage xgo clean
+.PHONY: geth geth-cross evm all test xgo clean
 .PHONY: geth-linux geth-linux-386 geth-linux-amd64
 .PHONY: geth-linux-arm geth-linux-arm-5 geth-linux-arm-6 geth-linux-arm-7 geth-linux-arm64
 .PHONY: geth-darwin geth-darwin-386 geth-darwin-amd64
@@ -13,10 +13,29 @@ GOBIN = build/bin
 GO ?= latest
 
 geth:
-	build/env.sh go build -i -v $(shell build/flags.sh) -o $(GOBIN)/geth ./cmd/geth
+	build/env.sh go run build/ci.go install ./cmd/geth
 	@echo "Done building."
 	@echo "Run \"$(GOBIN)/geth\" to launch geth."
 
+evm:
+	build/env.sh go run build/ci.go install ./cmd/evm
+	@echo "Done building."
+	@echo "Run \"$(GOBIN)/evm to start the evm."
+
+all:
+	build/env.sh go run build/ci.go install
+
+test: all
+	build/env.sh go run build/ci.go test
+
+clean:
+	rm -fr build/_workspace/pkg/ Godeps/_workspace/pkg $(GOBIN)/*
+
+# Cross Compilation Targets (xgo)
+
+xgo:
+	build/env.sh go get github.com/karalabe/xgo
+
 geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios
 	@echo "Full cross compilation done:"
 	@ls -ld $(GOBIN)/geth-*
@@ -96,26 +115,3 @@ geth-ios: xgo
 	build/env.sh $(GOBIN)/xgo --go=$(GO) --dest=$(GOBIN) --targets=ios-7.0/framework -v $(shell build/flags.sh) ./cmd/geth
 	@echo "iOS framework cross compilation done:"
 	@ls -ld $(GOBIN)/geth-ios-*
-
-evm:
-	build/env.sh $(GOROOT)/bin/go install -v $(shell build/flags.sh) ./cmd/evm
-	@echo "Done building."
-	@echo "Run \"$(GOBIN)/evm to start the evm."
-
-all:
-	for cmd in `ls ./cmd/`; do \
-		 build/env.sh go build -i -v $(shell build/flags.sh) -o $(GOBIN)/$$cmd ./cmd/$$cmd; \
-	done
-
-test: all
-	build/env.sh go test ./...
-
-travis-test-with-coverage: all
-	build/env.sh go vet ./...
-	build/env.sh build/test-global-coverage.sh
-
-xgo:
-	build/env.sh go get github.com/karalabe/xgo
-
-clean:
-	rm -fr build/_workspace/pkg/ Godeps/_workspace/pkg $(GOBIN)/*

appveyor.yml

+os: Visual Studio 2015
+
+# Clone directly into GOPATH.
+clone_folder: c:\gopath\src\github.com\ethereum\go-ethereum
+clone_depth: 5
+version: "{branch}.{build}"
+environment:
+  global:
+    GOPATH: c:\gopath
+
+# cache choco package files so we don't hit sourceforge all
+# the time.
+cache:
+  - c:\cache
+
+install:
+  - cmd: choco install --cache c:\cache golang mingw | find /v "Extracting  "
+  - refreshenv
+  - cd c:\gopath\src\github.com\ethereum\go-ethereum
+
+build_script:
+  - go run build\ci.go install
+
+test_script:
+  - go run build\ci.go test -vet -coverage
+
+after_build:
+  - go run build\ci.go archive -type zip
+
+artifacts:
+  - path: geth-*.zip

build/ci-notes.md

+Debian Packaging
+----------------
+
+Tagged releases and develop branch commits are available as installable Debian packages
+for Ubuntu. Packages are built for the all Ubuntu versions which are supported by
+Canonical:
+
+- Trusty Tahr (14.04 LTS)
+- Wily Werewolf (15.10)
+- Xenial Xerus (16.04 LTS)
+
+Packages of develop branch commits have suffix -unstable and cannot be installed alongside
+the stable version. Switching between release streams requires user intervention.
+
+The packages are built and served by launchpad.net. We generate a Debian source package
+for each distribution and upload it. Their builder picks up the source package, builds it
+and installs the new version into the PPA repository. Launchpad requires a valid signature
+by a team member for source package uploads. The signing key is stored in an environment
+variable which Travis CI makes available to certain builds.
+
+We want to build go-ethereum with the most recent version of Go, irrespective of the Go
+version that is available in the main Ubuntu repository. In order to make this possible,
+our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
+golang-1.6, which is co-installable alongside the regular golang package. PPA dependencies
+can be edited at https://launchpad.net/%7Elp-fjl/+archive/ubuntu/geth-ci-testing/+edit-dependencies
+

build/deb.changelog

+{{.Name}} ({{.VersionString}}) {{.Distro}}; urgency=low
+
+  * git build of {{.Commit}}
+
+ -- {{.Author}}  {{.Time}}

build/deb.control

+Source: {{.Name}}
+Section: science
+Priority: extra
+Maintainer: {{.Author}}
+Build-Depends: debhelper (>= 8.0.0), golang-1.6
+Standards-Version: 3.9.5
+Homepage: https://ethereum.org
+Vcs-Git: git://github.com/ethereum/go-ethereum.git
+Vcs-Browser: https://github.com/ethereum/go-ethereum
+
+Package: {{.Name}}
+Architecture: any
+Depends: ${misc:Depends}, {{.ExeList}}
+Description: Meta-package to install geth and other tools
+ Meta-package to install geth and other tools
+ 
+{{range .Executables}}
+Package: {{$.ExeName .}}
+Conflicts: {{$.ExeConflicts .}}
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Built-Using: ${misc:Built-Using}
+Description: {{.Description}}
+ {{.Description}}
+{{end}}

build/deb.copyright

+Copyright 2016 The go-ethereum Authors
+
+go-ethereum is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+go-ethereum is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

build/deb.docs

+AUTHORS

build/deb.install

+build/bin/{{.Name}} usr/bin

build/deb.rules

+#!/usr/bin/make -f
+# -*- makefile -*-
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+override_dh_auto_build:
+	build/env.sh /usr/lib/go-1.6/bin/go run build/ci.go install -gitcommit {{.Commit}}
+
+override_dh_auto_test:
+
+%:
+	dh $@

build/env.sh

@@ -20,9 +20,8 @@ fi
 
 # Set up the environment to use the workspace.
 # Also add Godeps workspace so we build using canned dependencies.
-GOPATH="$ethdir/go-ethereum/Godeps/_workspace:$workspace"
-GOBIN="$PWD/build/bin"
-export GOPATH GOBIN
+GOPATH="$workspace"
+export GOPATH
 
 # Run the command inside the workspace.
 cd "$ethdir/go-ethereum"

build/test-global-coverage.sh

-#!/usr/bin/env bash
-
-set -e
-echo "" > coverage.txt
-
-for d in $(find ./* -maxdepth 10 -type d -not -path "./build" -not -path "./Godeps/*" ); do
-    if ls $d/*.go &> /dev/null; then
-        go test  -coverprofile=profile.out -covermode=atomic $d
-        if [ -f profile.out ]; then
-            cat profile.out >> coverage.txt
-            echo '<<<<<< EOF' >> coverage.txt
-            rm profile.out
-        fi
-    fi
-done

build/win-ci-compile.bat

-@echo off
-if not exist .\build\win-ci-compile.bat (
-   echo This script must be run from the root of the repository.
-   exit /b
-)
-if not defined GOPATH (
-   echo GOPATH is not set.
-   exit /b
-)
-
-set GOPATH=%GOPATH%;%cd%\Godeps\_workspace
-set GOBIN=%cd%\build\bin
-
-rem set gitCommit when running from a Git checkout.
-set goLinkFlags=""
-if exist ".git\HEAD" (
-   where /q git
-   if not errorlevel 1 (
-      for /f %%h in ('git rev-parse HEAD') do (
-          set goLinkFlags="-X main.gitCommit=%%h"
-      )
-   )
-)
-
-@echo on
-go install -v -ldflags %goLinkFlags% ./...

build/win-ci-test.bat

-@echo off
-if not exist .\build\win-ci-test.bat (
-   echo This script must be run from the root of the repository.
-   exit /b
-)
-if not defined GOPATH (
-   echo GOPATH is not set.
-   exit /b
-)
-
-set GOPATH=%GOPATH%;%cd%\Godeps\_workspace
-set GOBIN=%cd%\build\bin
-
-@echo on
-go test ./...

internal/build/archive.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type Archive interface {
+	// Directory adds a new directory entry to the archive and sets the
+	// directory for subsequent calls to Header.
+	Directory(name string) error
+
+	// Header adds a new file to the archive. The file is added to the directory
+	// set by Directory. The content of the file must be written to the returned
+	// writer.
+	Header(os.FileInfo) (io.Writer, error)
+
+	// Close flushes the archive and closes the underlying file.
+	Close() error
+}
+
+func NewArchive(file *os.File) Archive {
+	switch {
+	case strings.HasSuffix(file.Name(), ".zip"):
+		return NewZipArchive(file)
+	case strings.HasSuffix(file.Name(), ".tar.gz"):
+		return NewTarballArchive(file)
+	default:
+		return nil
+	}
+}
+
+// AddFile appends an existing file to an archive.
+func AddFile(a Archive, file string) error {
+	fd, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+	fi, err := fd.Stat()
+	if err != nil {
+		return err
+	}
+	w, err := a.Header(fi)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(w, fd); err != nil {
+		return err
+	}
+	return nil
+}
+
+// WriteArchive creates an archive containing the given files.
+func WriteArchive(basename, ext string, files []string) error {
+	archfd, err := os.Create(basename + ext)
+	if err != nil {
+		return err
+	}
+	defer archfd.Close()
+	archive := NewArchive(archfd)
+	if archive == nil {
+		return fmt.Errorf("unknown archive extension: %s", ext)
+	}
+	fmt.Println(basename + ext)
+	if err := archive.Directory(basename); err != nil {
+		return err
+	}
+	for _, file := range files {
+		fmt.Println("   +", filepath.Base(file))
+		if err := AddFile(archive, file); err != nil {
+			return err
+		}
+	}
+	return archive.Close()
+}
+
+type ZipArchive struct {
+	dir  string
+	zipw *zip.Writer
+	file io.Closer
+}
+
+func NewZipArchive(w io.WriteCloser) Archive {
+	return &ZipArchive{"", zip.NewWriter(w), w}
+}
+
+func (a *ZipArchive) Directory(name string) error {
+	a.dir = name + "/"
+	return nil
+}
+
+func (a *ZipArchive) Header(fi os.FileInfo) (io.Writer, error) {
+	head, err := zip.FileInfoHeader(fi)
+	if err != nil {
+		return nil, fmt.Errorf("can't make zip header: %v", err)
+	}
+	head.Name = a.dir + head.Name
+	w, err := a.zipw.CreateHeader(head)
+	if err != nil {
+		return nil, fmt.Errorf("can't add zip header: %v", err)
+	}
+	return w, nil
+}
+
+func (a *ZipArchive) Close() error {
+	if err := a.zipw.Close(); err != nil {
+		return err
+	}
+	return a.file.Close()
+}
+
+type TarballArchive struct {
+	dir  string
+	tarw *tar.Writer
+	gzw  *gzip.Writer
+	file io.Closer
+}
+
+func NewTarballArchive(w io.WriteCloser) Archive {
+	gzw := gzip.NewWriter(w)
+	tarw := tar.NewWriter(gzw)
+	return &TarballArchive{"", tarw, gzw, w}
+}
+
+func (a *TarballArchive) Directory(name string) error {
+	a.dir = name + "/"
+	return a.tarw.WriteHeader(&tar.Header{
+		Name:     a.dir,
+		Mode:     0755,
+		Typeflag: tar.TypeDir,
+	})
+}
+
+func (a *TarballArchive) Header(fi os.FileInfo) (io.Writer, error) {
+	head, err := tar.FileInfoHeader(fi, "")
+	if err != nil {
+		return nil, fmt.Errorf("can't make tar header: %v", err)
+	}
+	head.Name = a.dir + head.Name
+	if err := a.tarw.WriteHeader(head); err != nil {
+		return nil, fmt.Errorf("can't add tar header: %v", err)
+	}
+	return a.tarw, nil
+}
+
+func (a *TarballArchive) Close() error {
+	if err := a.tarw.Close(); err != nil {
+		return err
+	}
+	if err := a.gzw.Close(); err != nil {
+		return err
+	}
+	return a.file.Close()
+}

internal/build/util.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"text/template"
+)
+
+var (
+	DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands")
+)
+
+// MustRun executes the given command and exits the host process for
+// any error.
+func MustRun(cmd *exec.Cmd) {
+	fmt.Println(">>>", strings.Join(cmd.Args, " "))
+	if !*DryRunFlag {
+		cmd.Stderr = os.Stderr
+		cmd.Stdout = os.Stdout
+		if err := cmd.Run(); err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func MustRunCommand(cmd string, args ...string) {
+	MustRun(exec.Command(cmd, args...))
+}
+
+// GOPATH returns the value that the GOPATH environment
+// variable should be set to.
+func GOPATH() string {
+	path := filepath.SplitList(os.Getenv("GOPATH"))
+	if len(path) == 0 {
+		log.Fatal("GOPATH is not set")
+	}
+	// Ensure Godeps workspace is present in the path.
+	godeps, _ := filepath.Abs(filepath.Join("Godeps", "_workspace"))
+	for _, dir := range path {
+		if dir == godeps {
+			return strings.Join(path, string(filepath.ListSeparator))
+		}
+	}
+	newpath := append(path[:1], godeps)
+	newpath = append(newpath, path[1:]...)
+	return strings.Join(newpath, string(filepath.ListSeparator))
+}
+
+func VERSION() string {
+	version, err := ioutil.ReadFile("VERSION")
+	if err != nil {
+		log.Fatal(err)
+	}
+	return string(bytes.TrimSpace(version))
+}
+
+func GitCommit() string {
+	return RunGit("rev-parse", "HEAD")
+}
+
+func RunGit(args ...string) string {
+	cmd := exec.Command("git", args...)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout, cmd.Stderr = &stdout, &stderr
+	if err := cmd.Run(); err == exec.ErrNotFound {
+		log.Println("no git in PATH")
+		return ""
+	} else if err != nil {
+		log.Fatal(strings.Join(cmd.Args, " "), ": ", err, "\n", stderr.String())
+	}
+	return strings.TrimSpace(stdout.String())
+}
+
+// Render renders the given template file.
+func Render(templateFile, outputFile string, outputPerm os.FileMode, x interface{}) {
+	tpl := template.Must(template.ParseFiles(templateFile))
+	render(tpl, outputFile, outputPerm, x)
+}
+
+func RenderString(templateContent, outputFile string, outputPerm os.FileMode, x interface{}) {
+	tpl := template.Must(template.New("").Parse(templateContent))
+	render(tpl, outputFile, outputPerm, x)
+}
+
+func render(tpl *template.Template, outputFile string, outputPerm os.FileMode, x interface{}) {
+	if err := os.MkdirAll(filepath.Dir(outputFile), 0755); err != nil {
+		log.Fatal(err)
+	}
+	out, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY|os.O_EXCL, outputPerm)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := tpl.Execute(out, x); err != nil {
+		log.Fatal(err)
+	}
+	if err := out.Close(); err != nil {
+		log.Fatal(err)
+	}
+}