travis, build: implement uploading archives to azure

.travis.yml

@@ -6,6 +6,8 @@ matrix:
     - os: linux
       dist: trusty
       go: 1.5.4
+      env:
+        - GO15VENDOREXPERIMENT=1
     - os: linux
       dist: trusty
       go: 1.6.2
@@ -15,11 +17,13 @@ matrix:
     - os: osx
       go: 1.7
 
-    # This builder does the PPA upload (and nothing else).
+    # This builder does the Ubuntu PPA and Linux Azure uploads
     - os: linux
       dist: trusty
       go: 1.7
-      env: PPA
+      env:
+        - ubuntu-ppa
+        - azure-linux-amd64
       addons:
         apt:
           packages:
@@ -28,14 +32,23 @@ matrix:
             - dput
       script:
         - go run build/ci.go debsrc -signer "Felix Lange (Geth CI Testing Key) <fjl@twurst.com>" -upload ppa:lp-fjl/geth-ci-testing
+        - go run build/ci.go install
+        - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+
+    # This builder does the OSX Azure uploads
+    - os: osx
+      go: 1.7
+      env:
+        - azure-osx-amd64
+      script:
+        - go run build/ci.go install
+        - go run build/ci.go archive -type zip -signer OSX_SIGNING_KEY -upload gethstore/builds
 
 install:
   - go get golang.org/x/tools/cmd/cover
 script:
   - go run build/ci.go install
   - go run build/ci.go test -coverage -vet
-after_success:
-  # - go run build/ci.go archive -type tar
 
 notifications:
   webhooks:

build/ci.go

@@ -23,12 +23,12 @@ Usage: go run ci.go <command> <command flags/arguments>
 
 Available commands are:
 
-   install    [ packages... ]                          -- builds packages and executables
-   test       [ -coverage ] [ -vet ] [ packages... ]   -- runs the tests
-   archive    [ -type zip|tar ]                        -- archives build artefacts
-   importkeys                                          -- imports signing keys from env
-   debsrc     [ -sign key-id ] [ -upload dest ]        -- creates a debian source package
-   xgo        [ options ]                              -- cross builds according to options
+   install    [ packages... ]                                           -- builds packages and executables
+   test       [ -coverage ] [ -vet ] [ packages... ]                    -- runs the tests
+   archive    [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
+   importkeys                                                           -- imports signing keys from env
+   debsrc     [ -signer key-id ] [ -upload dest ]                       -- creates a debian source package
+   xgo        [ options ]                                               -- cross builds according to options
 
 For all commands, -n prevents execution of external programs (dry run mode).
 
@@ -49,7 +49,7 @@ import (
 	"strings"
 	"time"
 
-	"../internal/build"
+	"github.com/ethereum/go-ethereum/internal/build"
 )
 
 var (
@@ -239,8 +239,10 @@ func doTest(cmdline []string) {
 
 func doArchive(cmdline []string) {
 	var (
-		atype = flag.String("type", "zip", "Type of archive to write (zip|tar)")
-		ext   string
+		atype  = flag.String("type", "zip", "Type of archive to write (zip|tar)")
+		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`)
+		upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
+		ext    string
 	)
 	flag.CommandLine.Parse(cmdline)
 	switch *atype {
@@ -262,6 +264,12 @@ func doArchive(cmdline []string) {
 	if err := build.WriteArchive("geth-alltools-"+base, ext, allToolsArchiveFiles); err != nil {
 		log.Fatal(err)
 	}
+
+	for _, archive := range []string{"geth-" + base + ext, "geth-alltools-" + base + ext} {
+		if err := archiveUpload(archive, *upload, *signer); err != nil {
+			log.Fatal(err)
+		}
+	}
 }
 
 func archiveBasename(env build.Environment) string {
@@ -274,6 +282,36 @@ func archiveBasename(env build.Environment) string {
 	return archive
 }
 
+func archiveUpload(archive string, blobstore string, signer string) error {
+	// If signing was requested, generate the signature files
+	if signer != "" {
+		pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
+		if err != nil {
+			return fmt.Errorf("invalid base64 %s", signer)
+		}
+		if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
+			return err
+		}
+	}
+	// If uploading to Azure was requested, push the archive possibly with its signature
+	if blobstore != "" {
+		auth := build.AzureBlobstoreConfig{
+			Account:   strings.Split(blobstore, "/")[0],
+			Token:     os.Getenv("AZURE_BLOBSTORE_TOKEN"),
+			Container: strings.SplitN(blobstore, "/", 2)[1],
+		}
+		if err := build.AzureBlobstoreUpload(archive, archive, auth); err != nil {
+			return err
+		}
+		if signer != "" {
+			if err := build.AzureBlobstoreUpload(archive+".asc", archive+".asc", auth); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // skips archiving for some build configurations.
 func maybeSkipArchive(env build.Environment) {
 	if env.IsPullRequest {

internal/build/azure.go

+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+	"os"
+
+	"github.com/Azure/azure-sdk-for-go/storage"
+)
+
+// AzureBlobstoreConfig is an authentication and configuration struct containing
+// the data needed by the Azure SDK to interact with a speicifc container in the
+// blobstore.
+type AzureBlobstoreConfig struct {
+	Account   string // Account name to authorize API requests with
+	Token     string // Access token for the above account
+	Container string // Blob container to upload files into
+}
+
+// AzureBlobstoreUpload uploads a local file to the Azure Blob Storage. Note, this
+// method assumes a max file size of 64MB (Azure limitation). Larger files will
+// need a multi API call approach implemented.
+//
+// See: https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx#Anchor_3
+func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig) error {
+	// Create an authenticated client against the Azure cloud
+	rawClient, err := storage.NewBasicClient(config.Account, config.Token)
+	if err != nil {
+		return err
+	}
+	client := rawClient.GetBlobService()
+
+	// Stream the file to upload into the designated blobstore container
+	in, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	info, err := in.Stat()
+	if err != nil {
+		return err
+	}
+	return client.CreateBlockBlobFromReader(config.Container, name, uint64(info.Size()), in, nil)
+}

internal/build/pgp.go

+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// signFile reads the contents of an input file and signs it (in armored format)
+// with the key provided, placing the signature into the output file.
+
+package build
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+
+	"golang.org/x/crypto/openpgp"
+)
+
+// PGPSignFile parses a PGP private key from the specified string and creates a
+// signature file into the output parameter of the input file.
+//
+// Note, this method assumes a single key will be container in the pgpkey arg,
+// furthermore that it is in armored format.
+func PGPSignFile(input string, output string, pgpkey string) error {
+	// Parse the keyring and make sure we only have a single private key in it
+	keys, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(pgpkey))
+	if err != nil {
+		return err
+	}
+	if len(keys) != 1 {
+		return fmt.Errorf("key count mismatch: have %d, want %d", len(keys), 1)
+	}
+	// Create the input and output streams for signing
+	in, err := os.Open(input)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(output)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	// Generate the signature and return
+	return openpgp.ArmoredDetachSignText(out, keys[0], in, nil)
+}