swarm: plan bee for content storage and distribution on web3

This change imports the Swarm protocol codebase. Compared to the 'swarm'
branch, a few mostly cosmetic changes had to be made:

* The various redundant log message prefixes are gone.
* All files now have LGPLv3 license headers.
* Minor code changes were needed to please go vet and make the tests
  pass on Windows.
* Further changes were required to adapt to the go-ethereum develop
  branch and its new Go APIs.

Some code has not (yet) been brought over:

* swarm/cmd/bzzhash: will reappear as cmd/bzzhash later
* swarm/cmd/bzzup.sh: will be reimplemented in cmd/bzzup
* swarm/cmd/makegenesis: will reappear somehow
* swarm/examples/album: will move to a separate repository
* swarm/examples/filemanager: ditto
* swarm/examples/files: will not be merged
* swarm/test/*: will not be merged
* swarm/services/swear: will reappear as contracts/swear when needed

contracts/chequebook/cheque.go

@@ -42,6 +42,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/swarm/services/swap/swap"
 	"golang.org/x/net/context"
 )
 
@@ -408,8 +409,7 @@ func NewOutbox(chbook *Chequebook, beneficiary common.Address) *Outbox {
 }
 
 // Issue creates cheque.
-func (self *Outbox) Issue(amount *big.Int) (interface{}, error) {
-	// TODO(fjl): the return type should be more descriptive.
+func (self *Outbox) Issue(amount *big.Int) (swap.Promise, error) {
 	return self.chequeBook.Issue(self.beneficiary, amount)
 }
 
@@ -546,8 +546,7 @@ func (self *Inbox) autoCash(cashInterval time.Duration) {
 
 // Receive is called to deposit the latest cheque to the incoming Inbox.
 // The given promise must be a *Cheque.
-func (self *Inbox) Receive(promise interface{}) (*big.Int, error) {
-	// TODO(fjl): the type of promise should be safer
+func (self *Inbox) Receive(promise swap.Promise) (*big.Int, error) {
 	ch := promise.(*Cheque)
 
 	defer self.lock.Unlock()

internal/web3ext/web3ext.go

@@ -18,17 +18,159 @@
 package web3ext
 
 var Modules = map[string]string{
-	"admin":    Admin_JS,
-	"debug":    Debug_JS,
-	"eth":      Eth_JS,
-	"miner":    Miner_JS,
-	"net":      Net_JS,
-	"personal": Personal_JS,
-	"rpc":      RPC_JS,
-	"shh":      Shh_JS,
-	"txpool":   TxPool_JS,
+	"admin":      Admin_JS,
+	"bzz":        Bzz_JS,
+	"chequebook": Chequebook_JS,
+	"debug":      Debug_JS,
+	"ens":        ENS_JS,
+	"eth":        Eth_JS,
+	"miner":      Miner_JS,
+	"net":        Net_JS,
+	"personal":   Personal_JS,
+	"rpc":        RPC_JS,
+	"shh":        Shh_JS,
+	"txpool":     TxPool_JS,
 }
 
+const Bzz_JS = `
+web3._extend({
+	property: 'bzz',
+	methods:
+	[
+		new web3._extend.Method({
+			name: 'blockNetworkRead',
+			call: 'bzz_blockNetworkRead',
+			params: 1,
+			inputFormatter: [null]
+		}),
+		new web3._extend.Method({
+			name: 'syncEnabled',
+			call: 'bzz_syncEnabled',
+			params: 1,
+			inputFormatter: [null]
+		}),
+		new web3._extend.Method({
+			name: 'swapEnabled',
+			call: 'bzz_swapEnabled',
+			params: 1,
+			inputFormatter: [null]
+		}),
+		new web3._extend.Method({
+			name: 'download',
+			call: 'bzz_download',
+			params: 2,
+			inputFormatter: [null, null]
+		}),
+		new web3._extend.Method({
+			name: 'upload',
+			call: 'bzz_upload',
+			params: 2,
+			inputFormatter: [null, null]
+		}),
+		new web3._extend.Method({
+			name: 'retrieve',
+			call: 'bzz_retrieve',
+			params: 1,
+			inputFormatter: [null]
+		}),
+		new web3._extend.Method({
+			name: 'store',
+			call: 'bzz_store',
+			params: 2,
+			inputFormatter: [null]
+		}),
+		new web3._extend.Method({
+			name: 'get',
+			call: 'bzz_get',
+			params: 1,
+			inputFormatter: [null]
+		}),
+		new web3._extend.Method({
+			name: 'put',
+			call: 'bzz_put',
+			params: 2,
+			inputFormatter: [null, null]
+		}),
+		new web3._extend.Method({
+			name: 'modify',
+			call: 'bzz_modify',
+			params: 4,
+			inputFormatter: [null, null, null, null]
+		})
+	],
+	properties:
+	[
+		new web3._extend.Property({
+			name: 'hive',
+			getter: 'bzz_hive'
+		}),
+		new web3._extend.Property({
+			name: 'info',
+			getter: 'bzz_info',
+		}),
+	]
+});
+`
+
+const ENS_JS = `
+web3._extend({
+  property: 'ens',
+  methods:
+  [
+    new web3._extend.Method({
+			name: 'register',
+			call: 'ens_register',
+			params: 1,
+			inputFormatter: [null]
+		}),
+	new web3._extend.Method({
+			name: 'setContentHash',
+			call: 'ens_setContentHash',
+			params: 2,
+			inputFormatter: [null, null]
+		}),
+	new web3._extend.Method({
+			name: 'resolve',
+			call: 'ens_resolve',
+			params: 1,
+			inputFormatter: [null]
+		}),
+	]
+})
+`
+
+const Chequebook_JS = `
+web3._extend({
+  property: 'chequebook',
+  methods:
+  [
+    new web3._extend.Method({
+      name: 'deposit',
+      call: 'chequebook_deposit',
+      params: 1,
+      inputFormatter: [null]
+    }),
+    new web3._extend.Property({
+			name: 'balance',
+			getter: 'chequebook_balance',
+				outputFormatter: web3._extend.utils.toDecimal
+		}),
+    new web3._extend.Method({
+      name: 'cash',
+      call: 'chequebook_cash',
+      params: 1,
+      inputFormatter: [null]
+    }),
+    new web3._extend.Method({
+      name: 'issue',
+      call: 'chequebook_issue',
+      params: 2,
+      inputFormatter: [null, null]
+    }),
+  ]
+});
+`
+
 const Admin_JS = `
 web3._extend({
 	property: 'admin',

swarm/api/api.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+var (
+	hashMatcher      = regexp.MustCompile("^[0-9A-Fa-f]{64}")
+	slashes          = regexp.MustCompile("/+")
+	domainAndVersion = regexp.MustCompile("[@:;,]+")
+)
+
+type Resolver interface {
+	Resolve(string) (common.Hash, error)
+}
+
+/*
+Api implements webserver/file system related content storage and retrieval
+on top of the dpa
+it is the public interface of the dpa which is included in the ethereum stack
+*/
+type Api struct {
+	dpa *storage.DPA
+	dns Resolver
+}
+
+//the api constructor initialises
+func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) {
+	self = &Api{
+		dpa: dpa,
+		dns: dns,
+	}
+	return
+}
+
+// DPA reader API
+func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader {
+	return self.dpa.Retrieve(key)
+}
+
+func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) {
+	return self.dpa.Store(data, size, wg, nil)
+}
+
+type ErrResolve error
+
+// DNS Resolver
+func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) {
+	if hashMatcher.MatchString(hostPort) || self.dns == nil {
+		glog.V(logger.Detail).Infof("host is a contentHash: '%v'", hostPort)
+		return storage.Key(common.Hex2Bytes(hostPort)), nil
+	}
+	if !nameresolver {
+		return nil, fmt.Errorf("'%s' is not a content hash value.", hostPort)
+	}
+	contentHash, err := self.dns.Resolve(hostPort)
+	if err != nil {
+		err = ErrResolve(err)
+		glog.V(logger.Warn).Infof("DNS error : %v", err)
+	}
+	glog.V(logger.Detail).Infof("host lookup: %v -> %v", err)
+	return contentHash[:], err
+}
+
+func parse(uri string) (hostPort, path string) {
+	parts := slashes.Split(uri, 3)
+	var i int
+	if len(parts) == 0 {
+		return
+	}
+	// beginning with slash is now optional
+	for len(parts[i]) == 0 {
+		i++
+	}
+	hostPort = parts[i]
+	for i < len(parts)-1 {
+		i++
+		if len(path) > 0 {
+			path = path + "/" + parts[i]
+		} else {
+			path = parts[i]
+		}
+	}
+	glog.V(logger.Debug).Infof("host: '%s', path '%s' requested.", hostPort, path)
+	return
+}
+
+func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) {
+	hostPort, path = parse(uri)
+	//resolving host and port
+	contentHash, err := self.Resolve(hostPort, nameresolver)
+	glog.V(logger.Debug).Infof("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path)
+	return contentHash[:], hostPort, path, err
+}
+
+// Put provides singleton manifest creation on top of dpa store
+func (self *Api) Put(content, contentType string) (string, error) {
+	r := strings.NewReader(content)
+	wg := &sync.WaitGroup{}
+	key, err := self.dpa.Store(r, int64(len(content)), wg, nil)
+	if err != nil {
+		return "", err
+	}
+	manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
+	r = strings.NewReader(manifest)
+	key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil)
+	if err != nil {
+		return "", err
+	}
+	wg.Wait()
+	return key.String(), nil
+}
+
+// Get uses iterative manifest retrieval and prefix matching
+// to resolve path to content using dpa retrieve
+// it returns a section reader, mimeType, status and an error
+func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) {
+
+	key, _, path, err := self.parseAndResolve(uri, nameresolver)
+	quitC := make(chan bool)
+	trie, err := loadManifest(self.dpa, key, quitC)
+	if err != nil {
+		glog.V(logger.Warn).Infof("loadManifestTrie error: %v", err)
+		return
+	}
+
+	glog.V(logger.Detail).Infof("getEntry(%s)", path)
+	entry, _ := trie.getEntry(path)
+	if entry != nil {
+		key = common.Hex2Bytes(entry.Hash)
+		status = entry.Status
+		mimeType = entry.ContentType
+		glog.V(logger.Detail).Infof("content lookup key: '%v' (%v)", key, mimeType)
+		reader = self.dpa.Retrieve(key)
+	} else {
+		err = fmt.Errorf("manifest entry for '%s' not found", path)
+		glog.V(logger.Warn).Infof("%v", err)
+	}
+	return
+}
+
+func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) {
+	root, _, path, err := self.parseAndResolve(uri, nameresolver)
+	quitC := make(chan bool)
+	trie, err := loadManifest(self.dpa, root, quitC)
+	if err != nil {
+		return
+	}
+
+	if contentHash != "" {
+		entry := &manifestTrieEntry{
+			Path:        path,
+			Hash:        contentHash,
+			ContentType: contentType,
+		}
+		trie.addEntry(entry, quitC)
+	} else {
+		trie.deleteEntry(path, quitC)
+	}
+
+	err = trie.recalcAndStore()
+	if err != nil {
+		return
+	}
+	return trie.hash.String(), nil
+}

swarm/api/api_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+func testApi(t *testing.T, f func(*Api)) {
+	datadir, err := ioutil.TempDir("", "bzz-test")
+	if err != nil {
+		t.Fatalf("unable to create temp dir: %v", err)
+	}
+	os.RemoveAll(datadir)
+	defer os.RemoveAll(datadir)
+	dpa, err := storage.NewLocalDPA(datadir)
+	if err != nil {
+		return
+	}
+	api := NewApi(dpa, nil)
+	dpa.Start()
+	f(api)
+	dpa.Stop()
+}
+
+type testResponse struct {
+	reader storage.LazySectionReader
+	*Response
+}
+
+func checkResponse(t *testing.T, resp *testResponse, exp *Response) {
+
+	if resp.MimeType != exp.MimeType {
+		t.Errorf("incorrect mimeType. expected '%s', got '%s'", exp.MimeType, resp.MimeType)
+	}
+	if resp.Status != exp.Status {
+		t.Errorf("incorrect status. expected '%d', got '%d'", exp.Status, resp.Status)
+	}
+	if resp.Size != exp.Size {
+		t.Errorf("incorrect size. expected '%d', got '%d'", exp.Size, resp.Size)
+	}
+	if resp.reader != nil {
+		content := make([]byte, resp.Size)
+		read, _ := resp.reader.Read(content)
+		if int64(read) != exp.Size {
+			t.Errorf("incorrect content length. expected '%d...', got '%d...'", read, exp.Size)
+		}
+		resp.Content = string(content)
+	}
+	if resp.Content != exp.Content {
+		// if !bytes.Equal(resp.Content, exp.Content)
+		t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Content))
+	}
+}
+
+// func expResponse(content []byte, mimeType string, status int) *Response {
+func expResponse(content string, mimeType string, status int) *Response {
+	glog.V(logger.Detail).Infof("expected content (%v): %v ", len(content), content)
+	return &Response{mimeType, status, int64(len(content)), content}
+}
+
+// func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
+func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
+	reader, mimeType, status, err := api.Get(bzzhash, true)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	quitC := make(chan bool)
+	size, err := reader.Size(quitC)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	glog.V(logger.Detail).Infof("reader size: %v ", size)
+	s := make([]byte, size)
+	_, err = reader.Read(s)
+	if err != io.EOF {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	reader.Seek(0, 0)
+	return &testResponse{reader, &Response{mimeType, status, size, string(s)}}
+	// return &testResponse{reader, &Response{mimeType, status, reader.Size(), nil}}
+}
+
+func TestApiPut(t *testing.T) {
+	testApi(t, func(api *Api) {
+		content := "hello"
+		exp := expResponse(content, "text/plain", 0)
+		// exp := expResponse([]byte(content), "text/plain", 0)
+		bzzhash, err := api.Put(content, exp.MimeType)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		resp := testGet(t, api, bzzhash)
+		checkResponse(t, resp, exp)
+	})
+}

swarm/api/config.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"crypto/ecdsa"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/swarm/network"
+	"github.com/ethereum/go-ethereum/swarm/services/swap"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+const (
+	port = "8500"
+)
+
+//  by default ens root is  north internal
+var (
+	toyNetEnsRoot = common.HexToAddress("0xd344889e0be3e9ef6c26b0f60ef66a32e83c1b69")
+)
+
+// separate bzz directories
+// allow several bzz nodes running in parallel
+type Config struct {
+	// serialised/persisted fields
+	*storage.StoreParams
+	*storage.ChunkerParams
+	*network.HiveParams
+	Swap *swap.SwapParams
+	*network.SyncParams
+	Path      string
+	Port      string
+	PublicKey string
+	BzzKey    string
+	EnsRoot   common.Address
+}
+
+// config is agnostic to where private key is coming from
+// so managing accounts is outside swarm and left to wrappers
+func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey) (self *Config, err error) {
+
+	address := crypto.PubkeyToAddress(prvKey.PublicKey) // default beneficiary address
+	dirpath := filepath.Join(path, common.Bytes2Hex(address.Bytes()))
+	err = os.MkdirAll(dirpath, os.ModePerm)
+	if err != nil {
+		return
+	}
+	confpath := filepath.Join(dirpath, "config.json")
+	var data []byte
+	pubkey := crypto.FromECDSAPub(&prvKey.PublicKey)
+	pubkeyhex := common.ToHex(pubkey)
+	keyhex := crypto.Sha3Hash(pubkey).Hex()
+
+	self = &Config{
+		SyncParams:    network.NewSyncParams(dirpath),
+		HiveParams:    network.NewHiveParams(dirpath),
+		ChunkerParams: storage.NewChunkerParams(),
+		StoreParams:   storage.NewStoreParams(dirpath),
+		Port:          port,
+		Path:          dirpath,
+		Swap:          swap.DefaultSwapParams(contract, prvKey),
+		PublicKey:     pubkeyhex,
+		BzzKey:        keyhex,
+		EnsRoot:       toyNetEnsRoot,
+	}
+	data, err = ioutil.ReadFile(confpath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+		// file does not exist
+		// write out config file
+		err = self.Save()
+		if err != nil {
+			err = fmt.Errorf("error writing config: %v", err)
+		}
+		return
+	}
+	// file exists, deserialise
+	err = json.Unmarshal(data, self)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse config: %v", err)
+	}
+	// check public key
+	if pubkeyhex != self.PublicKey {
+		return nil, fmt.Errorf("public key does not match the one in the config file %v != %v", pubkeyhex, self.PublicKey)
+	}
+	if keyhex != self.BzzKey {
+		return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey)
+	}
+	self.Swap.SetKey(prvKey)
+
+	if (self.EnsRoot == common.Address{}) {
+		self.EnsRoot = toyNetEnsRoot
+	}
+
+	return
+}
+
+func (self *Config) Save() error {
+	data, err := json.MarshalIndent(self, "", "    ")
+	if err != nil {
+		return err
+	}
+	err = os.MkdirAll(self.Path, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	confpath := filepath.Join(self.Path, "config.json")
+	return ioutil.WriteFile(confpath, data, os.ModePerm)
+}

swarm/api/config_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+var (
+	hexprvkey     = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c"
+	defaultConfig = `{
+    "ChunkDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "chunks") + `",
+    "DbCapacity": 5000000,
+    "CacheCapacity": 5000,
+    "Radius": 0,
+    "Branches": 128,
+    "Hash": "SHA3",
+    "CallInterval": 3000000000,
+    "KadDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "bzz-peers.json") + `",
+    "MaxProx": 8,
+    "ProxBinSize": 2,
+    "BucketSize": 4,
+    "PurgeInterval": 151200000000000,
+    "InitialRetryInterval": 42000000,
+    "MaxIdleInterval": 42000000000,
+    "ConnRetryExp": 2,
+    "Swap": {
+        "BuyAt": 20000000000,
+        "SellAt": 20000000000,
+        "PayAt": 100,
+        "DropAt": 10000,
+        "AutoCashInterval": 300000000000,
+        "AutoCashThreshold": 50000000000000,
+        "AutoDepositInterval": 300000000000,
+        "AutoDepositThreshold": 50000000000000,
+        "AutoDepositBuffer": 100000000000000,
+        "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3",
+        "Contract": "0x0000000000000000000000000000000000000000",
+        "Beneficiary": "0x0d2f62485607cf38d9d795d93682a517661e513e"
+    },
+    "RequestDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "requests") + `",
+    "RequestDbBatchSize": 512,
+    "KeyBufferSize": 1024,
+    "SyncBatchSize": 128,
+    "SyncBufferSize": 128,
+    "SyncCacheSize": 1024,
+    "SyncPriorities": [
+        2,
+        1,
+        1,
+        0,
+        0
+    ],
+    "SyncModes": [
+        true,
+        true,
+        true,
+        true,
+        false
+    ],
+    "Path": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e") + `",
+    "Port": "8500",
+    "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3",
+    "BzzKey": "0xe861964402c0b78e2d44098329b8545726f215afa737d803714a4338552fcb81",
+    "EnsRoot": "0xd344889e0be3e9ef6c26b0f60ef66a32e83c1b69"
+}`
+)
+
+func TestConfigWriteRead(t *testing.T) {
+	tmp, err := ioutil.TempDir(os.TempDir(), "bzz-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+
+	prvkey := crypto.ToECDSA(common.Hex2Bytes(hexprvkey))
+	orig, err := NewConfig(tmp, common.Address{}, prvkey)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	account := crypto.PubkeyToAddress(prvkey.PublicKey)
+	dirpath := filepath.Join(tmp, common.Bytes2Hex(account.Bytes()))
+	confpath := filepath.Join(dirpath, "config.json")
+	data, err := ioutil.ReadFile(confpath)
+	if err != nil {
+		t.Fatalf("default config file cannot be read: %v", err)
+	}
+	exp := strings.Replace(defaultConfig, "TMPDIR", tmp, -1)
+	exp = strings.Replace(exp, "\\", "\\\\", -1)
+
+	if string(data) != exp {
+		t.Fatalf("default config mismatch:\nexpected: %v\ngot: %v", exp, string(data))
+	}
+
+	conf, err := NewConfig(tmp, common.Address{}, prvkey)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if conf.Swap.Beneficiary.Hex() != orig.Swap.Beneficiary.Hex() {
+		t.Fatalf("expected beneficiary from loaded config %v to match original %v", conf.Swap.Beneficiary.Hex(), orig.Swap.Beneficiary.Hex())
+	}
+
+}

swarm/api/filesystem.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+const maxParallelFiles = 5
+
+type FileSystem struct {
+	api *Api
+}
+
+func NewFileSystem(api *Api) *FileSystem {
+	return &FileSystem{api}
+}
+
+// Upload replicates a local directory as a manifest file and uploads it
+// using dpa store
+// TODO: localpath should point to a manifest
+func (self *FileSystem) Upload(lpath, index string) (string, error) {
+	var list []*manifestTrieEntry
+	localpath, err := filepath.Abs(filepath.Clean(lpath))
+	if err != nil {
+		return "", err
+	}
+
+	f, err := os.Open(localpath)
+	if err != nil {
+		return "", err
+	}
+	stat, err := f.Stat()
+	if err != nil {
+		return "", err
+	}
+
+	var start int
+	if stat.IsDir() {
+		start = len(localpath)
+		glog.V(logger.Debug).Infof("uploading '%s'", localpath)
+		err = filepath.Walk(localpath, func(path string, info os.FileInfo, err error) error {
+			if (err == nil) && !info.IsDir() {
+				//fmt.Printf("lp %s  path %s\n", localpath, path)
+				if len(path) <= start {
+					return fmt.Errorf("Path is too short")
+				}
+				if path[:start] != localpath {
+					return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath)
+				}
+				entry := &manifestTrieEntry{
+					Path: filepath.ToSlash(path),
+				}
+				list = append(list, entry)
+			}
+			return err
+		})
+		if err != nil {
+			return "", err
+		}
+	} else {
+		dir := filepath.Dir(localpath)
+		start = len(dir)
+		if len(localpath) <= start {
+			return "", fmt.Errorf("Path is too short")
+		}
+		if localpath[:start] != dir {
+			return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir)
+		}
+		entry := &manifestTrieEntry{
+			Path: filepath.ToSlash(localpath),
+		}
+		list = append(list, entry)
+	}
+
+	cnt := len(list)
+	errors := make([]error, cnt)
+	done := make(chan bool, maxParallelFiles)
+	dcnt := 0
+	awg := &sync.WaitGroup{}
+
+	for i, entry := range list {
+		if i >= dcnt+maxParallelFiles {
+			<-done
+			dcnt++
+		}
+		awg.Add(1)
+		go func(i int, entry *manifestTrieEntry, done chan bool) {
+			f, err := os.Open(entry.Path)
+			if err == nil {
+				stat, _ := f.Stat()
+				var hash storage.Key
+				wg := &sync.WaitGroup{}
+				hash, err = self.api.dpa.Store(f, stat.Size(), wg, nil)
+				if hash != nil {
+					list[i].Hash = hash.String()
+				}
+				wg.Wait()
+				awg.Done()
+				if err == nil {
+					first512 := make([]byte, 512)
+					fread, _ := f.ReadAt(first512, 0)
+					if fread > 0 {
+						mimeType := http.DetectContentType(first512[:fread])
+						if filepath.Ext(entry.Path) == ".css" {
+							mimeType = "text/css"
+						}
+						list[i].ContentType = mimeType
+					}
+				}
+				f.Close()
+			}
+			errors[i] = err
+			done <- true
+		}(i, entry, done)
+	}
+	for dcnt < cnt {
+		<-done
+		dcnt++
+	}
+
+	trie := &manifestTrie{
+		dpa: self.api.dpa,
+	}
+	quitC := make(chan bool)
+	for i, entry := range list {
+		if errors[i] != nil {
+			return "", errors[i]
+		}
+		entry.Path = RegularSlashes(entry.Path[start:])
+		if entry.Path == index {
+			ientry := &manifestTrieEntry{
+				Path:        "",
+				Hash:        entry.Hash,
+				ContentType: entry.ContentType,
+			}
+			trie.addEntry(ientry, quitC)
+		}
+		trie.addEntry(entry, quitC)
+	}
+
+	err2 := trie.recalcAndStore()
+	var hs string
+	if err2 == nil {
+		hs = trie.hash.String()
+	}
+	awg.Wait()
+	return hs, err2
+}
+
+// Download replicates the manifest path structure on the local filesystem
+// under localpath
+func (self *FileSystem) Download(bzzpath, localpath string) error {
+	lpath, err := filepath.Abs(filepath.Clean(localpath))
+	if err != nil {
+		return err
+	}
+	err = os.MkdirAll(lpath, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	//resolving host and port
+	key, _, path, err := self.api.parseAndResolve(bzzpath, true)
+	if err != nil {
+		return err
+	}
+
+	if len(path) > 0 {
+		path += "/"
+	}
+
+	quitC := make(chan bool)
+	trie, err := loadManifest(self.api.dpa, key, quitC)
+	if err != nil {
+		glog.V(logger.Warn).Infof("fs.Download: loadManifestTrie error: %v", err)
+		return err
+	}
+
+	type downloadListEntry struct {
+		key  storage.Key
+		path string
+	}
+
+	var list []*downloadListEntry
+	var mde error
+
+	prevPath := lpath
+	err = trie.listWithPrefix(path, quitC, func(entry *manifestTrieEntry, suffix string) {
+		glog.V(logger.Detail).Infof("fs.Download: %#v", entry)
+
+		key = common.Hex2Bytes(entry.Hash)
+		path := lpath + "/" + suffix
+		dir := filepath.Dir(path)
+		if dir != prevPath {
+			mde = os.MkdirAll(dir, os.ModePerm)
+			prevPath = dir
+		}
+		if (mde == nil) && (path != dir+"/") {
+			list = append(list, &downloadListEntry{key: key, path: path})
+		}
+	})
+	if err != nil {
+		return err
+	}
+
+	wg := sync.WaitGroup{}
+	errC := make(chan error)
+	done := make(chan bool, maxParallelFiles)
+	for i, entry := range list {
+		select {
+		case done <- true:
+			wg.Add(1)
+		case <-quitC:
+			return fmt.Errorf("aborted")
+		}
+		go func(i int, entry *downloadListEntry) {
+			defer wg.Done()
+			f, err := os.Create(entry.path) // TODO: path separators
+			if err == nil {
+
+				reader := self.api.dpa.Retrieve(entry.key)
+				writer := bufio.NewWriter(f)
+				size, err := reader.Size(quitC)
+				if err == nil {
+					_, err = io.CopyN(writer, reader, size) // TODO: handle errors
+					err2 := writer.Flush()
+					if err == nil {
+						err = err2
+					}
+					err2 = f.Close()
+					if err == nil {
+						err = err2
+					}
+				}
+			}
+			if err != nil {
+				select {
+				case errC <- err:
+				case <-quitC:
+				}
+				return
+			}
+			<-done
+		}(i, entry)
+	}
+	go func() {
+		wg.Wait()
+		close(errC)
+	}()
+	select {
+	case err = <-errC:
+		return err
+	case <-quitC:
+		return fmt.Errorf("aborted")
+	}
+
+}

swarm/api/filesystem_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test")
+
+func testFileSystem(t *testing.T, f func(*FileSystem)) {
+	testApi(t, func(api *Api) {
+		f(NewFileSystem(api))
+	})
+}
+
+func readPath(t *testing.T, parts ...string) string {
+	file := filepath.Join(parts...)
+	content, err := ioutil.ReadFile(file)
+
+	if err != nil {
+		t.Fatalf("unexpected error reading '%v': %v", file, err)
+	}
+	return string(content)
+}
+
+func TestApiDirUpload0(t *testing.T) {
+	testFileSystem(t, func(fs *FileSystem) {
+		api := fs.api
+		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "")
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		content := readPath(t, "testdata", "test0", "index.html")
+		resp := testGet(t, api, bzzhash+"/index.html")
+		exp := expResponse(content, "text/html; charset=utf-8", 0)
+		checkResponse(t, resp, exp)
+
+		content = readPath(t, "testdata", "test0", "index.css")
+		resp = testGet(t, api, bzzhash+"/index.css")
+		exp = expResponse(content, "text/css", 0)
+		checkResponse(t, resp, exp)
+
+		_, _, _, err = api.Get(bzzhash, true)
+		if err == nil {
+			t.Fatalf("expected error: %v", err)
+		}
+
+		downloadDir := filepath.Join(testDownloadDir, "test0")
+		defer os.RemoveAll(downloadDir)
+		err = fs.Download(bzzhash, downloadDir)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		newbzzhash, err := fs.Upload(downloadDir, "")
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if bzzhash != newbzzhash {
+			t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash)
+		}
+	})
+}
+
+func TestApiDirUploadModify(t *testing.T) {
+	testFileSystem(t, func(fs *FileSystem) {
+		api := fs.api
+		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "")
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+
+		bzzhash, err = api.Modify(bzzhash+"/index.html", "", "", true)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+		index, err := ioutil.ReadFile(filepath.Join("testdata", "test0", "index.html"))
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+		wg := &sync.WaitGroup{}
+		hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg)
+		wg.Wait()
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+		bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+		bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+
+		content := readPath(t, "testdata", "test0", "index.html")
+		resp := testGet(t, api, bzzhash+"/index2.html")
+		exp := expResponse(content, "text/html; charset=utf-8", 0)
+		checkResponse(t, resp, exp)
+
+		resp = testGet(t, api, bzzhash+"/img/logo.png")
+		exp = expResponse(content, "text/html; charset=utf-8", 0)
+		checkResponse(t, resp, exp)
+
+		content = readPath(t, "testdata", "test0", "index.css")
+		resp = testGet(t, api, bzzhash+"/index.css")
+		exp = expResponse(content, "text/css", 0)
+
+		_, _, _, err = api.Get(bzzhash, true)
+		if err == nil {
+			t.Errorf("expected error: %v", err)
+		}
+	})
+}
+
+func TestApiDirUploadWithRootFile(t *testing.T) {
+	testFileSystem(t, func(fs *FileSystem) {
+		api := fs.api
+		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "index.html")
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+
+		content := readPath(t, "testdata", "test0", "index.html")
+		resp := testGet(t, api, bzzhash)
+		exp := expResponse(content, "text/html; charset=utf-8", 0)
+		checkResponse(t, resp, exp)
+	})
+}
+
+func TestApiFileUpload(t *testing.T) {
+	testFileSystem(t, func(fs *FileSystem) {
+		api := fs.api
+		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "")
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+
+		content := readPath(t, "testdata", "test0", "index.html")
+		resp := testGet(t, api, bzzhash+"/index.html")
+		exp := expResponse(content, "text/html; charset=utf-8", 0)
+		checkResponse(t, resp, exp)
+	})
+}
+
+func TestApiFileUploadWithRootFile(t *testing.T) {
+	testFileSystem(t, func(fs *FileSystem) {
+		api := fs.api
+		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "index.html")
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+
+		content := readPath(t, "testdata", "test0", "index.html")
+		resp := testGet(t, api, bzzhash)
+		exp := expResponse(content, "text/html; charset=utf-8", 0)
+		checkResponse(t, resp, exp)
+	})
+}

swarm/api/http/roundtripper.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+)
+
+/*
+http roundtripper to register for bzz url scheme
+see https://github.com/ethereum/go-ethereum/issues/2040
+Usage:
+
+import (
+ "github.com/ethereum/go-ethereum/common/httpclient"
+ "github.com/ethereum/go-ethereum/swarm/api/http"
+)
+client := httpclient.New()
+// for (private) swarm proxy running locally
+client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzzi", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzzr", &http.RoundTripper{Port: port})
+
+The port you give the Roundtripper is the port the swarm proxy is listening on.
+If Host is left empty, localhost is assumed.
+
+Using a public gateway, the above few lines gives you the leanest
+bzz-scheme aware read-only http client. You really only ever need this
+if you need go-native swarm access to bzz addresses, e.g.,
+github.com/ethereum/go-ethereum/common/natspec
+
+*/
+
+type RoundTripper struct {
+	Host string
+	Port string
+}
+
+func (self *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
+	host := self.Host
+	if len(host) == 0 {
+		host = "localhost"
+	}
+	url := fmt.Sprintf("http://%s:%s/%s:/%s/%s", host, self.Port, req.Proto, req.URL.Host, req.URL.Path)
+	glog.V(logger.Info).Infof("roundtripper: proxying request '%s' to '%s'", req.RequestURI, url)
+	reqProxy, err := http.NewRequest(req.Method, url, req.Body)
+	if err != nil {
+		return nil, err
+	}
+	return http.DefaultClient.Do(reqProxy)
+}

swarm/api/http/roundtripper_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/httpclient"
+)
+
+const port = "3222"
+
+func TestRoundTripper(t *testing.T) {
+	serveMux := http.NewServeMux()
+	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == "GET" {
+			w.Header().Set("Content-Type", "text/plain")
+			http.ServeContent(w, r, "", time.Unix(0, 0), strings.NewReader(r.RequestURI))
+		} else {
+			http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
+		}
+	})
+	go http.ListenAndServe(":"+port, serveMux)
+
+	rt := &RoundTripper{Port: port}
+	client := httpclient.New("/")
+	client.RegisterProtocol("bzz", rt)
+
+	resp, err := client.Client().Get("bzz://test.com/path")
+	if err != nil {
+		t.Errorf("expected no error, got %v", err)
+		return
+	}
+
+	defer func() {
+		if resp != nil {
+			resp.Body.Close()
+		}
+	}()
+
+	content, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Errorf("expected no error, got %v", err)
+		return
+	}
+	if string(content) != "/HTTP/1.1:/test.com/path" {
+		t.Errorf("incorrect response from http server: expected '%v', got '%v'", "/HTTP/1.1:/test.com/path", string(content))
+	}
+
+}

swarm/api/http/server.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+/*
+A simple http server interface to Swarm
+*/
+package http
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/swarm/api"
+)
+
+const (
+	rawType = "application/octet-stream"
+)
+
+var (
+	// accepted protocols: bzz (traditional), bzzi (immutable) and bzzr (raw)
+	bzzPrefix       = regexp.MustCompile("^/+bzz[ir]?:/+")
+	trailingSlashes = regexp.MustCompile("/+$")
+	rootDocumentUri = regexp.MustCompile("^/+bzz[i]?:/+[^/]+$")
+	// forever         = func() time.Time { return time.Unix(0, 0) }
+	forever = time.Now
+)
+
+type sequentialReader struct {
+	reader io.Reader
+	pos    int64
+	ahead  map[int64](chan bool)
+	lock   sync.Mutex
+}
+
+// browser API for registering bzz url scheme handlers:
+// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
+// electron (chromium) api for registering bzz url scheme handlers:
+// https://github.com/atom/electron/blob/master/docs/api/protocol.md
+
+// starts up http server
+func StartHttpServer(api *api.Api, port string) {
+	serveMux := http.NewServeMux()
+	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		handler(w, r, api)
+	})
+	go http.ListenAndServe(":"+port, serveMux)
+	glog.V(logger.Info).Infof("Swarm HTTP proxy started on localhost:%s", port)
+}
+
+func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
+	requestURL := r.URL
+	// This is wrong
+	//	if requestURL.Host == "" {
+	//		var err error
+	//		requestURL, err = url.Parse(r.Referer() + requestURL.String())
+	//		if err != nil {
+	//			http.Error(w, err.Error(), http.StatusBadRequest)
+	//			return
+	//		}
+	//	}
+	glog.V(logger.Debug).Infof("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept"))
+	uri := requestURL.Path
+	var raw, nameresolver bool
+	var proto string
+
+	// HTTP-based URL protocol handler
+	glog.V(logger.Debug).Infof("BZZ request URI: '%s'", uri)
+
+	path := bzzPrefix.ReplaceAllStringFunc(uri, func(p string) string {
+		proto = p
+		return ""
+	})
+
+	// protocol identification (ugly)
+	if proto == "" {
+		if glog.V(logger.Error) {
+			glog.Errorf(
+				"[BZZ] Swarm: Protocol error in request `%s`.",
+				uri,
+			)
+			http.Error(w, "BZZ protocol error", http.StatusBadRequest)
+			return
+		}
+	}
+	if len(proto) > 4 {
+		raw = proto[1:5] == "bzzr"
+		nameresolver = proto[1:5] != "bzzi"
+	}
+
+	glog.V(logger.Debug).Infof(
+		"[BZZ] Swarm: %s request over protocol %s '%s' received.",
+		r.Method, proto, path,
+	)
+
+	switch {
+	case r.Method == "POST" || r.Method == "PUT":
+		key, err := a.Store(r.Body, r.ContentLength, nil)
+		if err == nil {
+			glog.V(logger.Debug).Infof("Content for %v stored", key.Log())
+		} else {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		if r.Method == "POST" {
+			if raw {
+				w.Header().Set("Content-Type", "text/plain")
+				http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(common.Bytes2Hex(key))))
+			} else {
+				http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest)
+				return
+			}
+		} else {
+			// PUT
+			if raw {
+				http.Error(w, "No PUT to /raw allowed.", http.StatusBadRequest)
+				return
+			} else {
+				path = api.RegularSlashes(path)
+				mime := r.Header.Get("Content-Type")
+				// TODO proper root hash separation
+				glog.V(logger.Debug).Infof("Modify '%s' to store %v as '%s'.", path, key.Log(), mime)
+				newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
+				if err == nil {
+					glog.V(logger.Debug).Infof("Swarm replaced manifest by '%s'", newKey)
+					w.Header().Set("Content-Type", "text/plain")
+					http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey)))
+				} else {
+					http.Error(w, "PUT to "+path+"failed.", http.StatusBadRequest)
+					return
+				}
+			}
+		}
+	case r.Method == "DELETE":
+		if raw {
+			http.Error(w, "No DELETE to /raw allowed.", http.StatusBadRequest)
+			return
+		} else {
+			path = api.RegularSlashes(path)
+			glog.V(logger.Debug).Infof("Delete '%s'.", path)
+			newKey, err := a.Modify(path, "", "", nameresolver)
+			if err == nil {
+				glog.V(logger.Debug).Infof("Swarm replaced manifest by '%s'", newKey)
+				w.Header().Set("Content-Type", "text/plain")
+				http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey)))
+			} else {
+				http.Error(w, "DELETE to "+path+"failed.", http.StatusBadRequest)
+				return
+			}
+		}
+	case r.Method == "GET" || r.Method == "HEAD":
+		path = trailingSlashes.ReplaceAllString(path, "")
+		if raw {
+			// resolving host
+			key, err := a.Resolve(path, nameresolver)
+			if err != nil {
+				glog.V(logger.Error).Infof("%v", err)
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+
+			// retrieving content
+			reader := a.Retrieve(key)
+			quitC := make(chan bool)
+			size, err := reader.Size(quitC)
+			glog.V(logger.Debug).Infof("Reading %d bytes.", size)
+
+			// setting mime type
+			qv := requestURL.Query()
+			mimeType := qv.Get("content_type")
+			if mimeType == "" {
+				mimeType = rawType
+			}
+
+			w.Header().Set("Content-Type", mimeType)
+			http.ServeContent(w, r, uri, forever(), reader)
+			glog.V(logger.Debug).Infof("Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType)
+
+			// retrieve path via manifest
+		} else {
+			glog.V(logger.Debug).Infof("Structured GET request '%s' received.", uri)
+			// add trailing slash, if missing
+			if rootDocumentUri.MatchString(uri) {
+				http.Redirect(w, r, path+"/", http.StatusFound)
+				return
+			}
+			reader, mimeType, status, err := a.Get(path, nameresolver)
+			if err != nil {
+				if _, ok := err.(api.ErrResolve); ok {
+					glog.V(logger.Debug).Infof("%v", err)
+					status = http.StatusBadRequest
+				} else {
+					glog.V(logger.Debug).Infof("error retrieving '%s': %v", uri, err)
+					status = http.StatusNotFound
+				}
+				http.Error(w, err.Error(), status)
+				return
+			}
+			// set mime type and status headers
+			w.Header().Set("Content-Type", mimeType)
+			if status > 0 {
+				w.WriteHeader(status)
+			} else {
+				status = 200
+			}
+			quitC := make(chan bool)
+			size, err := reader.Size(quitC)
+			glog.V(logger.Debug).Infof("Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status)
+
+			http.ServeContent(w, r, path, forever(), reader)
+
+		}
+	default:
+		http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
+	}
+}
+
+func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error) {
+	self.lock.Lock()
+	// assert self.pos <= off
+	if self.pos > off {
+		glog.V(logger.Error).Infof("non-sequential read attempted from sequentialReader; %d > %d",
+			self.pos, off)
+		panic("Non-sequential read attempt")
+	}
+	if self.pos != off {
+		glog.V(logger.Debug).Infof("deferred read in POST at position %d, offset %d.",
+			self.pos, off)
+		wait := make(chan bool)
+		self.ahead[off] = wait
+		self.lock.Unlock()
+		if <-wait {
+			// failed read behind
+			n = 0
+			err = io.ErrUnexpectedEOF
+			return
+		}
+		self.lock.Lock()
+	}
+	localPos := 0
+	for localPos < len(target) {
+		n, err = self.reader.Read(target[localPos:])
+		localPos += n
+		glog.V(logger.Debug).Infof("Read %d bytes into buffer size %d from POST, error %v.",
+			n, len(target), err)
+		if err != nil {
+			glog.V(logger.Debug).Infof("POST stream's reading terminated with %v.", err)
+			for i := range self.ahead {
+				self.ahead[i] <- true
+				delete(self.ahead, i)
+			}
+			self.lock.Unlock()
+			return localPos, err
+		}
+		self.pos += int64(n)
+	}
+	wait := self.ahead[self.pos]
+	if wait != nil {
+		glog.V(logger.Debug).Infof("deferred read in POST at position %d triggered.",
+			self.pos)
+		delete(self.ahead, self.pos)
+		close(wait)
+	}
+	self.lock.Unlock()
+	return localPos, err
+}

swarm/api/manifest.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+const (
+	manifestType = "application/bzz-manifest+json"
+)
+
+type manifestTrie struct {
+	dpa     *storage.DPA
+	entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry
+	hash    storage.Key             // if hash != nil, it is stored
+}
+
+type manifestJSON struct {
+	Entries []*manifestTrieEntry `json:"entries"`
+}
+
+type manifestTrieEntry struct {
+	Path        string `json:"path"`
+	Hash        string `json:"hash"` // for manifest content type, empty until subtrie is evaluated
+	ContentType string `json:"contentType"`
+	Status      int    `json:"status"`
+	subtrie     *manifestTrie
+}
+
+func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
+
+	glog.V(logger.Detail).Infof("manifest lookup key: '%v'.", hash.Log())
+	// retrieve manifest via DPA
+	manifestReader := dpa.Retrieve(hash)
+	return readManifest(manifestReader, hash, dpa, quitC)
+}
+
+func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
+
+	// TODO check size for oversized manifests
+	size, err := manifestReader.Size(quitC)
+	manifestData := make([]byte, size)
+	read, err := manifestReader.Read(manifestData)
+	if int64(read) < size {
+		glog.V(logger.Detail).Infof("Manifest %v not found.", hash.Log())
+		if err == nil {
+			err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size)
+		}
+		return
+	}
+
+	glog.V(logger.Detail).Infof("Manifest %v retrieved", hash.Log())
+	man := manifestJSON{}
+	err = json.Unmarshal(manifestData, &man)
+	if err != nil {
+		err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err)
+		glog.V(logger.Detail).Infof("%v", err)
+		return
+	}
+
+	glog.V(logger.Detail).Infof("Manifest %v has %d entries.", hash.Log(), len(man.Entries))
+
+	trie = &manifestTrie{
+		dpa: dpa,
+	}
+	for _, entry := range man.Entries {
+		trie.addEntry(entry, quitC)
+	}
+	return
+}
+
+func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
+	self.hash = nil // trie modified, hash needs to be re-calculated on demand
+
+	if len(entry.Path) == 0 {
+		self.entries[256] = entry
+		return
+	}
+
+	b := byte(entry.Path[0])
+	if (self.entries[b] == nil) || (self.entries[b].Path == entry.Path) {
+		self.entries[b] = entry
+		return
+	}
+
+	oldentry := self.entries[b]
+	cpl := 0
+	for (len(entry.Path) > cpl) && (len(oldentry.Path) > cpl) && (entry.Path[cpl] == oldentry.Path[cpl]) {
+		cpl++
+	}
+
+	if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) {
+		if self.loadSubTrie(oldentry, quitC) != nil {
+			return
+		}
+		entry.Path = entry.Path[cpl:]
+		oldentry.subtrie.addEntry(entry, quitC)
+		oldentry.Hash = ""
+		return
+	}
+
+	commonPrefix := entry.Path[:cpl]
+
+	subtrie := &manifestTrie{
+		dpa: self.dpa,
+	}
+	entry.Path = entry.Path[cpl:]
+	oldentry.Path = oldentry.Path[cpl:]
+	subtrie.addEntry(entry, quitC)
+	subtrie.addEntry(oldentry, quitC)
+
+	self.entries[b] = &manifestTrieEntry{
+		Path:        commonPrefix,
+		Hash:        "",
+		ContentType: manifestType,
+		subtrie:     subtrie,
+	}
+}
+
+func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
+	for _, e := range self.entries {
+		if e != nil {
+			cnt++
+			entry = e
+		}
+	}
+	return
+}
+
+func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
+	self.hash = nil // trie modified, hash needs to be re-calculated on demand
+
+	if len(path) == 0 {
+		self.entries[256] = nil
+		return
+	}
+
+	b := byte(path[0])
+	entry := self.entries[b]
+	if entry == nil {
+		return
+	}
+	if entry.Path == path {
+		self.entries[b] = nil
+		return
+	}
+
+	epl := len(entry.Path)
+	if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
+		if self.loadSubTrie(entry, quitC) != nil {
+			return
+		}
+		entry.subtrie.deleteEntry(path[epl:], quitC)
+		entry.Hash = ""
+		// remove subtree if it has less than 2 elements
+		cnt, lastentry := entry.subtrie.getCountLast()
+		if cnt < 2 {
+			if lastentry != nil {
+				lastentry.Path = entry.Path + lastentry.Path
+			}
+			self.entries[b] = lastentry
+		}
+	}
+}
+
+func (self *manifestTrie) recalcAndStore() error {
+	if self.hash != nil {
+		return nil
+	}
+
+	var buffer bytes.Buffer
+	buffer.WriteString(`{"entries":[`)
+
+	list := &manifestJSON{}
+	for _, entry := range self.entries {
+		if entry != nil {
+			if entry.Hash == "" { // TODO: paralellize
+				err := entry.subtrie.recalcAndStore()
+				if err != nil {
+					return err
+				}
+				entry.Hash = entry.subtrie.hash.String()
+			}
+			list.Entries = append(list.Entries, entry)
+		}
+	}
+
+	manifest, err := json.Marshal(list)
+	if err != nil {
+		return err
+	}
+
+	sr := bytes.NewReader(manifest)
+	wg := &sync.WaitGroup{}
+	key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg, nil)
+	wg.Wait()
+	self.hash = key
+	return err2
+}
+
+func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
+	if entry.subtrie == nil {
+		hash := common.Hex2Bytes(entry.Hash)
+		entry.subtrie, err = loadManifest(self.dpa, hash, quitC)
+		entry.Hash = "" // might not match, should be recalculated
+	}
+	return
+}
+
+func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error {
+	plen := len(prefix)
+	var start, stop int
+	if plen == 0 {
+		start = 0
+		stop = 256
+	} else {
+		start = int(prefix[0])
+		stop = start
+	}
+
+	for i := start; i <= stop; i++ {
+		select {
+		case <-quitC:
+			return fmt.Errorf("aborted")
+		default:
+		}
+		entry := self.entries[i]
+		if entry != nil {
+			epl := len(entry.Path)
+			if entry.ContentType == manifestType {
+				l := plen
+				if epl < l {
+					l = epl
+				}
+				if prefix[:l] == entry.Path[:l] {
+					err := self.loadSubTrie(entry, quitC)
+					if err != nil {
+						return err
+					}
+					err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb)
+					if err != nil {
+						return err
+					}
+				}
+			} else {
+				if (epl >= plen) && (prefix == entry.Path[:plen]) {
+					cb(entry, rp+entry.Path[plen:])
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (self *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
+	return self.listWithPrefixInt(prefix, "", quitC, cb)
+}
+
+func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) {
+
+	glog.V(logger.Detail).Infof("findPrefixOf(%s)", path)
+
+	if len(path) == 0 {
+		return self.entries[256], 0
+	}
+
+	b := byte(path[0])
+	entry = self.entries[b]
+	if entry == nil {
+		return self.entries[256], 0
+	}
+	epl := len(entry.Path)
+	glog.V(logger.Detail).Infof("path = %v  entry.Path = %v  epl = %v", path, entry.Path, epl)
+	if (len(path) >= epl) && (path[:epl] == entry.Path) {
+		glog.V(logger.Detail).Infof("entry.ContentType = %v", entry.ContentType)
+		if entry.ContentType == manifestType {
+			if self.loadSubTrie(entry, quitC) != nil {
+				return nil, 0
+			}
+			entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC)
+			if entry != nil {
+				pos += epl
+			}
+		} else {
+			pos = epl
+		}
+	} else {
+		entry = nil
+	}
+	return
+}
+
+// file system manifest always contains regularized paths
+// no leading or trailing slashes, only single slashes inside
+func RegularSlashes(path string) (res string) {
+	for i := 0; i < len(path); i++ {
+		if (path[i] != '/') || ((i > 0) && (path[i-1] != '/')) {
+			res = res + path[i:i+1]
+		}
+	}
+	if (len(res) > 0) && (res[len(res)-1] == '/') {
+		res = res[:len(res)-1]
+	}
+	return
+}
+
+func (self *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) {
+	path := RegularSlashes(spath)
+	var pos int
+	quitC := make(chan bool)
+	entry, pos = self.findPrefixOf(path, quitC)
+	return entry, path[:pos]
+}

swarm/api/manifest_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	// "encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+func manifest(paths ...string) (manifestReader storage.LazySectionReader) {
+	var entries []string
+	for _, path := range paths {
+		entry := fmt.Sprintf(`{"path":"%s"}`, path)
+		entries = append(entries, entry)
+	}
+	manifest := fmt.Sprintf(`{"entries":[%s]}`, strings.Join(entries, ","))
+	return &storage.LazyTestSectionReader{
+		SectionReader: io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest))),
+	}
+}
+
+func testGetEntry(t *testing.T, path, match string, paths ...string) *manifestTrie {
+	quitC := make(chan bool)
+	trie, err := readManifest(manifest(paths...), nil, nil, quitC)
+	if err != nil {
+		t.Errorf("unexpected error making manifest: %v", err)
+	}
+	checkEntry(t, path, match, trie)
+	return trie
+}
+
+func checkEntry(t *testing.T, path, match string, trie *manifestTrie) {
+	entry, fullpath := trie.getEntry(path)
+	if match == "-" && entry != nil {
+		t.Errorf("expected no match for '%s', got '%s'", path, fullpath)
+	} else if entry == nil {
+		if match != "-" {
+			t.Errorf("expected entry '%s' to match '%s', got no match", match, path)
+		}
+	} else if fullpath != match {
+		t.Errorf("incorrect entry retrieved for '%s'. expected path '%v', got '%s'", path, match, fullpath)
+	}
+}
+
+func TestGetEntry(t *testing.T) {
+	// file system manifest always contains regularized paths
+	testGetEntry(t, "a", "a", "a")
+	testGetEntry(t, "b", "-", "a")
+	testGetEntry(t, "/a//", "a", "a")
+	// fallback
+	testGetEntry(t, "/a", "", "")
+	testGetEntry(t, "/a/b", "a/b", "a/b")
+	// longest/deepest math
+	testGetEntry(t, "a/b", "-", "a", "a/ba", "a/b/c")
+	testGetEntry(t, "a/b", "a/b", "a", "a/b", "a/bb", "a/b/c")
+	testGetEntry(t, "//a//b//", "a/b", "a", "a/b", "a/bb", "a/b/c")
+}
+
+func TestDeleteEntry(t *testing.T) {
+
+}

swarm/api/storage.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+type Response struct {
+	MimeType string
+	Status   int
+	Size     int64
+	// Content  []byte
+	Content string
+}
+
+// implements a service
+type Storage struct {
+	api *Api
+}
+
+func NewStorage(api *Api) *Storage {
+	return &Storage{api}
+}
+
+// Put uploads the content to the swarm with a simple manifest speficying
+// its content type
+func (self *Storage) Put(content, contentType string) (string, error) {
+	return self.api.Put(content, contentType)
+}
+
+// Get retrieves the content from bzzpath and reads the response in full
+// It returns the Response object, which serialises containing the
+// response body as the value of the Content field
+// NOTE: if error is non-nil, sResponse may still have partial content
+// the actual size of which is given in len(resp.Content), while the expected
+// size is resp.Size
+func (self *Storage) Get(bzzpath string) (*Response, error) {
+	reader, mimeType, status, err := self.api.Get(bzzpath, true)
+	if err != nil {
+		return nil, err
+	}
+	quitC := make(chan bool)
+	expsize, err := reader.Size(quitC)
+	if err != nil {
+		return nil, err
+	}
+	body := make([]byte, expsize)
+	size, err := reader.Read(body)
+	if int64(size) == expsize {
+		err = nil
+	}
+	return &Response{mimeType, status, expsize, string(body[:size])}, err
+}
+
+// Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash,
+// and merge on  to it. creating an entry w conentType (mime)
+func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
+	return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true)
+}

swarm/api/storage_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"testing"
+)
+
+func testStorage(t *testing.T, f func(*Storage)) {
+	testApi(t, func(api *Api) {
+		f(NewStorage(api))
+	})
+}
+
+func TestStoragePutGet(t *testing.T) {
+	testStorage(t, func(api *Storage) {
+		content := "hello"
+		exp := expResponse(content, "text/plain", 0)
+		// exp := expResponse([]byte(content), "text/plain", 0)
+		bzzhash, err := api.Put(content, exp.MimeType)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		// to check put against the Api#Get
+		resp0 := testGet(t, api.api, bzzhash)
+		checkResponse(t, resp0, exp)
+
+		// check storage#Get
+		resp, err := api.Get(bzzhash)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		checkResponse(t, &testResponse{nil, resp}, exp)
+	})
+}

swarm/api/testapi.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"github.com/ethereum/go-ethereum/swarm/network"
+)
+
+type Control struct {
+	api  *Api
+	hive *network.Hive
+}
+
+func NewControl(api *Api, hive *network.Hive) *Control {
+	return &Control{api, hive}
+}
+
+func (self *Control) BlockNetworkRead(on bool) {
+	self.hive.BlockNetworkRead(on)
+}
+
+func (self *Control) SyncEnabled(on bool) {
+	self.hive.SyncEnabled(on)
+}
+
+func (self *Control) SwapEnabled(on bool) {
+	self.hive.SwapEnabled(on)
+}
+
+func (self *Control) Hive() string {
+	return self.hive.String()
+}

swarm/api/testdata/test0/index.css

+h1 {
+  color: black;
+  font-size: 12px;
+  background-color: orange;
+  border: 4px solid black;
+}
+body {
+  background-color: orange
+}
\ No newline at end of file

swarm/api/testdata/test0/index.html

+<!DOCTYPE html>
+<html>
+  <head>
+    <link rel="stylesheet" href="index.css">
+  </head>
+  <body>
+    <h1>Swarm Test</h1>
+    <img src="img/logo.gif" align="center", alt="Ethereum logo">
+  </body>
+</html>
\ No newline at end of file

swarm/network/depo.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package network
+
+import (
+	"bytes"
+	"encoding/binary"
+	"time"
+
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+// Handler for storage/retrieval related protocol requests
+// implements the StorageHandler interface used by the bzz protocol
+type Depo struct {
+	hashfunc   storage.Hasher
+	localStore storage.ChunkStore
+	netStore   storage.ChunkStore
+}
+
+func NewDepo(hash storage.Hasher, localStore, remoteStore storage.ChunkStore) *Depo {
+	return &Depo{
+		hashfunc:   hash,
+		localStore: localStore,
+		netStore:   remoteStore, // entrypoint internal
+	}
+}
+
+// Handles UnsyncedKeysMsg after msg decoding - unsynced hashes upto sync state
+// * the remote sync state is just stored and handled in protocol
+// * filters through the new syncRequests and send the ones missing
+// * back immediately as a deliveryRequest message
+// * empty message just pings back for more (is this needed?)
+// * strict signed sync states may be needed.
+func (self *Depo) HandleUnsyncedKeysMsg(req *unsyncedKeysMsgData, p *peer) error {
+	unsynced := req.Unsynced
+	var missing []*syncRequest
+	var chunk *storage.Chunk
+	var err error
+	for _, req := range unsynced {
+		// skip keys that are found,
+		chunk, err = self.localStore.Get(storage.Key(req.Key[:]))
+		if err != nil || chunk.SData == nil {
+			missing = append(missing, req)
+		}
+	}
+	glog.V(logger.Debug).Infof("Depo.HandleUnsyncedKeysMsg: received %v unsynced keys: %v missing. new state: %v", len(unsynced), len(missing), req.State)
+	glog.V(logger.Detail).Infof("Depo.HandleUnsyncedKeysMsg: received %v", unsynced)
+	// send delivery request with missing keys
+	err = p.deliveryRequest(missing)
+	if err != nil {
+		return err
+	}
+	// set peers state to persist
+	p.syncState = req.State
+	return nil
+}
+
+// Handles deliveryRequestMsg
+// * serves actual chunks asked by the remote peer
+// by pushing to the delivery queue (sync db) of the correct priority
+// (remote peer is free to reprioritize)
+// * the message implies remote peer wants more, so trigger for
+// * new outgoing unsynced keys message is fired
+func (self *Depo) HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer) error {
+	deliver := req.Deliver
+	// queue the actual delivery of a chunk ()
+	glog.V(logger.Detail).Infof("Depo.HandleDeliveryRequestMsg: received %v delivery requests: %v", len(deliver), deliver)
+	for _, sreq := range deliver {
+		// TODO: look up in cache here or in deliveries
+		// priorities are taken from the message so the remote party can
+		// reprioritise to at their leisure
+		// r = self.pullCached(sreq.Key) // pulls and deletes from cache
+		Push(p, sreq.Key, sreq.Priority)
+	}
+
+	// sends it out as unsyncedKeysMsg
+	p.syncer.sendUnsyncedKeys()
+	return nil
+}
+
+// the entrypoint for store requests coming from the bzz wire protocol
+// if key found locally, return. otherwise
+// remote is untrusted, so hash is verified and chunk passed on to NetStore
+func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) {
+	req.from = p
+	chunk, err := self.localStore.Get(req.Key)
+	switch {
+	case err != nil:
+		glog.V(logger.Detail).Infof("Depo.handleStoreRequest: %v not found locally. create new chunk/request", req.Key)
+		// not found in memory cache, ie., a genuine store request
+		// create chunk
+		chunk = storage.NewChunk(req.Key, nil)
+
+	case chunk.SData == nil:
+		// found chunk in memory store, needs the data, validate now
+		hasher := self.hashfunc()
+		hasher.Write(req.SData)
+		if !bytes.Equal(hasher.Sum(nil), req.Key) {
+			// data does not validate, ignore
+			// TODO: peer should be penalised/dropped?
+			glog.V(logger.Warn).Infof("Depo.HandleStoreRequest: chunk invalid. store request ignored: %v", req)
+			return
+		}
+		glog.V(logger.Detail).Infof("Depo.HandleStoreRequest: %v. request entry found", req)
+
+	default:
+		// data is found, store request ignored
+		// this should update access count?
+		glog.V(logger.Detail).Infof("Depo.HandleStoreRequest: %v found locally. ignore.", req)
+		return
+	}
+
+	// update chunk with size and data
+	chunk.SData = req.SData // protocol validates that SData is minimum 9 bytes long (int64 size  + at least one byte of data)
+	chunk.Size = int64(binary.LittleEndian.Uint64(req.SData[0:8]))
+	glog.V(logger.Detail).Infof("delivery of %p from %v", chunk, p)
+	chunk.Source = p
+	self.netStore.Put(chunk)
+}
+
+// entrypoint for retrieve requests coming from the bzz wire protocol
+// checks swap balance - return if peer has no credit
+func (self *Depo) HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer) {
+	req.from = p
+	// swap - record credit for 1 request
+	// note that only charge actual reqsearches
+	var err error
+	if p.swap != nil {
+		err = p.swap.Add(1)
+	}
+	if err != nil {
+		glog.V(logger.Warn).Infof("Depo.HandleRetrieveRequest: %v - cannot process request: %v", req.Key.Log(), err)
+		return
+	}
+
+	// call storage.NetStore#Get which
+	// blocks until local retrieval finished
+	// launches cloud retrieval
+	chunk, _ := self.netStore.Get(req.Key)
+	req = self.strategyUpdateRequest(chunk.Req, req)
+	// check if we can immediately deliver
+	if chunk.SData != nil {
+		glog.V(logger.Detail).Infof("Depo.HandleRetrieveRequest: %v - content found, delivering...", req.Key.Log())
+
+		if req.MaxSize == 0 || int64(req.MaxSize) >= chunk.Size {
+			sreq := &storeRequestMsgData{
+				Id:             req.Id,
+				Key:            chunk.Key,
+				SData:          chunk.SData,
+				requestTimeout: req.timeout, //
+			}
+			p.syncer.addRequest(sreq, DeliverReq)
+		} else {
+			glog.V(logger.Detail).Infof("Depo.HandleRetrieveRequest: %v - content found, not wanted", req.Key.Log())
+		}
+	} else {
+		glog.V(logger.Detail).Infof("Depo.HandleRetrieveRequest: %v - content not found locally. asked swarm for help. will get back", req.Key.Log())
+	}
+}
+
+// add peer request the chunk and decides the timeout for the response if still searching
+func (self *Depo) strategyUpdateRequest(rs *storage.RequestStatus, origReq *retrieveRequestMsgData) (req *retrieveRequestMsgData) {
+	glog.V(logger.Detail).Infof("Depo.strategyUpdateRequest: key %v", origReq.Key.Log())
+	// we do not create an alternative one
+	req = origReq
+	if rs != nil {
+		self.addRequester(rs, req)
+		req.setTimeout(self.searchTimeout(rs, req))
+	}
+	return
+}
+
+// decides the timeout promise sent with the immediate peers response to a retrieve request
+// if timeout is explicitly set and expired
+func (self *Depo) searchTimeout(rs *storage.RequestStatus, req *retrieveRequestMsgData) (timeout *time.Time) {
+	reqt := req.getTimeout()
+	t := time.Now().Add(searchTimeout)
+	if reqt != nil && reqt.Before(t) {
+		return reqt
+	} else {
+		return &t
+	}
+}
+
+/*
+adds a new peer to an existing open request
+only add if less than requesterCount peers forwarded the same request id so far
+note this is done irrespective of status (searching or found)
+*/
+func (self *Depo) addRequester(rs *storage.RequestStatus, req *retrieveRequestMsgData) {
+	glog.V(logger.Detail).Infof("Depo.addRequester: key %v - add peer to req.Id %v", req.Key.Log(), req.from, req.Id)
+	list := rs.Requesters[req.Id]
+	rs.Requesters[req.Id] = append(list, req)
+}

swarm/network/forwarding.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package network
+
+import (
+	"math/rand"
+	"time"
+
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+const requesterCount = 3
+
+/*
+forwarder implements the CloudStore interface (use by storage.NetStore)
+and serves as the cloud store backend orchestrating storage/retrieval/delivery
+via the native bzz protocol
+which uses an MSB logarithmic distance-based semi-permanent Kademlia table for
+* recursive forwarding style routing for retrieval
+* smart syncronisation
+*/
+
+type forwarder struct {
+	hive *Hive
+}
+
+func NewForwarder(hive *Hive) *forwarder {
+	return &forwarder{hive: hive}
+}
+
+// generate a unique id uint64
+func generateId() uint64 {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	return uint64(r.Int63())
+}
+
+var searchTimeout = 3 * time.Second
+
+// forwarding logic
+// logic propagating retrieve requests to peers given by the kademlia hive
+func (self *forwarder) Retrieve(chunk *storage.Chunk) {
+	peers := self.hive.getPeers(chunk.Key, 0)
+	glog.V(logger.Detail).Infof("forwarder.Retrieve: %v - received %d peers from KΛÐΞMLIΛ...", chunk.Key.Log(), len(peers))
+OUT:
+	for _, p := range peers {
+		glog.V(logger.Detail).Infof("forwarder.Retrieve: sending retrieveRequest %v to peer [%v]", chunk.Key.Log(), p)
+		for _, recipients := range chunk.Req.Requesters {
+			for _, recipient := range recipients {
+				req := recipient.(*retrieveRequestMsgData)
+				if req.from.Addr() == p.Addr() {
+					continue OUT
+				}
+			}
+		}
+		req := &retrieveRequestMsgData{
+			Key: chunk.Key,
+			Id:  generateId(),
+		}
+		var err error
+		if p.swap != nil {
+			err = p.swap.Add(-1)
+		}
+		if err == nil {
+			p.retrieve(req)
+			break OUT
+		}
+		glog.V(logger.Warn).Infof("forwarder.Retrieve: unable to send retrieveRequest to peer [%v]: %v", chunk.Key.Log(), err)
+	}
+}
+
+// requests to specific peers given by the kademlia hive
+// except for peers that the store request came from (if any)
+// delivery queueing taken care of by syncer
+func (self *forwarder) Store(chunk *storage.Chunk) {
+	var n int
+	msg := &storeRequestMsgData{
+		Key:   chunk.Key,
+		SData: chunk.SData,
+	}
+	var source *peer
+	if chunk.Source != nil {
+		source = chunk.Source.(*peer)
+	}
+	for _, p := range self.hive.getPeers(chunk.Key, 0) {
+		glog.V(logger.Detail).Infof("forwarder.Store: %v %v", p, chunk)
+
+		if p.syncer != nil && (source == nil || p.Addr() != source.Addr()) {
+			n++
+			Deliver(p, msg, PropagateReq)
+		}
+	}
+	glog.V(logger.Detail).Infof("forwarder.Store: sent to %v peers (chunk = %v)", n, chunk)
+}
+
+// once a chunk is found deliver it to its requesters unless timed out
+func (self *forwarder) Deliver(chunk *storage.Chunk) {
+	// iterate over request entries
+	for id, requesters := range chunk.Req.Requesters {
+		counter := requesterCount
+		msg := &storeRequestMsgData{
+			Key:   chunk.Key,
+			SData: chunk.SData,
+		}
+		var n int
+		var req *retrieveRequestMsgData
+		// iterate over requesters with the same id
+		for id, r := range requesters {
+			req = r.(*retrieveRequestMsgData)
+			if req.timeout == nil || req.timeout.After(time.Now()) {
+				glog.V(logger.Detail).Infof("forwarder.Deliver: %v -> %v", req.Id, req.from)
+				msg.Id = uint64(id)
+				Deliver(req.from, msg, DeliverReq)
+				n++
+				counter--
+				if counter <= 0 {
+					break
+				}
+			}
+		}
+		glog.V(logger.Detail).Infof("forwarder.Deliver: submit chunk %v (request id %v) for delivery to %v peers", chunk.Key.Log(), id, n)
+	}
+}
+
+// initiate delivery of a chunk to a particular peer via syncer#addRequest
+// depending on syncer mode and priority settings and sync request type
+// this either goes via confirmation roundtrip or queued or pushed directly
+func Deliver(p *peer, req interface{}, ty int) {
+	p.syncer.addRequest(req, ty)
+}
+
+// push chunk over to peer
+func Push(p *peer, key storage.Key, priority uint) {
+	p.syncer.doDelivery(key, priority, p.syncer.quit)
+}

swarm/network/kademlia/address.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package kademlia
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+type Address common.Hash
+
+func (a Address) String() string {
+	return fmt.Sprintf("%x", a[:])
+}
+
+func (a *Address) MarshalJSON() (out []byte, err error) {
+	return []byte(`"` + a.String() + `"`), nil
+}
+
+func (a *Address) UnmarshalJSON(value []byte) error {
+	*a = Address(common.HexToHash(string(value[1 : len(value)-1])))
+	return nil
+}
+
+// the string form of the binary representation of an address (only first 8 bits)
+func (a Address) Bin() string {
+	var bs []string
+	for _, b := range a[:] {
+		bs = append(bs, fmt.Sprintf("%08b", b))
+	}
+	return strings.Join(bs, "")
+}
+
+/*
+Proximity(x, y) returns the proximity order of the MSB distance between x and y
+
+The distance metric MSB(x, y) of two equal length byte sequences x an y is the
+value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed.
+the binary cast is big endian: most significant bit first (=MSB).
+
+Proximity(x, y) is a discrete logarithmic scaling of the MSB distance.
+It is defined as the reverse rank of the integer part of the base 2
+logarithm of the distance.
+It is calculated by counting the number of common leading zeros in the (MSB)
+binary representation of the x^y.
+
+(0 farthest, 255 closest, 256 self)
+*/
+func proximity(one, other Address) (ret int) {
+	for i := 0; i < len(one); i++ {
+		oxo := one[i] ^ other[i]
+		for j := 0; j < 8; j++ {
+			if (uint8(oxo)>>uint8(7-j))&0x01 != 0 {
+				return i*8 + j
+			}
+		}
+	}
+	return len(one) * 8
+}
+
+// Address.ProxCmp compares the distances a->target and b->target.
+// Returns -1 if a is closer to target, 1 if b is closer to target
+// and 0 if they are equal.
+func (target Address) ProxCmp(a, b Address) int {
+	for i := range target {
+		da := a[i] ^ target[i]
+		db := b[i] ^ target[i]
+		if da > db {
+			return 1
+		} else if da < db {
+			return -1
+		}
+	}
+	return 0
+}
+
+// randomAddressAt(address, prox) generates a random address
+// at proximity order prox relative to address
+// if prox is negative a random address is generated
+func RandomAddressAt(self Address, prox int) (addr Address) {
+	addr = self
+	var pos int
+	if prox >= 0 {
+		pos = prox / 8
+		trans := prox % 8
+		transbytea := byte(0)
+		for j := 0; j <= trans; j++ {
+			transbytea |= 1 << uint8(7-j)
+		}
+		flipbyte := byte(1 << uint8(7-trans))
+		transbyteb := transbytea ^ byte(255)
+		randbyte := byte(rand.Intn(255))
+		addr[pos] = ((addr[pos] & transbytea) ^ flipbyte) | randbyte&transbyteb
+	}
+	for i := pos + 1; i < len(addr); i++ {
+		addr[i] = byte(rand.Intn(255))
+	}
+
+	return
+}
+
+// KeyRange(a0, a1, proxLimit) returns the address inclusive address
+// range that contain addresses closer to one than other
+func KeyRange(one, other Address, proxLimit int) (start, stop Address) {
+	prox := proximity(one, other)
+	if prox >= proxLimit {
+		prox = proxLimit
+	}
+	start = CommonBitsAddrByte(one, other, byte(0x00), prox)
+	stop = CommonBitsAddrByte(one, other, byte(0xff), prox)
+	return
+}
+
+func CommonBitsAddrF(self, other Address, f func() byte, p int) (addr Address) {
+	prox := proximity(self, other)
+	var pos int
+	if p <= prox {
+		prox = p
+	}
+	pos = prox / 8
+	addr = self
+	trans := byte(prox % 8)
+	var transbytea byte
+	if p > prox {
+		transbytea = byte(0x7f)
+	} else {
+		transbytea = byte(0xff)
+	}
+	transbytea >>= trans
+	transbyteb := transbytea ^ byte(0xff)
+	addrpos := addr[pos]
+	addrpos &= transbyteb
+	if p > prox {
+		addrpos ^= byte(0x80 >> trans)
+	}
+	addrpos |= transbytea & f()
+	addr[pos] = addrpos
+	for i := pos + 1; i < len(addr); i++ {
+		addr[i] = f()
+	}
+
+	return
+}
+
+func CommonBitsAddr(self, other Address, prox int) (addr Address) {
+	return CommonBitsAddrF(self, other, func() byte { return byte(rand.Intn(255)) }, prox)
+}
+
+func CommonBitsAddrByte(self, other Address, b byte, prox int) (addr Address) {
+	return CommonBitsAddrF(self, other, func() byte { return b }, prox)
+}
+
+// randomAddressAt() generates a random address
+func RandomAddress() Address {
+	return RandomAddressAt(Address{}, -1)
+}

swarm/network/kademlia/address_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package kademlia
+
+import (
+	"math/rand"
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+func (Address) Generate(rand *rand.Rand, size int) reflect.Value {
+	var id Address
+	for i := 0; i < len(id); i++ {
+		id[i] = byte(uint8(rand.Intn(255)))
+	}
+	return reflect.ValueOf(id)
+}
+
+func TestCommonBitsAddrF(t *testing.T) {
+	a := Address(common.HexToHash("0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
+	b := Address(common.HexToHash("0x8123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
+	c := Address(common.HexToHash("0x4123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
+	d := Address(common.HexToHash("0x0023456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
+	e := Address(common.HexToHash("0x01A3456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
+	ab := CommonBitsAddrF(a, b, func() byte { return byte(0x00) }, 10)
+	expab := Address(common.HexToHash("0x8000000000000000000000000000000000000000000000000000000000000000"))
+
+	if ab != expab {
+		t.Fatalf("%v != %v", ab, expab)
+	}
+	ac := CommonBitsAddrF(a, c, func() byte { return byte(0x00) }, 10)
+	expac := Address(common.HexToHash("0x4000000000000000000000000000000000000000000000000000000000000000"))
+
+	if ac != expac {
+		t.Fatalf("%v != %v", ac, expac)
+	}
+	ad := CommonBitsAddrF(a, d, func() byte { return byte(0x00) }, 10)
+	expad := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"))
+
+	if ad != expad {
+		t.Fatalf("%v != %v", ad, expad)
+	}
+	ae := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 10)
+	expae := Address(common.HexToHash("0x0180000000000000000000000000000000000000000000000000000000000000"))
+
+	if ae != expae {
+		t.Fatalf("%v != %v", ae, expae)
+	}
+	acf := CommonBitsAddrF(a, c, func() byte { return byte(0xff) }, 10)
+	expacf := Address(common.HexToHash("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"))
+
+	if acf != expacf {
+		t.Fatalf("%v != %v", acf, expacf)
+	}
+	aeo := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 2)
+	expaeo := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"))
+
+	if aeo != expaeo {
+		t.Fatalf("%v != %v", aeo, expaeo)
+	}
+	aep := CommonBitsAddrF(a, e, func() byte { return byte(0xff) }, 2)
+	expaep := Address(common.HexToHash("0x3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"))
+
+	if aep != expaep {
+		t.Fatalf("%v != %v", aep, expaep)
+	}
+
+}
+
+func TestRandomAddressAt(t *testing.T) {
+	var a Address
+	for i := 0; i < 100; i++ {
+		a = RandomAddress()
+		prox := rand.Intn(255)
+		b := RandomAddressAt(a, prox)
+		if proximity(a, b) != prox {
+			t.Fatalf("incorrect address prox(%v, %v) == %v (expected %v)", a, b, proximity(a, b), prox)
+		}
+	}
+}

swarm/network/kademlia/kademlia_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package kademlia
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"testing/quick"
+	"time"
+)
+
+var (
+	quickrand           = rand.New(rand.NewSource(time.Now().Unix()))
+	quickcfgFindClosest = &quick.Config{MaxCount: 50, Rand: quickrand}
+	quickcfgBootStrap   = &quick.Config{MaxCount: 100, Rand: quickrand}
+)
+
+type testNode struct {
+	addr Address
+}
+
+func (n *testNode) String() string {
+	return fmt.Sprintf("%x", n.addr[:])
+}
+
+func (n *testNode) Addr() Address {
+	return n.addr
+}
+
+func (n *testNode) Drop() {
+}
+
+func (n *testNode) Url() string {
+	return ""
+}
+
+func (n *testNode) LastActive() time.Time {
+	return time.Now()
+}
+
+func TestOn(t *testing.T) {
+	addr, ok := gen(Address{}, quickrand).(Address)
+	other, ok := gen(Address{}, quickrand).(Address)
+	if !ok {
+		t.Errorf("oops")
+	}
+	kad := New(addr, NewKadParams())
+	err := kad.On(&testNode{addr: other}, nil)
+	_ = err
+}
+
+func TestBootstrap(t *testing.T) {
+
+	test := func(test *bootstrapTest) bool {
+		// for any node kad.le, Target and N
+		params := NewKadParams()
+		params.MaxProx = test.MaxProx
+		params.BucketSize = test.BucketSize
+		params.ProxBinSize = test.BucketSize
+		kad := New(test.Self, params)
+		var err error
+
+		for p := 0; p < 9; p++ {
+			var nrs []*NodeRecord
+			n := math.Pow(float64(2), float64(7-p))
+			for i := 0; i < int(n); i++ {
+				addr := RandomAddressAt(test.Self, p)
+				nrs = append(nrs, &NodeRecord{
+					Addr: addr,
+				})
+			}
+			kad.Add(nrs)
+		}
+
+		node := &testNode{test.Self}
+
+		n := 0
+		for n < 100 {
+			err = kad.On(node, nil)
+			if err != nil {
+				t.Fatalf("backend not accepting node: %v", err)
+			}
+
+			record, need, _ := kad.Suggest()
+			if !need {
+				break
+			}
+			n++
+			if record == nil {
+				continue
+			}
+			node = &testNode{record.Addr}
+		}
+		exp := test.BucketSize * (test.MaxProx + 1)
+		if kad.Count() != exp {
+			t.Errorf("incorrect number of peers, expected %d, got %d\n%v", exp, kad.Count(), kad)
+			return false
+		}
+		return true
+	}
+	if err := quick.Check(test, quickcfgBootStrap); err != nil {
+		t.Error(err)
+	}
+
+}
+
+func TestFindClosest(t *testing.T) {
+
+	test := func(test *FindClosestTest) bool {
+		// for any node kad.le, Target and N
+		params := NewKadParams()
+		params.MaxProx = 7
+		kad := New(test.Self, params)
+		var err error
+		for _, node := range test.All {
+			err = kad.On(node, nil)
+			if err != nil && err.Error() != "bucket full" {
+				t.Fatalf("backend not accepting node: %v", err)
+			}
+		}
+
+		if len(test.All) == 0 || test.N == 0 {
+			return true
+		}
+		nodes := kad.FindClosest(test.Target, test.N)
+
+		// check that the number of results is min(N, kad.len)
+		wantN := test.N
+		if tlen := kad.Count(); tlen < test.N {
+			wantN = tlen
+		}
+
+		if len(nodes) != wantN {
+			t.Errorf("wrong number of nodes: got %d, want %d", len(nodes), wantN)
+			return false
+		}
+
+		if hasDuplicates(nodes) {
+			t.Errorf("result contains duplicates")
+			return false
+		}
+
+		if !sortedByDistanceTo(test.Target, nodes) {
+			t.Errorf("result is not sorted by distance to target")
+			return false
+		}
+
+		// check that the result nodes have minimum distance to target.
+		farthestResult := nodes[len(nodes)-1].Addr()
+		for i, b := range kad.buckets {
+			for j, n := range b {
+				if contains(nodes, n.Addr()) {
+					continue // don't run the check below for nodes in result
+				}
+				if test.Target.ProxCmp(n.Addr(), farthestResult) < 0 {
+					_ = i * j
+					t.Errorf("kad.le contains node that is closer to target but it's not in result")
+					return false
+				}
+			}
+		}
+		return true
+	}
+	if err := quick.Check(test, quickcfgFindClosest); err != nil {
+		t.Error(err)
+	}
+}
+
+type proxTest struct {
+	add   bool
+	index int
+	addr  Address
+}
+
+var (
+	addresses []Address
+)
+
+func TestProxAdjust(t *testing.T) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	self := gen(Address{}, r).(Address)
+	params := NewKadParams()
+	params.MaxProx = 7
+	kad := New(self, params)
+
+	var err error
+	for i := 0; i < 100; i++ {
+		a := gen(Address{}, r).(Address)
+		addresses = append(addresses, a)
+		err = kad.On(&testNode{addr: a}, nil)
+		if err != nil && err.Error() != "bucket full" {
+			t.Fatalf("backend not accepting node: %v", err)
+		}
+		if !kad.proxCheck(t) {
+			return
+		}
+	}
+	test := func(test *proxTest) bool {
+		node := &testNode{test.addr}
+		if test.add {
+			kad.On(node, nil)
+		} else {
+			kad.Off(node, nil)
+		}
+		return kad.proxCheck(t)
+	}
+	if err := quick.Check(test, quickcfgFindClosest); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestSaveLoad(t *testing.T) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	addresses := gen([]Address{}, r).([]Address)
+	self := RandomAddress()
+	params := NewKadParams()
+	params.MaxProx = 7
+	kad := New(self, params)
+
+	var err error
+
+	for _, a := range addresses {
+		err = kad.On(&testNode{addr: a}, nil)
+		if err != nil && err.Error() != "bucket full" {
+			t.Fatalf("backend not accepting node: %v", err)
+		}
+	}
+	nodes := kad.FindClosest(self, 100)
+
+	path := filepath.Join(os.TempDir(), "bzz-kad-test-save-load.peers")
+	err = kad.Save(path, nil)
+	if err != nil && err.Error() != "bucket full" {
+		t.Fatalf("unepected error saving kaddb: %v", err)
+	}
+	kad = New(self, params)
+	err = kad.Load(path, nil)
+	if err != nil && err.Error() != "bucket full" {
+		t.Fatalf("unepected error loading kaddb: %v", err)
+	}
+	for _, b := range kad.db.Nodes {
+		for _, node := range b {
+			err = kad.On(&testNode{node.Addr}, nil)
+			if err != nil && err.Error() != "bucket full" {
+				t.Fatalf("backend not accepting node: %v", err)
+			}
+		}
+	}
+	loadednodes := kad.FindClosest(self, 100)
+	for i, node := range loadednodes {
+		if nodes[i].Addr() != node.Addr() {
+			t.Errorf("node mismatch at %d/%d: %v != %v", i, len(nodes), nodes[i].Addr(), node.Addr())
+		}
+	}
+}
+
+func (self *Kademlia) proxCheck(t *testing.T) bool {
+	var sum int
+	for i, b := range self.buckets {
+		l := len(b)
+		// if we are in the high prox multibucket
+		if i >= self.proxLimit {
+			sum += l
+		} else if l == 0 {
+			t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b), self.proxLimit, self)
+			return false
+		}
+	}
+	// check if merged high prox bucket does not exceed size
+	if sum > 0 {
+		if sum != self.proxSize {
+			t.Errorf("proxSize incorrect, expected %v, got %v", sum, self.proxSize)
+			return false
+		}
+		last := len(self.buckets[self.proxLimit])
+		if last > 0 && sum >= self.ProxBinSize+last {
+			t.Errorf("proxLimit %v incorrect, redundant non-empty bucket %d added to proxBin with %v (target %v)\n%v", self.proxLimit, last, sum-last, self.ProxBinSize, self)
+			return false
+		}
+		if self.proxLimit > 0 && sum < self.ProxBinSize {
+			t.Errorf("proxLimit %v incorrect. proxSize %v is less than target %v, yet there is more peers", self.proxLimit, sum, self.ProxBinSize)
+			return false
+		}
+	}
+	return true
+}
+
+type bootstrapTest struct {
+	MaxProx    int
+	BucketSize int
+	Self       Address
+}
+
+func (*bootstrapTest) Generate(rand *rand.Rand, size int) reflect.Value {
+	t := &bootstrapTest{
+		Self:       gen(Address{}, rand).(Address),
+		MaxProx:    5 + rand.Intn(2),
+		BucketSize: rand.Intn(3) + 1,
+	}
+	return reflect.ValueOf(t)
+}
+
+type FindClosestTest struct {
+	Self   Address
+	Target Address
+	All    []Node
+	N      int
+}
+
+func (c FindClosestTest) String() string {
+	return fmt.Sprintf("A: %064x\nT: %064x\n(%d)\n", c.Self[:], c.Target[:], c.N)
+}
+
+func (*FindClosestTest) Generate(rand *rand.Rand, size int) reflect.Value {
+	t := &FindClosestTest{
+		Self:   gen(Address{}, rand).(Address),
+		Target: gen(Address{}, rand).(Address),
+		N:      rand.Intn(bucketSize),
+	}
+	for _, a := range gen([]Address{}, rand).([]Address) {
+		t.All = append(t.All, &testNode{addr: a})
+	}
+	return reflect.ValueOf(t)
+}
+
+func (*proxTest) Generate(rand *rand.Rand, size int) reflect.Value {
+	var add bool
+	if rand.Intn(1) == 0 {
+		add = true
+	}
+	var t *proxTest
+	if add {
+		t = &proxTest{
+			addr: gen(Address{}, rand).(Address),
+			add:  add,
+		}
+	} else {
+		t = &proxTest{
+			index: rand.Intn(len(addresses)),
+			add:   add,
+		}
+	}
+	return reflect.ValueOf(t)
+}
+
+func hasDuplicates(slice []Node) bool {
+	seen := make(map[Address]bool)
+	for _, node := range slice {
+		if seen[node.Addr()] {
+			return true
+		}
+		seen[node.Addr()] = true
+	}
+	return false
+}
+
+func contains(nodes []Node, addr Address) bool {
+	for _, n := range nodes {
+		if n.Addr() == addr {
+			return true
+		}
+	}
+	return false
+}
+
+// gen wraps quick.Value so it's easier to use.
+// it generates a random value of the given value's type.
+func gen(typ interface{}, rand *rand.Rand) interface{} {
+	v, ok := quick.Value(reflect.TypeOf(typ), rand)
+	if !ok {
+		panic(fmt.Sprintf("couldn't generate random value of type %T", typ))
+	}
+	return v.Interface()
+}

swarm/network/protocol_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package network

swarm/network/syncdb_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package network
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+func init() {
+	glog.SetV(0)
+	glog.SetToStderr(true)
+}
+
+type testSyncDb struct {
+	*syncDb
+	c         int
+	t         *testing.T
+	fromDb    chan bool
+	delivered [][]byte
+	sent      []int
+	dbdir     string
+	at        int
+}
+
+func newTestSyncDb(priority, bufferSize, batchSize int, dbdir string, t *testing.T) *testSyncDb {
+	if len(dbdir) == 0 {
+		tmp, err := ioutil.TempDir(os.TempDir(), "syncdb-test")
+		if err != nil {
+			t.Fatalf("unable to create temporary direcory %v: %v", tmp, err)
+		}
+		dbdir = tmp
+	}
+	db, err := storage.NewLDBDatabase(filepath.Join(dbdir, "requestdb"))
+	if err != nil {
+		t.Fatalf("unable to create db: %v", err)
+	}
+	self := &testSyncDb{
+		fromDb: make(chan bool),
+		dbdir:  dbdir,
+		t:      t,
+	}
+	h := crypto.Sha3Hash([]byte{0})
+	key := storage.Key(h[:])
+	self.syncDb = newSyncDb(db, key, uint(priority), uint(bufferSize), uint(batchSize), self.deliver)
+	// kick off db iterator right away, if no items on db this will allow
+	// reading from the buffer
+	return self
+
+}
+
+func (self *testSyncDb) close() {
+	self.db.Close()
+	os.RemoveAll(self.dbdir)
+}
+
+func (self *testSyncDb) push(n int) {
+	for i := 0; i < n; i++ {
+		self.buffer <- storage.Key(crypto.Sha3([]byte{byte(self.c)}))
+		self.sent = append(self.sent, self.c)
+		self.c++
+	}
+	glog.V(logger.Debug).Infof("pushed %v requests", n)
+}
+
+func (self *testSyncDb) draindb() {
+	it := self.db.NewIterator()
+	defer it.Release()
+	for {
+		it.Seek(self.start)
+		if !it.Valid() {
+			return
+		}
+		k := it.Key()
+		if len(k) == 0 || k[0] == 1 {
+			return
+		}
+		it.Release()
+		it = self.db.NewIterator()
+	}
+}
+
+func (self *testSyncDb) deliver(req interface{}, quit chan bool) bool {
+	_, db := req.(*syncDbEntry)
+	key, _, _, _, err := parseRequest(req)
+	if err != nil {
+		self.t.Fatalf("unexpected error of key %v: %v", key, err)
+	}
+	self.delivered = append(self.delivered, key)
+	select {
+	case self.fromDb <- db:
+		return true
+	case <-quit:
+		return false
+	}
+}
+
+func (self *testSyncDb) expect(n int, db bool) {
+	var ok bool
+	// for n items
+	for i := 0; i < n; i++ {
+		ok = <-self.fromDb
+		if self.at+1 > len(self.delivered) {
+			self.t.Fatalf("expected %v, got %v", self.at+1, len(self.delivered))
+		}
+		if len(self.sent) > self.at && !bytes.Equal(crypto.Sha3([]byte{byte(self.sent[self.at])}), self.delivered[self.at]) {
+			self.t.Fatalf("expected delivery %v/%v/%v to be hash of  %v, from db: %v = %v", i, n, self.at, self.sent[self.at], ok, db)
+			glog.V(logger.Debug).Infof("%v/%v/%v to be hash of  %v, from db: %v = %v", i, n, self.at, self.sent[self.at], ok, db)
+		}
+		if !ok && db {
+			self.t.Fatalf("expected delivery %v/%v/%v from db", i, n, self.at)
+		}
+		if ok && !db {
+			self.t.Fatalf("expected delivery %v/%v/%v from cache", i, n, self.at)
+		}
+		self.at++
+	}
+}
+
+func TestSyncDb(t *testing.T) {
+	priority := High
+	bufferSize := 5
+	batchSize := 2 * bufferSize
+	s := newTestSyncDb(priority, bufferSize, batchSize, "", t)
+	defer s.close()
+	defer s.stop()
+	s.dbRead(false, 0, s.deliver)
+	s.draindb()
+
+	s.push(4)
+	s.expect(1, false)
+	// 3 in buffer
+	time.Sleep(100 * time.Millisecond)
+	s.push(3)
+	// push over limit
+	s.expect(1, false)
+	// one popped from the buffer, then contention detected
+	s.expect(4, true)
+	s.push(4)
+	s.expect(5, true)
+	// depleted db, switch back to buffer
+	s.draindb()
+	s.push(5)
+	s.expect(4, false)
+	s.push(3)
+	s.expect(4, false)
+	// buffer depleted
+	time.Sleep(100 * time.Millisecond)
+	s.push(6)
+	s.expect(1, false)
+	// push into buffer full, switch to db
+	s.expect(5, true)
+	s.draindb()
+	s.push(1)
+	s.expect(1, false)
+}
+
+func TestSaveSyncDb(t *testing.T) {
+	amount := 30
+	priority := High
+	bufferSize := amount
+	batchSize := 10
+	s := newTestSyncDb(priority, bufferSize, batchSize, "", t)
+	go s.dbRead(false, 0, s.deliver)
+	s.push(amount)
+	s.stop()
+	s.db.Close()
+
+	s = newTestSyncDb(priority, bufferSize, batchSize, s.dbdir, t)
+	go s.dbRead(false, 0, s.deliver)
+	s.expect(amount, true)
+	for i, key := range s.delivered {
+		expKey := crypto.Sha3([]byte{byte(i)})
+		if !bytes.Equal(key, expKey) {
+			t.Fatalf("delivery %v expected to be key %x, got %x", i, expKey, key)
+		}
+	}
+	s.push(amount)
+	s.expect(amount, false)
+	for i := amount; i < 2*amount; i++ {
+		key := s.delivered[i]
+		expKey := crypto.Sha3([]byte{byte(i - amount)})
+		if !bytes.Equal(key, expKey) {
+			t.Fatalf("delivery %v expected to be key %x, got %x", i, expKey, key)
+		}
+	}
+	s.stop()
+	s.db.Close()
+
+	s = newTestSyncDb(priority, bufferSize, batchSize, s.dbdir, t)
+	defer s.close()
+	defer s.stop()
+
+	go s.dbRead(false, 0, s.deliver)
+	s.push(1)
+	s.expect(1, false)
+
+}