Skip to content

G
Geth-Modification
Unverified Commit eaff8929 authored by Viktor Trón's avatar Viktor Trón Committed by GitHub
Browse files
Options

Merge pull request #17041 from ethersphere/swarm-network-rewrite-merge 

Swarm POC3 - happy solstice
parents d926bf2c e187711c
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 39389 additions and 8406 deletions
.github/CODEOWNERS
View file @ eaff8929
# Lines starting with '#' are comments.
# Each line is a file pattern followed by one or more owners.
accounts/usbwallet @karalabe
consensus @karalabe
core/ @karalabe @holiman
eth/ @karalabe
les/ @zsfelfoldi
light/ @zsfelfoldi
mobile/ @karalabe
p2p/ @fjl @zsfelfoldi
whisper/ @gballet @gluk256
accounts/usbwallet @karalabe
consensus @karalabe
core/ @karalabe @holiman
eth/ @karalabe
les/ @zsfelfoldi
light/ @zsfelfoldi
mobile/ @karalabe
p2p/ @fjl @zsfelfoldi
swarm/bmt @zelig
swarm/dev @lmars
swarm/fuse @jmozah @holisticode
swarm/grafana_dashboards @nonsense
swarm/metrics @nonsense @holisticode
swarm/multihash @nolash
swarm/network/bitvector @zelig @janos @gbalint
swarm/network/priorityqueue @zelig @janos @gbalint
swarm/network/simulations @zelig
swarm/network/stream @janos @zelig @gbalint @holisticode @justelad
swarm/network/stream/intervals @janos
swarm/network/stream/testing @zelig
swarm/pot @zelig
swarm/pss @nolash @zelig @nonsense
swarm/services @zelig
swarm/state @justelad
swarm/storage/encryption @gbalint @zelig @nagydani
swarm/storage/mock @janos
swarm/storage/mru @nolash
swarm/testutil @lmars
whisper/ @gballet @gluk256
cmd/p2psim/main.go
View file @ eaff8929
......@@ -275,9 +275,8 @@ func createNode(ctx *cli.Context) error {
if len(ctx.Args()) != 0 {
return cli.ShowCommandHelp(ctx, ctx.Command.Name)
}
config := &adapters.NodeConfig{
Name: ctx.String("name"),
}
config := adapters.RandomNodeConfig()
config.Name = ctx.String("name")
if key := ctx.String("key"); key != "" {
privKey, err := crypto.HexToECDSA(key)
if err != nil {
......
cmd/swarm/config.go
View file @ eaff8929
......@@ -24,6 +24,7 @@ import (
"reflect"
"strconv"
"strings"
"time"
"unicode"
cli "gopkg.in/urfave/cli.v1"
......@@ -37,6 +38,8 @@ import (
bzzapi "github.com/ethereum/go-ethereum/swarm/api"
)
const SWARM_VERSION = "0.3"
var (
//flag definition for the dumpconfig command
DumpConfigCommand = cli.Command{
......@@ -58,19 +61,25 @@ var (
//constants for environment variables
const (
SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR"
SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT"
SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR"
SWARM_ENV_PORT = "SWARM_PORT"
SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID"
SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE"
SWARM_ENV_SWAP_API = "SWARM_SWAP_API"
SWARM_ENV_SYNC_ENABLE = "SWARM_SYNC_ENABLE"
SWARM_ENV_ENS_API = "SWARM_ENS_API"
SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR"
SWARM_ENV_CORS = "SWARM_CORS"
SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES"
GETH_ENV_DATADIR = "GETH_DATADIR"
SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR"
SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT"
SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR"
SWARM_ENV_PORT = "SWARM_PORT"
SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID"
SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE"
SWARM_ENV_SWAP_API = "SWARM_SWAP_API"
SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE"
SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY"
SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK"
SWARM_ENV_ENS_API = "SWARM_ENS_API"
SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR"
SWARM_ENV_CORS = "SWARM_CORS"
SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES"
SWARM_ENV_PSS_ENABLE = "SWARM_PSS_ENABLE"
SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH"
SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY"
SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
GETH_ENV_DATADIR = "GETH_DATADIR"
)
// These settings ensure that TOML keys use the same names as Go struct fields.
......@@ -92,10 +101,8 @@ var tomlSettings = toml.Config{
//before booting the swarm node, build the configuration
func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) {
//check for deprecated flags
checkDeprecated(ctx)
//start by creating a default config
config = bzzapi.NewDefaultConfig()
config = bzzapi.NewConfig()
//first load settings from config file (if provided)
config, err = configFileOverride(config, ctx)
if err != nil {
......@@ -168,7 +175,7 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" {
if id, _ := strconv.Atoi(networkid); id != 0 {
currentConfig.NetworkId = uint64(id)
currentConfig.NetworkID = uint64(id)
}
}
......@@ -191,12 +198,20 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
currentConfig.SwapEnabled = true
}
if ctx.GlobalIsSet(SwarmSyncEnabledFlag.Name) {
currentConfig.SyncEnabled = true
if ctx.GlobalIsSet(SwarmSyncDisabledFlag.Name) {
currentConfig.SyncEnabled = false
}
if d := ctx.GlobalDuration(SwarmSyncUpdateDelay.Name); d > 0 {
currentConfig.SyncUpdateDelay = d
}
currentConfig.SwapApi = ctx.GlobalString(SwarmSwapAPIFlag.Name)
if currentConfig.SwapEnabled && currentConfig.SwapApi == "" {
if ctx.GlobalIsSet(SwarmDeliverySkipCheckFlag.Name) {
currentConfig.DeliverySkipCheck = true
}
currentConfig.SwapAPI = ctx.GlobalString(SwarmSwapAPIFlag.Name)
if currentConfig.SwapEnabled && currentConfig.SwapAPI == "" {
utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
}
......@@ -209,10 +224,6 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
currentConfig.EnsAPIs = ensAPIs
}
if ensaddr := ctx.GlobalString(DeprecatedEnsAddrFlag.Name); ensaddr != "" {
currentConfig.EnsRoot = common.HexToAddress(ensaddr)
}
if cors := ctx.GlobalString(CorsStringFlag.Name); cors != "" {
currentConfig.Cors = cors
}
......@@ -221,6 +232,18 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
currentConfig.BootNodes = ctx.GlobalString(utils.BootnodesFlag.Name)
}
if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" {
currentConfig.LocalStoreParams.ChunkDbPath = storePath
}
if storeCapacity := ctx.GlobalUint64(SwarmStoreCapacity.Name); storeCapacity != 0 {
currentConfig.LocalStoreParams.DbCapacity = storeCapacity
}
if storeCacheCapacity := ctx.GlobalUint(SwarmStoreCacheCapacity.Name); storeCacheCapacity != 0 {
currentConfig.LocalStoreParams.CacheCapacity = storeCacheCapacity
}
return currentConfig
}
......@@ -239,7 +262,7 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" {
if id, _ := strconv.Atoi(networkid); id != 0 {
currentConfig.NetworkId = uint64(id)
currentConfig.NetworkID = uint64(id)
}
}
......@@ -262,17 +285,29 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
}
}
if syncenable := os.Getenv(SWARM_ENV_SYNC_ENABLE); syncenable != "" {
if sync, err := strconv.ParseBool(syncenable); err != nil {
currentConfig.SyncEnabled = sync
if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" {
if sync, err := strconv.ParseBool(syncdisable); err != nil {
currentConfig.SyncEnabled = !sync
}
}
if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" {
if skipCheck, err := strconv.ParseBool(v); err != nil {
currentConfig.DeliverySkipCheck = skipCheck
}
}
if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" {
if d, err := time.ParseDuration(v); err != nil {
currentConfig.SyncUpdateDelay = d
}
}
if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {
currentConfig.SwapApi = swapapi
currentConfig.SwapAPI = swapapi
}
if currentConfig.SwapEnabled && currentConfig.SwapApi == "" {
if currentConfig.SwapEnabled && currentConfig.SwapAPI == "" {
utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
}
......@@ -312,18 +347,6 @@ func dumpConfig(ctx *cli.Context) error {
return nil
}
//deprecated flags checked here
func checkDeprecated(ctx *cli.Context) {
// exit if the deprecated --ethapi flag is set
if ctx.GlobalString(DeprecatedEthAPIFlag.Name) != "" {
utils.Fatalf("--ethapi is no longer a valid command line flag, please use --ens-api and/or --swap-api.")
}
// warn if --ens-api flag is set
if ctx.GlobalString(DeprecatedEnsAddrFlag.Name) != "" {
log.Warn("--ens-addr is no longer a valid command line flag, please use --ens-api to specify contract address.")
}
}
//validate configuration parameters
func validateConfig(cfg *bzzapi.Config) (err error) {
for _, ensAPI := range cfg.EnsAPIs {
......
cmd/swarm/config_test.go
View file @ eaff8929
This diff is collapsed.
cmd/swarm/db.go
View file @ eaff8929
......@@ -23,6 +23,7 @@ import (
"path/filepath"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/storage"
"gopkg.in/urfave/cli.v1"
......@@ -30,11 +31,11 @@ import (
func dbExport(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 2 {
utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database) and <file> (path to write the tar archive to, - for stdout)")
if len(args) != 3 {
utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database), <file> (path to write the tar archive to, - for stdout) and the base key")
}
store, err := openDbStore(args[0])
store, err := openLDBStore(args[0], common.Hex2Bytes(args[2]))
if err != nil {
utils.Fatalf("error opening local chunk database: %s", err)
}
......@@ -62,11 +63,11 @@ func dbExport(ctx *cli.Context) {
func dbImport(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 2 {
utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database) and <file> (path to read the tar archive from, - for stdin)")
if len(args) != 3 {
utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database), <file> (path to read the tar archive from, - for stdin) and the base key")
}
store, err := openDbStore(args[0])
store, err := openLDBStore(args[0], common.Hex2Bytes(args[2]))
if err != nil {
utils.Fatalf("error opening local chunk database: %s", err)
}
......@@ -94,11 +95,11 @@ func dbImport(ctx *cli.Context) {
func dbClean(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database)")
if len(args) != 2 {
utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database) and the base key")
}
store, err := openDbStore(args[0])
store, err := openLDBStore(args[0], common.Hex2Bytes(args[1]))
if err != nil {
utils.Fatalf("error opening local chunk database: %s", err)
}
......@@ -107,10 +108,13 @@ func dbClean(ctx *cli.Context) {
store.Cleanup()
}
func openDbStore(path string) (*storage.DbStore, error) {
func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) {
if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
return nil, fmt.Errorf("invalid chunkdb path: %s", err)
}
hash := storage.MakeHashFunc("SHA3")
return storage.NewDbStore(path, hash, 10000000, 0)
storeparams := storage.NewDefaultStoreParams()
ldbparams := storage.NewLDBStoreParams(storeparams, path)
ldbparams.BaseKey = basekey
return storage.NewLDBStore(ldbparams)
}
cmd/swarm/download.go 0 → 100644
View file @ eaff8929
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
func download(ctx *cli.Context) {
log.Debug("downloading content using swarm down")
args := ctx.Args()
dest := "."
switch len(args) {
case 0:
utils.Fatalf("Usage: swarm down [options] <bzz locator> [<destination path>]")
case 1:
log.Trace(fmt.Sprintf("swarm down: no destination path - assuming working dir"))
default:
log.Trace(fmt.Sprintf("destination path arg: %s", args[1]))
if absDest, err := filepath.Abs(args[1]); err == nil {
dest = absDest
} else {
utils.Fatalf("could not get download path: %v", err)
}
}
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
isRecursive = ctx.Bool(SwarmRecursiveFlag.Name)
client = swarm.NewClient(bzzapi)
)
if fi, err := os.Stat(dest); err == nil {
if isRecursive && !fi.Mode().IsDir() {
utils.Fatalf("destination path is not a directory!")
}
} else {
if !os.IsNotExist(err) {
utils.Fatalf("could not stat path: %v", err)
}
}
uri, err := api.Parse(args[0])
if err != nil {
utils.Fatalf("could not parse uri argument: %v", err)
}
// assume behaviour according to --recursive switch
if isRecursive {
if err := client.DownloadDirectory(uri.Addr, uri.Path, dest); err != nil {
utils.Fatalf("encoutered an error while downloading directory: %v", err)
}
} else {
// we are downloading a file
log.Debug(fmt.Sprintf("downloading file/path from a manifest. hash: %s, path:%s", uri.Addr, uri.Path))
err := client.DownloadFile(uri.Addr, uri.Path, dest)
if err != nil {
utils.Fatalf("could not download %s from given address: %s. error: %v", uri.Path, uri.Addr, err)
}
}
}
cmd/swarm/export_test.go 0 → 100644
View file @ eaff8929
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"crypto/md5"
"crypto/rand"
"io"
"io/ioutil"
"net/http"
"os"
"strings"
"testing"
"github.com/ethereum/go-ethereum/swarm"
)
// TestCLISwarmExportImport perform the following test:
// 1. runs swarm node
// 2. uploads a random file
// 3. runs an export of the local datastore
// 4. runs a second swarm node
// 5. imports the exported datastore
// 6. fetches the uploaded random file from the second node
func TestCLISwarmExportImport(t *testing.T) {
cluster := newTestCluster(t, 1)
// generate random 10mb file
f, cleanup := generateRandomFile(t, 10000000)
defer cleanup()
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name())
_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
up.ExpectExit()
hash := matches[0]
var info swarm.Info
if err := cluster.Nodes[0].Client.Call(&info, "bzz_info"); err != nil {
t.Fatal(err)
}
cluster.Stop()
defer cluster.Cleanup()
// generate an export.tar
exportCmd := runSwarm(t, "db", "export", info.Path+"/chunks", info.Path+"/export.tar", strings.TrimPrefix(info.BzzKey, "0x"))
exportCmd.ExpectExit()
// start second cluster
cluster2 := newTestCluster(t, 1)
var info2 swarm.Info
if err := cluster2.Nodes[0].Client.Call(&info2, "bzz_info"); err != nil {
t.Fatal(err)
}
// stop second cluster, so that we close LevelDB
cluster2.Stop()
defer cluster2.Cleanup()
// import the export.tar
importCmd := runSwarm(t, "db", "import", info2.Path+"/chunks", info.Path+"/export.tar", strings.TrimPrefix(info2.BzzKey, "0x"))
importCmd.ExpectExit()
// spin second cluster back up
cluster2.StartExistingNodes(t, 1, strings.TrimPrefix(info2.BzzAccount, "0x"))
// try to fetch imported file
res, err := http.Get(cluster2.Nodes[0].URL + "/bzz:/" + hash)
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 200 {
t.Fatalf("expected HTTP status %d, got %s", 200, res.Status)
}
// compare downloaded file with the generated random file
mustEqualFiles(t, f, res.Body)
}
func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
h := md5.New()
upLen, err := io.Copy(h, up)
if err != nil {
t.Fatal(err)
}
upHash := h.Sum(nil)
h.Reset()
downLen, err := io.Copy(h, down)
if err != nil {
t.Fatal(err)
}
downHash := h.Sum(nil)
if !bytes.Equal(upHash, downHash) || upLen != downLen {
t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
}
}
func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) {
// create a tmp file
tmp, err := ioutil.TempFile("", "swarm-test")
if err != nil {
t.Fatal(err)
}
// callback for tmp file cleanup
teardown = func() {
tmp.Close()
os.Remove(tmp.Name())
}
// write 10mb random data to file
buf := make([]byte, 10000000)
_, err = rand.Read(buf)
if err != nil {
t.Fatal(err)
}
ioutil.WriteFile(tmp.Name(), buf, 0755)
return tmp, teardown
}
cmd/swarm/fs.go 0 → 100644
View file @ eaff8929
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/fuse"
"gopkg.in/urfave/cli.v1"
)
func mount(cliContext *cli.Context) {
args := cliContext.Args()
if len(args) < 2 {
utils.Fatalf("Usage: swarm fs mount --ipcpath <path to bzzd.ipc> <manifestHash> <file name>")
}
client, err := dialRPC(cliContext)
if err != nil {
utils.Fatalf("had an error dailing to RPC endpoint: %v", err)
}
defer client.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
mf := &fuse.MountInfo{}
mountPoint, err := filepath.Abs(filepath.Clean(args[1]))
if err != nil {
utils.Fatalf("error expanding path for mount point: %v", err)
}
err = client.CallContext(ctx, mf, "swarmfs_mount", args[0], mountPoint)
if err != nil {
utils.Fatalf("had an error calling the RPC endpoint while mounting: %v", err)
}
}
func unmount(cliContext *cli.Context) {
args := cliContext.Args()
if len(args) < 1 {
utils.Fatalf("Usage: swarm fs unmount --ipcpath <path to bzzd.ipc> <mount path>")
}
client, err := dialRPC(cliContext)
if err != nil {
utils.Fatalf("had an error dailing to RPC endpoint: %v", err)
}
defer client.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
mf := fuse.MountInfo{}
err = client.CallContext(ctx, &mf, "swarmfs_unmount", args[0])
if err != nil {
utils.Fatalf("encountered an error calling the RPC endpoint while unmounting: %v", err)
}
fmt.Printf("%s\n", mf.LatestManifest) //print the latest manifest hash for user reference
}
func listMounts(cliContext *cli.Context) {
client, err := dialRPC(cliContext)
if err != nil {
utils.Fatalf("had an error dailing to RPC endpoint: %v", err)
}
defer client.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
mf := []fuse.MountInfo{}
err = client.CallContext(ctx, &mf, "swarmfs_listmounts")
if err != nil {
utils.Fatalf("encountered an error calling the RPC endpoint while unmounting: %v", err)
}
if len(mf) == 0 {
fmt.Print("Could not found any swarmfs mounts. Please make sure you've specified the correct RPC endpoint\n")
} else {
fmt.Printf("Found %d swarmfs mount(s):\n", len(mf))
for i, mountInfo := range mf {
fmt.Printf("%d:\n", i)
fmt.Printf("\tMount point: %s\n", mountInfo.MountPoint)
fmt.Printf("\tLatest Manifest: %s\n", mountInfo.LatestManifest)
fmt.Printf("\tStart Manifest: %s\n", mountInfo.StartManifest)
}
}
}
func dialRPC(ctx *cli.Context) (*rpc.Client, error) {
var endpoint string
if ctx.IsSet(utils.IPCPathFlag.Name) {
endpoint = ctx.String(utils.IPCPathFlag.Name)
} else {
utils.Fatalf("swarm ipc endpoint not specified")
}
if endpoint == "" {
endpoint = node.DefaultIPCEndpoint(clientIdentifier)
} else if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
// Backwards compatibility with geth < 1.5 which required
// these prefixes.
endpoint = endpoint[4:]
}
return rpc.Dial(endpoint)
}
cmd/swarm/fs_test.go 0 → 100644
View file @ eaff8929
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/ethereum/go-ethereum/log"
colorable "github.com/mattn/go-colorable"
)
func init() {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
}
type testFile struct {
filePath string
content string
}
// TestCLISwarmFs is a high-level test of swarmfs
func TestCLISwarmFs(t *testing.T) {
cluster := newTestCluster(t, 3)
defer cluster.Shutdown()
// create a tmp dir
mountPoint, err := ioutil.TempDir("", "swarm-test")
log.Debug("swarmfs cli test", "1st mount", mountPoint)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(mountPoint)
handlingNode := cluster.Nodes[0]
mhash := doUploadEmptyDir(t, handlingNode)
log.Debug("swarmfs cli test: mounting first run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
mount := runSwarm(t, []string{
"fs",
"mount",
"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
mhash,
mountPoint,
}...)
mount.ExpectExit()
filesToAssert := []*testFile{}
dirPath, err := createDirInDir(mountPoint, "testSubDir")
if err != nil {
t.Fatal(err)
}
dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir")
dummyContent := "somerandomtestcontentthatshouldbeasserted"
dirs := []string{
mountPoint,
dirPath,
dirPath2,
}
files := []string{"f1.tmp", "f2.tmp"}
for _, d := range dirs {
for _, entry := range files {
tFile, err := createTestFileInPath(d, entry, dummyContent)
if err != nil {
t.Fatal(err)
}
filesToAssert = append(filesToAssert, tFile)
}
}
if len(filesToAssert) != len(dirs)*len(files) {
t.Fatalf("should have %d files to assert now, got %d", len(dirs)*len(files), len(filesToAssert))
}
hashRegexp := `[a-f\d]{64}`
log.Debug("swarmfs cli test: unmounting first run...", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
unmount := runSwarm(t, []string{
"fs",
"unmount",
"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
mountPoint,
}...)
_, matches := unmount.ExpectRegexp(hashRegexp)
unmount.ExpectExit()
hash := matches[0]
if hash == mhash {
t.Fatal("this should not be equal")
}
log.Debug("swarmfs cli test: asserting no files in mount point")
//check that there's nothing in the mount folder
filesInDir, err := ioutil.ReadDir(mountPoint)
if err != nil {
t.Fatalf("had an error reading the directory: %v", err)
}
if len(filesInDir) != 0 {
t.Fatal("there shouldn't be anything here")
}
secondMountPoint, err := ioutil.TempDir("", "swarm-test")
log.Debug("swarmfs cli test", "2nd mount point at", secondMountPoint)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(secondMountPoint)
log.Debug("swarmfs cli test: remounting at second mount point", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
//remount, check files
newMount := runSwarm(t, []string{
"fs",
"mount",
"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
hash, // the latest hash
secondMountPoint,
}...)
newMount.ExpectExit()
time.Sleep(1 * time.Second)
filesInDir, err = ioutil.ReadDir(secondMountPoint)
if err != nil {
t.Fatal(err)
}
if len(filesInDir) == 0 {
t.Fatal("there should be something here")
}
log.Debug("swarmfs cli test: traversing file tree to see it matches previous mount")
for _, file := range filesToAssert {
file.filePath = strings.Replace(file.filePath, mountPoint, secondMountPoint, -1)
fileBytes, err := ioutil.ReadFile(file.filePath)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(fileBytes, bytes.NewBufferString(file.content).Bytes()) {
t.Fatal("this should be equal")
}
}
log.Debug("swarmfs cli test: unmounting second run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
unmountSec := runSwarm(t, []string{
"fs",
"unmount",
"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
secondMountPoint,
}...)
_, matches = unmountSec.ExpectRegexp(hashRegexp)
unmountSec.ExpectExit()
if matches[0] != hash {
t.Fatal("these should be equal - no changes made")
}
}
func doUploadEmptyDir(t *testing.T, node *testNode) string {
// create a tmp dir
tmpDir, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
hashRegexp := `[a-f\d]{64}`
flags := []string{
"--bzzapi", node.URL,
"--recursive",
"up",
tmpDir}
log.Info("swarmfs cli test: uploading dir with 'swarm up'")
up := runSwarm(t, flags...)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
hash := matches[0]
log.Info("swarmfs cli test: dir uploaded", "hash", hash)
return hash
}
func createDirInDir(createInDir string, dirToCreate string) (string, error) {
fullpath := filepath.Join(createInDir, dirToCreate)
err := os.MkdirAll(fullpath, 0777)
if err != nil {
return "", err
}
return fullpath, nil
}
func createTestFileInPath(dir, filename, content string) (*testFile, error) {
tFile := &testFile{}
filePath := filepath.Join(dir, filename)
if file, err := os.Create(filePath); err == nil {
tFile.content = content
tFile.filePath = filePath
_, err = io.WriteString(file, content)
if err != nil {
return nil, err
}
file.Close()
}
return tFile, nil
}
cmd/swarm/hash.go
View file @ eaff8929
......@@ -38,11 +38,11 @@ func hash(ctx *cli.Context) {
defer f.Close()
stat, _ := f.Stat()
chunker := storage.NewTreeChunker(storage.NewChunkerParams())
key, err := chunker.Split(f, stat.Size(), nil, nil, nil)
fileStore := storage.NewFileStore(storage.NewMapChunkStore(), storage.NewFileStoreParams())
addr, _, err := fileStore.Store(f, stat.Size(), false)
if err != nil {
utils.Fatalf("%v\n", err)
} else {
fmt.Printf("%v\n", key)
fmt.Printf("%v\n", addr)
}
}
cmd/swarm/main.go
View file @ eaff8929
This diff is collapsed.
cmd/swarm/manifest.go
View file @ eaff8929
......@@ -131,13 +131,13 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
longestPathEntry = api.ManifestEntry{}
)
mroot, err := client.DownloadManifest(mhash)
mroot, isEncrypted, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
//TODO: check if the "hash" to add is valid and present in swarm
_, err = client.DownloadManifest(hash)
_, _, err = client.DownloadManifest(hash)
if err != nil {
utils.Fatalf("Hash to add is not present: %v", err)
}
......@@ -180,7 +180,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot.Entries = append(mroot.Entries, newEntry)
}
newManifestHash, err := client.UploadManifest(mroot)
newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
......@@ -197,7 +197,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
longestPathEntry = api.ManifestEntry{}
)
mroot, err := client.DownloadManifest(mhash)
mroot, isEncrypted, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
......@@ -257,7 +257,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
mroot = newMRoot
}
newManifestHash, err := client.UploadManifest(mroot)
newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
......@@ -273,7 +273,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
longestPathEntry = api.ManifestEntry{}
)
mroot, err := client.DownloadManifest(mhash)
mroot, isEncrypted, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
......@@ -323,7 +323,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
mroot = newMRoot
}
newManifestHash, err := client.UploadManifest(mroot)
newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
......
cmd/swarm/run_test.go
View file @ eaff8929
......@@ -81,6 +81,7 @@ type testCluster struct {
//
// When starting more than one node, they are connected together using the
// admin SetPeer RPC method.
func newTestCluster(t *testing.T, size int) *testCluster {
cluster := &testCluster{}
defer func() {
......@@ -96,18 +97,7 @@ func newTestCluster(t *testing.T, size int) *testCluster {
cluster.TmpDir = tmpdir
// start the nodes
cluster.Nodes = make([]*testNode, 0, size)
for i := 0; i < size; i++ {
dir := filepath.Join(cluster.TmpDir, fmt.Sprintf("swarm%02d", i))
if err := os.Mkdir(dir, 0700); err != nil {
t.Fatal(err)
}
node := newTestNode(t, dir)
node.Name = fmt.Sprintf("swarm%02d", i)
cluster.Nodes = append(cluster.Nodes, node)
}
cluster.StartNewNodes(t, size)
if size == 1 {
return cluster
......@@ -145,14 +135,51 @@ func (c *testCluster) Shutdown() {
os.RemoveAll(c.TmpDir)
}
func (c *testCluster) Stop() {
for _, node := range c.Nodes {
node.Shutdown()
}
}
func (c *testCluster) StartNewNodes(t *testing.T, size int) {
c.Nodes = make([]*testNode, 0, size)
for i := 0; i < size; i++ {
dir := filepath.Join(c.TmpDir, fmt.Sprintf("swarm%02d", i))
if err := os.Mkdir(dir, 0700); err != nil {
t.Fatal(err)
}
node := newTestNode(t, dir)
node.Name = fmt.Sprintf("swarm%02d", i)
c.Nodes = append(c.Nodes, node)
}
}
func (c *testCluster) StartExistingNodes(t *testing.T, size int, bzzaccount string) {
c.Nodes = make([]*testNode, 0, size)
for i := 0; i < size; i++ {
dir := filepath.Join(c.TmpDir, fmt.Sprintf("swarm%02d", i))
node := existingTestNode(t, dir, bzzaccount)
node.Name = fmt.Sprintf("swarm%02d", i)
c.Nodes = append(c.Nodes, node)
}
}
func (c *testCluster) Cleanup() {
os.RemoveAll(c.TmpDir)
}
type testNode struct {
Name string
Addr string
URL string
Enode string
Dir string
Client *rpc.Client
Cmd *cmdtest.TestCmd
Name string
Addr string
URL string
Enode string
Dir string
IpcPath string
Client *rpc.Client
Cmd *cmdtest.TestCmd
}
const testPassphrase = "swarm-test-passphrase"
......@@ -181,6 +208,72 @@ func getTestAccount(t *testing.T, dir string) (conf *node.Config, account accoun
return conf, account
}
func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
conf, _ := getTestAccount(t, dir)
node := &testNode{Dir: dir}
// use a unique IPCPath when running tests on Windows
if runtime.GOOS == "windows" {
conf.IPCPath = fmt.Sprintf("bzzd-%s.ipc", bzzaccount)
}
// assign ports
httpPort, err := assignTCPPort()
if err != nil {
t.Fatal(err)
}
p2pPort, err := assignTCPPort()
if err != nil {
t.Fatal(err)
}
// start the node
node.Cmd = runSwarm(t,
"--port", p2pPort,
"--nodiscover",
"--datadir", dir,
"--ipcpath", conf.IPCPath,
"--ens-api", "",
"--bzzaccount", bzzaccount,
"--bzznetworkid", "321",
"--bzzport", httpPort,
"--verbosity", "6",
)
node.Cmd.InputLine(testPassphrase)
defer func() {
if t.Failed() {
node.Shutdown()
}
}()
// wait for the node to start
for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
node.Client, err = rpc.Dial(conf.IPCEndpoint())
if err == nil {
break
}
}
if node.Client == nil {
t.Fatal(err)
}
// load info
var info swarm.Info
if err := node.Client.Call(&info, "bzz_info"); err != nil {
t.Fatal(err)
}
node.Addr = net.JoinHostPort("127.0.0.1", info.Port)
node.URL = "http://" + node.Addr
var nodeInfo p2p.NodeInfo
if err := node.Client.Call(&nodeInfo, "admin_nodeInfo"); err != nil {
t.Fatal(err)
}
node.Enode = fmt.Sprintf("enode://%s@127.0.0.1:%s", nodeInfo.ID, p2pPort)
return node
}
func newTestNode(t *testing.T, dir string) *testNode {
conf, account := getTestAccount(t, dir)
......@@ -239,6 +332,7 @@ func newTestNode(t *testing.T, dir string) *testNode {
t.Fatal(err)
}
node.Enode = fmt.Sprintf("enode://%s@127.0.0.1:%s", nodeInfo.ID, p2pPort)
node.IpcPath = conf.IPCPath
return node
}
......
cmd/swarm/swarm-smoke/main.go 0 → 100644
View file @ eaff8929
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"os"
"sort"
"github.com/ethereum/go-ethereum/log"
colorable "github.com/mattn/go-colorable"
cli "gopkg.in/urfave/cli.v1"
)
var (
endpoints []string
includeLocalhost bool
cluster string
scheme string
filesize int
from int
to int
)
func main() {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
app := cli.NewApp()
app.Name = "smoke-test"
app.Usage = ""
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "cluster-endpoint",
Value: "testing",
Usage: "cluster to point to (open, or testing)",
Destination: &cluster,
},
cli.IntFlag{
Name: "cluster-from",
Value: 8501,
Usage: "swarm node (from)",
Destination: &from,
},
cli.IntFlag{
Name: "cluster-to",
Value: 8512,
Usage: "swarm node (to)",
Destination: &to,
},
cli.StringFlag{
Name: "cluster-scheme",
Value: "http",
Usage: "http or https",
Destination: &scheme,
},
cli.BoolFlag{
Name: "include-localhost",
Usage: "whether to include localhost:8500 as an endpoint",
Destination: &includeLocalhost,
},
cli.IntFlag{
Name: "filesize",
Value: 1,
Usage: "file size for generated random file in MB",
Destination: &filesize,
},
}
app.Commands = []cli.Command{
{
Name: "upload_and_sync",
Aliases: []string{"c"},
Usage: "upload and sync",
Action: cliUploadAndSync,
},
}
sort.Sort(cli.FlagsByName(app.Flags))
sort.Sort(cli.CommandsByName(app.Commands))
err := app.Run(os.Args)
if err != nil {
log.Error(err.Error())
}
}
cmd/swarm/swarm-smoke/upload_and_sync.go 0 → 100644
View file @ eaff8929
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"crypto/md5"
"crypto/rand"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"os/exec"
"strings"
"sync"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
func generateEndpoints(scheme string, cluster string, from int, to int) {
for port := from; port <= to; port++ {
endpoints = append(endpoints, fmt.Sprintf("%s://%v.%s.swarm-gateways.net", scheme, port, cluster))
}
if includeLocalhost {
endpoints = append(endpoints, "http://localhost:8500")
}
}
func cliUploadAndSync(c *cli.Context) error {
defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size", filesize) }(time.Now())
generateEndpoints(scheme, cluster, from, to)
log.Info("uploading to " + endpoints[0] + " and syncing")
f, cleanup := generateRandomFile(filesize * 1000000)
defer cleanup()
hash, err := upload(f, endpoints[0])
if err != nil {
log.Error(err.Error())
return err
}
fhash, err := digest(f)
if err != nil {
log.Error(err.Error())
return err
}
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash))
if filesize < 10 {
time.Sleep(15 * time.Second)
} else {
time.Sleep(2 * time.Duration(filesize) * time.Second)
}
wg := sync.WaitGroup{}
for _, endpoint := range endpoints {
endpoint := endpoint
ruid := uuid.New()[:8]
wg.Add(1)
go func(endpoint string, ruid string) {
for {
err := fetch(hash, endpoint, fhash, ruid)
if err != nil {
continue
}
wg.Done()
return
}
}(endpoint, ruid)
}
wg.Wait()
log.Info("all endpoints synced random file successfully")
return nil
}
// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file
func fetch(hash string, endpoint string, original []byte, ruid string) error {
log.Trace("sleeping", "ruid", ruid)
time.Sleep(1 * time.Second)
log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
res, err := http.Get(endpoint + "/bzz:/" + hash + "/")
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err
}
log.Trace("http get response", "ruid", ruid, "api", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength)
if res.StatusCode != 200 {
err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode)
log.Warn(err.Error(), "ruid", ruid)
return err
}
defer res.Body.Close()
rdigest, err := digest(res.Body)
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err
}
if !bytes.Equal(rdigest, original) {
err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
log.Warn(err.Error(), "ruid", ruid)
return err
}
log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
return nil
}
// upload is uploading a file `f` to `endpoint` via the `swarm up` cmd
func upload(f *os.File, endpoint string) (string, error) {
var out bytes.Buffer
cmd := exec.Command("swarm", "--bzzapi", endpoint, "up", f.Name())
cmd.Stdout = &out
err := cmd.Run()
if err != nil {
return "", err
}
hash := strings.TrimRight(out.String(), "\r\n")
return hash, nil
}
func digest(r io.Reader) ([]byte, error) {
h := md5.New()
_, err := io.Copy(h, r)
if err != nil {
return nil, err
}
return h.Sum(nil), nil
}
// generateRandomFile is creating a temporary file with the requested byte size
func generateRandomFile(size int) (f *os.File, teardown func()) {
// create a tmp file
tmp, err := ioutil.TempFile("", "swarm-test")
if err != nil {
panic(err)
}
// callback for tmp file cleanup
teardown = func() {
tmp.Close()
os.Remove(tmp.Name())
}
buf := make([]byte, size)
_, err = rand.Read(buf)
if err != nil {
panic(err)
}
ioutil.WriteFile(tmp.Name(), buf, 0755)
return tmp, teardown
}
cmd/swarm/upload.go
View file @ eaff8929
......@@ -40,12 +40,13 @@ func upload(ctx *cli.Context) {
args := ctx.Args()
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
recursive = ctx.GlobalBool(SwarmRecursiveUploadFlag.Name)
recursive = ctx.GlobalBool(SwarmRecursiveFlag.Name)
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name)
fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
mimeType = ctx.GlobalString(SwarmUploadMimeType.Name)
client = swarm.NewClient(bzzapi)
toEncrypt = ctx.Bool(SwarmEncryptedFlag.Name)
file string
)
......@@ -76,7 +77,7 @@ func upload(ctx *cli.Context) {
utils.Fatalf("Error opening file: %s", err)
}
defer f.Close()
hash, err := client.UploadRaw(f, f.Size)
hash, err := client.UploadRaw(f, f.Size, toEncrypt)
if err != nil {
utils.Fatalf("Upload failed: %s", err)
}
......@@ -97,7 +98,7 @@ func upload(ctx *cli.Context) {
if !recursive {
return "", errors.New("Argument is a directory and recursive upload is disabled")
}
return client.UploadDirectory(file, defaultPath, "")
return client.UploadDirectory(file, defaultPath, "", toEncrypt)
}
} else {
doUpload = func() (string, error) {
......@@ -110,7 +111,7 @@ func upload(ctx *cli.Context) {
mimeType = detectMimeType(file)
}
f.ContentType = mimeType
return client.Upload(f, "")
return client.Upload(f, "", toEncrypt)
}
}
hash, err := doUpload()
......
cmd/swarm/upload_test.go
View file @ eaff8929
......@@ -17,60 +17,259 @@
package main
import (
"bytes"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"testing"
"time"
"github.com/ethereum/go-ethereum/log"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
colorable "github.com/mattn/go-colorable"
)
var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
func init() {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
}
// TestCLISwarmUp tests that running 'swarm up' makes the resulting file
// available from all nodes via the HTTP API
func TestCLISwarmUp(t *testing.T) {
// start 3 node cluster
t.Log("starting 3 node cluster")
testCLISwarmUp(false, t)
}
func TestCLISwarmUpRecursive(t *testing.T) {
testCLISwarmUpRecursive(false, t)
}
// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file
// available from all nodes via the HTTP API
func TestCLISwarmUpEncrypted(t *testing.T) {
testCLISwarmUp(true, t)
}
func TestCLISwarmUpEncryptedRecursive(t *testing.T) {
testCLISwarmUpRecursive(true, t)
}
func testCLISwarmUp(toEncrypt bool, t *testing.T) {
log.Info("starting 3 node cluster")
cluster := newTestCluster(t, 3)
defer cluster.Shutdown()
// create a tmp file
tmp, err := ioutil.TempFile("", "swarm-test")
assertNil(t, err)
if err != nil {
t.Fatal(err)
}
defer tmp.Close()
defer os.Remove(tmp.Name())
_, err = io.WriteString(tmp, "data")
assertNil(t, err)
// write data to file
data := "notsorandomdata"
_, err = io.WriteString(tmp, data)
if err != nil {
t.Fatal(err)
}
hashRegexp := `[a-f\d]{64}`
flags := []string{
"--bzzapi", cluster.Nodes[0].URL,
"up",
tmp.Name()}
if toEncrypt {
hashRegexp = `[a-f\d]{128}`
flags = []string{
"--bzzapi", cluster.Nodes[0].URL,
"up",
"--encrypt",
tmp.Name()}
}
// upload the file with 'swarm up' and expect a hash
t.Log("uploading file with 'swarm up'")
up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", tmp.Name())
_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
up := runSwarm(t, flags...)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
hash := matches[0]
t.Logf("file uploaded with hash %s", hash)
log.Info("file uploaded", "hash", hash)
// get the file from the HTTP API of each node
for _, node := range cluster.Nodes {
t.Logf("getting file from %s", node.Name)
log.Info("getting file from node", "node", node.Name)
res, err := http.Get(node.URL + "/bzz:/" + hash)
assertNil(t, err)
assertHTTPResponse(t, res, http.StatusOK, "data")
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
reply, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 200 {
t.Fatalf("expected HTTP status 200, got %s", res.Status)
}
if string(reply) != data {
t.Fatalf("expected HTTP body %q, got %q", data, reply)
}
log.Debug("verifying uploaded file using `swarm down`")
//try to get the content with `swarm down`
tmpDownload, err := ioutil.TempDir("", "swarm-test")
tmpDownload = path.Join(tmpDownload, "tmpfile.tmp")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDownload)
bzzLocator := "bzz:/" + hash
flags = []string{
"--bzzapi", cluster.Nodes[0].URL,
"down",
bzzLocator,
tmpDownload,
}
down := runSwarm(t, flags...)
down.ExpectExit()
fi, err := os.Stat(tmpDownload)
if err != nil {
t.Fatalf("could not stat path: %v", err)
}
switch mode := fi.Mode(); {
case mode.IsRegular():
downloadedBytes, err := ioutil.ReadFile(tmpDownload)
if err != nil {
t.Fatalf("had an error reading the downloaded file: %v", err)
}
if !bytes.Equal(downloadedBytes, bytes.NewBufferString(data).Bytes()) {
t.Fatalf("retrieved data and posted data not equal!")
}
default:
t.Fatalf("expected to download regular file, got %s", fi.Mode())
}
}
timeout := time.Duration(2 * time.Second)
httpClient := http.Client{
Timeout: timeout,
}
// try to squeeze a timeout by getting an non-existent hash from each node
for _, node := range cluster.Nodes {
_, err := httpClient.Get(node.URL + "/bzz:/1023e8bae0f70be7d7b5f74343088ba408a218254391490c85ae16278e230340")
// we're speeding up the timeout here since netstore has a 60 seconds timeout on a request
if err != nil && !strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
t.Fatal(err)
}
// this is disabled since it takes 60s due to netstore timeout
// if res.StatusCode != 404 {
// t.Fatalf("expected HTTP status 404, got %s", res.Status)
// }
}
}
func assertNil(t *testing.T, err error) {
func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
fmt.Println("starting 3 node cluster")
cluster := newTestCluster(t, 3)
defer cluster.Shutdown()
tmpUploadDir, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
}
defer os.RemoveAll(tmpUploadDir)
// create tmp files
data := "notsorandomdata"
for _, path := range []string{"tmp1", "tmp2"} {
if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil {
t.Fatal(err)
}
}
func assertHTTPResponse(t *testing.T, res *http.Response, expectedStatus int, expectedBody string) {
defer res.Body.Close()
if res.StatusCode != expectedStatus {
t.Fatalf("expected HTTP status %d, got %s", expectedStatus, res.Status)
hashRegexp := `[a-f\d]{64}`
flags := []string{
"--bzzapi", cluster.Nodes[0].URL,
"--recursive",
"up",
tmpUploadDir}
if toEncrypt {
hashRegexp = `[a-f\d]{128}`
flags = []string{
"--bzzapi", cluster.Nodes[0].URL,
"--recursive",
"up",
"--encrypt",
tmpUploadDir}
}
data, err := ioutil.ReadAll(res.Body)
assertNil(t, err)
if string(data) != expectedBody {
t.Fatalf("expected HTTP body %q, got %q", expectedBody, data)
// upload the file with 'swarm up' and expect a hash
log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
up := runSwarm(t, flags...)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
hash := matches[0]
log.Info("dir uploaded", "hash", hash)
// get the file from the HTTP API of each node
for _, node := range cluster.Nodes {
log.Info("getting file from node", "node", node.Name)
//try to get the content with `swarm down`
tmpDownload, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDownload)
bzzLocator := "bzz:/" + hash
flagss := []string{}
flagss = []string{
"--bzzapi", cluster.Nodes[0].URL,
"down",
"--recursive",
bzzLocator,
tmpDownload,
}
fmt.Println("downloading from swarm with recursive")
down := runSwarm(t, flagss...)
down.ExpectExit()
files, err := ioutil.ReadDir(tmpDownload)
for _, v := range files {
fi, err := os.Stat(path.Join(tmpDownload, v.Name()))
if err != nil {
t.Fatalf("got an error: %v", err)
}
switch mode := fi.Mode(); {
case mode.IsRegular():
if file, err := swarm.Open(path.Join(tmpDownload, v.Name())); err != nil {
t.Fatalf("encountered an error opening the file returned from the CLI: %v", err)
} else {
ff := make([]byte, len(data))
io.ReadFull(file, ff)
buf := bytes.NewBufferString(data)
if !bytes.Equal(ff, buf.Bytes()) {
t.Fatalf("retrieved data and posted data not equal!")
}
}
default:
t.Fatalf("this shouldnt happen")
}
}
if err != nil {
t.Fatalf("could not list files at: %v", files)
}
}
}
p2p/metrics.go
View file @ eaff8929
......@@ -31,10 +31,10 @@ var (
egressTrafficMeter = metrics.NewRegisteredMeter("p2p/OutboundTraffic", nil)
)
// meteredConn is a wrapper around a network TCP connection that meters both the
// meteredConn is a wrapper around a net.Conn that meters both the
// inbound and outbound network traffic.
type meteredConn struct {
*net.TCPConn // Network connection to wrap with metering
net.Conn // Network connection to wrap with metering
}
// newMeteredConn creates a new metered connection, also bumping the ingress or
......@@ -51,13 +51,13 @@ func newMeteredConn(conn net.Conn, ingress bool) net.Conn {
} else {
egressConnectMeter.Mark(1)
}
return &meteredConn{conn.(*net.TCPConn)}
return &meteredConn{Conn: conn}
}
// Read delegates a network read to the underlying connection, bumping the ingress
// traffic meter along the way.
func (c *meteredConn) Read(b []byte) (n int, err error) {
n, err = c.TCPConn.Read(b)
n, err = c.Conn.Read(b)
ingressTrafficMeter.Mark(int64(n))
return
}
......@@ -65,7 +65,7 @@ func (c *meteredConn) Read(b []byte) (n int, err error) {
// Write delegates a network write to the underlying connection, bumping the
// egress traffic meter along the way.
func (c *meteredConn) Write(b []byte) (n int, err error) {
n, err = c.TCPConn.Write(b)
n, err = c.Conn.Write(b)
egressTrafficMeter.Mark(int64(n))
return
}
p2p/peer.go
View file @ eaff8929
......@@ -17,6 +17,7 @@
package p2p
import (
"errors"
"fmt"
"io"
"net"
......@@ -31,6 +32,10 @@ import (
"github.com/ethereum/go-ethereum/rlp"
)
var (
ErrShuttingDown = errors.New("shutting down")
)
const (
baseProtocolVersion = 5
baseProtocolLength = uint64(16)
......@@ -393,7 +398,7 @@ func (rw *protoRW) WriteMsg(msg Msg) (err error) {
// as well but we don't want to rely on that.
rw.werr <- err
case <-rw.closed:
err = fmt.Errorf("shutting down")
err = ErrShuttingDown
}
return err
}
......
p2p/protocols/protocol.go
View file @ eaff8929
......@@ -31,10 +31,12 @@ package protocols
import (
"context"
"fmt"
"io"
"reflect"
"sync"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p"
)
......@@ -202,6 +204,11 @@ func NewPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *Spec) *Peer {
func (p *Peer) Run(handler func(msg interface{}) error) error {
for {
if err := p.handleIncoming(handler); err != nil {
if err != io.EOF {
metrics.GetOrRegisterCounter("peer.handleincoming.error", nil).Inc(1)
log.Error("peer.handleIncoming", "err", err)
}
return err
}
}
......
p2p/simulations/network.go
View file @ eaff8929
......@@ -31,7 +31,7 @@ import (
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
)
var dialBanTimeout = 200 * time.Millisecond
var DialBanTimeout = 200 * time.Millisecond
// NetworkConfig defines configuration options for starting a Network
type NetworkConfig struct {
......@@ -78,41 +78,25 @@ func (net *Network) Events() *event.Feed {
return &net.events
}
// NewNode adds a new node to the network with a random ID
func (net *Network) NewNode() (*Node, error) {
conf := adapters.RandomNodeConfig()
conf.Services = []string{net.DefaultService}
return net.NewNodeWithConfig(conf)
}
// NewNodeWithConfig adds a new node to the network with the given config,
// returning an error if a node with the same ID or name already exists
func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) {
net.lock.Lock()
defer net.lock.Unlock()
// create a random ID and PrivateKey if not set
if conf.ID == (discover.NodeID{}) {
c := adapters.RandomNodeConfig()
conf.ID = c.ID
conf.PrivateKey = c.PrivateKey
}
id := conf.ID
if conf.Reachable == nil {
conf.Reachable = func(otherID discover.NodeID) bool {
_, err := net.InitConn(conf.ID, otherID)
return err == nil
if err != nil && bytes.Compare(conf.ID.Bytes(), otherID.Bytes()) < 0 {
return false
}
return true
}
}
// assign a name to the node if not set
if conf.Name == "" {
conf.Name = fmt.Sprintf("node%02d", len(net.Nodes)+1)
}
// check the node doesn't already exist
if node := net.getNode(id); node != nil {
return nil, fmt.Errorf("node with ID %q already exists", id)
if node := net.getNode(conf.ID); node != nil {
return nil, fmt.Errorf("node with ID %q already exists", conf.ID)
}
if node := net.getNodeByName(conf.Name); node != nil {
return nil, fmt.Errorf("node with name %q already exists", conf.Name)
......@@ -132,8 +116,8 @@ func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error)
Node: adapterNode,
Config: conf,
}
log.Trace(fmt.Sprintf("node %v created", id))
net.nodeMap[id] = len(net.Nodes)
log.Trace(fmt.Sprintf("node %v created", conf.ID))
net.nodeMap[conf.ID] = len(net.Nodes)
net.Nodes = append(net.Nodes, node)
// emit a "control" event
......@@ -181,7 +165,9 @@ func (net *Network) Start(id discover.NodeID) error {
// startWithSnapshots starts the node with the given ID using the give
// snapshots
func (net *Network) startWithSnapshots(id discover.NodeID, snapshots map[string][]byte) error {
node := net.GetNode(id)
net.lock.Lock()
defer net.lock.Unlock()
node := net.getNode(id)
if node == nil {
return fmt.Errorf("node %v does not exist", id)
}
......@@ -220,9 +206,13 @@ func (net *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEve
// assume the node is now down
net.lock.Lock()
defer net.lock.Unlock()
node := net.getNode(id)
if node == nil {
log.Error("Can not find node for id", "id", id)
return
}
node.Up = false
net.lock.Unlock()
net.events.Send(NewEvent(node))
}()
for {
......@@ -259,7 +249,9 @@ func (net *Network) watchPeerEvents(id discover.NodeID, events chan *p2p.PeerEve
// Stop stops the node with the given ID
func (net *Network) Stop(id discover.NodeID) error {
node := net.GetNode(id)
net.lock.Lock()
defer net.lock.Unlock()
node := net.getNode(id)
if node == nil {
return fmt.Errorf("node %v does not exist", id)
}
......@@ -312,7 +304,9 @@ func (net *Network) Disconnect(oneID, otherID discover.NodeID) error {
// DidConnect tracks the fact that the "one" node connected to the "other" node
func (net *Network) DidConnect(one, other discover.NodeID) error {
conn, err := net.GetOrCreateConn(one, other)
net.lock.Lock()
defer net.lock.Unlock()
conn, err := net.getOrCreateConn(one, other)
if err != nil {
return fmt.Errorf("connection between %v and %v does not exist", one, other)
}
......@@ -327,7 +321,9 @@ func (net *Network) DidConnect(one, other discover.NodeID) error {
// DidDisconnect tracks the fact that the "one" node disconnected from the
// "other" node
func (net *Network) DidDisconnect(one, other discover.NodeID) error {
conn := net.GetConn(one, other)
net.lock.Lock()
defer net.lock.Unlock()
conn := net.getConn(one, other)
if conn == nil {
return fmt.Errorf("connection between %v and %v does not exist", one, other)
}
......@@ -335,7 +331,7 @@ func (net *Network) DidDisconnect(one, other discover.NodeID) error {
return fmt.Errorf("%v and %v already disconnected", one, other)
}
conn.Up = false
conn.initiated = time.Now().Add(-dialBanTimeout)
conn.initiated = time.Now().Add(-DialBanTimeout)
net.events.Send(NewEvent(conn))
return nil
}
......@@ -476,16 +472,19 @@ func (net *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) {
if err != nil {
return nil, err
}
if time.Since(conn.initiated) < dialBanTimeout {
return nil, fmt.Errorf("connection between %v and %v recently attempted", oneID, otherID)
}
if conn.Up {
return nil, fmt.Errorf("%v and %v already connected", oneID, otherID)
}
if time.Since(conn.initiated) < DialBanTimeout {
return nil, fmt.Errorf("connection between %v and %v recently attempted", oneID, otherID)
}
err = conn.nodesUp()
if err != nil {
log.Trace(fmt.Sprintf("nodes not up: %v", err))
return nil, fmt.Errorf("nodes not up: %v", err)
}
log.Debug("InitConn - connection initiated")
conn.initiated = time.Now()
return conn, nil
}
......
p2p/testing/protocolsession.go
View file @ eaff8929
......@@ -91,7 +91,9 @@ func (s *ProtocolSession) trigger(trig Trigger) error {
errc := make(chan error)
go func() {
log.Trace(fmt.Sprintf("trigger %v (%v)....", trig.Msg, trig.Code))
errc <- mockNode.Trigger(&trig)
log.Trace(fmt.Sprintf("triggered %v (%v)", trig.Msg, trig.Code))
}()
t := trig.Timeout
......
swarm/AUTHORS 0 → 100644
View file @ eaff8929
# Core team members
Viktor Trón - @zelig
Louis Holbrook - @nolash
Lewis Marshall - @lmars
Anton Evangelatov - @nonsense
Janoš Guljaš - @janos
Balint Gabor - @gbalint
Elad Nachmias - @justelad
Daniel A. Nagy - @nagydani
Aron Fischer - @homotopycolimit
Fabio Barone - @holisticode
Zahoor Mohamed - @jmozah
Zsolt Felföldi - @zsfelfoldi
# External contributors
Kiel Barry
Gary Rong
Jared Wasinger
Leon Stanko
Javier Peletier [epiclabs.io]
Bartek Borkowski [tungsten-labs.com]
Shane Howley [mainframe.com]
Doug Leonard [mainframe.com]
Ivan Daniluk [status.im]
Felix Lange [EF]
Martin Holst Swende [EF]
Guillaume Ballet [EF]
ligi [EF]
Christopher Dro [blick-labs.com]
Sergii Bomko [ledgerleopard.com]
Domino Valdano
Rafael Matias
Coogan Brennan
\ No newline at end of file
swarm/OWNERS 0 → 100644
View file @ eaff8929
# Ownership by go packages
swarm
├── api ─────────────────── ethersphere
├── bmt ─────────────────── @zelig
├── dev ─────────────────── @lmars
├── fuse ────────────────── @jmozah, @holisticode
├── grafana_dashboards ──── @nonsense
├── metrics ─────────────── @nonsense, @holisticode
├── multihash ───────────── @nolash
├── network ─────────────── ethersphere
│ ├── bitvector ───────── @zelig, @janos, @gbalint
│ ├── priorityqueue ───── @zelig, @janos, @gbalint
│ ├── simulations ─────── @zelig
│ └── stream ──────────── @janos, @zelig, @gbalint, @holisticode, @justelad
│ ├── intervals ───── @janos
│ └── testing ─────── @zelig
├── pot ─────────────────── @zelig
├── pss ─────────────────── @nolash, @zelig, @nonsense
├── services ────────────── @zelig
├── state ───────────────── @justelad
├── storage ─────────────── ethersphere
│ ├── encryption ──────── @gbalint, @zelig, @nagydani
│ ├── mock ────────────── @janos
│ └── mru ─────────────── @nolash
└── testutil ────────────── @lmars
\ No newline at end of file
swarm/api/api.go
View file @ eaff8929
This diff is collapsed.
swarm/api/api_test.go
View file @ eaff8929
......@@ -17,33 +17,34 @@
package api
import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"math/big"
"os"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/storage"
)
func testApi(t *testing.T, f func(*Api)) {
func testAPI(t *testing.T, f func(*API, bool)) {
datadir, err := ioutil.TempDir("", "bzz-test")
if err != nil {
t.Fatalf("unable to create temp dir: %v", err)
}
os.RemoveAll(datadir)
defer os.RemoveAll(datadir)
dpa, err := storage.NewLocalDPA(datadir)
fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32))
if err != nil {
return
}
api := NewApi(dpa, nil)
dpa.Start()
f(api)
dpa.Stop()
api := NewAPI(fileStore, nil, nil)
f(api, false)
f(api, true)
}
type testResponse struct {
......@@ -82,10 +83,9 @@ func expResponse(content string, mimeType string, status int) *Response {
return &Response{mimeType, status, int64(len(content)), content}
}
// func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse {
key := storage.Key(common.Hex2Bytes(bzzhash))
reader, mimeType, status, err := api.Get(key, path)
func testGet(t *testing.T, api *API, bzzhash, path string) *testResponse {
addr := storage.Address(common.Hex2Bytes(bzzhash))
reader, mimeType, status, _, err := api.Get(addr, path)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
......@@ -106,27 +106,28 @@ func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse {
}
func TestApiPut(t *testing.T) {
testApi(t, func(api *Api) {
testAPI(t, func(api *API, toEncrypt bool) {
content := "hello"
exp := expResponse(content, "text/plain", 0)
// exp := expResponse([]byte(content), "text/plain", 0)
key, err := api.Put(content, exp.MimeType)
addr, wait, err := api.Put(content, exp.MimeType, toEncrypt)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
resp := testGet(t, api, key.String(), "")
wait()
resp := testGet(t, api, addr.Hex(), "")
checkResponse(t, resp, exp)
})
}
// testResolver implements the Resolver interface and either returns the given
// hash if it is set, or returns a "name not found" error
type testResolver struct {
type testResolveValidator struct {
hash *common.Hash
}
func newTestResolver(addr string) *testResolver {
r := &testResolver{}
func newTestResolveValidator(addr string) *testResolveValidator {
r := &testResolveValidator{}
if addr != "" {
hash := common.HexToHash(addr)
r.hash = &hash
......@@ -134,21 +135,28 @@ func newTestResolver(addr string) *testResolver {
return r
}
func (t *testResolver) Resolve(addr string) (common.Hash, error) {
func (t *testResolveValidator) Resolve(addr string) (common.Hash, error) {
if t.hash == nil {
return common.Hash{}, fmt.Errorf("DNS name not found: %q", addr)
}
return *t.hash, nil
}
func (t *testResolveValidator) Owner(node [32]byte) (addr common.Address, err error) {
return
}
func (t *testResolveValidator) HeaderByNumber(context.Context, *big.Int) (header *types.Header, err error) {
return
}
// TestAPIResolve tests resolving URIs which can either contain content hashes
// or ENS names
func TestAPIResolve(t *testing.T) {
ensAddr := "swarm.eth"
hashAddr := "1111111111111111111111111111111111111111111111111111111111111111"
resolvedAddr := "2222222222222222222222222222222222222222222222222222222222222222"
doesResolve := newTestResolver(resolvedAddr)
doesntResolve := newTestResolver("")
doesResolve := newTestResolveValidator(resolvedAddr)
doesntResolve := newTestResolveValidator("")
type test struct {
desc string
......@@ -213,7 +221,7 @@ func TestAPIResolve(t *testing.T) {
}
for _, x := range tests {
t.Run(x.desc, func(t *testing.T) {
api := &Api{dns: x.dns}
api := &API{dns: x.dns}
uri := &URI{Addr: x.addr, Scheme: "bzz"}
if x.immutable {
uri.Scheme = "bzz-immutable"
......@@ -239,15 +247,15 @@ func TestAPIResolve(t *testing.T) {
}
func TestMultiResolver(t *testing.T) {
doesntResolve := newTestResolver("")
doesntResolve := newTestResolveValidator("")
ethAddr := "swarm.eth"
ethHash := "0x2222222222222222222222222222222222222222222222222222222222222222"
ethResolve := newTestResolver(ethHash)
ethResolve := newTestResolveValidator(ethHash)
testAddr := "swarm.test"
testHash := "0x1111111111111111111111111111111111111111111111111111111111111111"
testResolve := newTestResolver(testHash)
testResolve := newTestResolveValidator(testHash)
tests := []struct {
desc string
......
swarm/api/client/client.go
View file @ eaff8929
......@@ -30,6 +30,7 @@ import (
"net/textproto"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
......@@ -52,12 +53,17 @@ type Client struct {
Gateway string
}
// UploadRaw uploads raw data to swarm and returns the resulting hash
func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
// UploadRaw uploads raw data to swarm and returns the resulting hash. If toEncrypt is true it
// uploads encrypted data
func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, error) {
if size <= 0 {
return "", errors.New("data size must be greater than zero")
}
req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/", r)
addr := ""
if toEncrypt {
addr = "encrypt"
}
req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/"+addr, r)
if err != nil {
return "", err
}
......@@ -77,18 +83,20 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
return string(data), nil
}
// DownloadRaw downloads raw data from swarm
func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
// DownloadRaw downloads raw data from swarm and it returns a ReadCloser and a bool whether the
// content was encrypted
func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) {
uri := c.Gateway + "/bzz-raw:/" + hash
res, err := http.DefaultClient.Get(uri)
if err != nil {
return nil, err
return nil, false, err
}
if res.StatusCode != http.StatusOK {
res.Body.Close()
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
return nil, false, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
return res.Body, nil
isEncrypted := (res.Header.Get("X-Decrypted") == "true")
return res.Body, isEncrypted, nil
}
// File represents a file in a swarm manifest and is used for uploading and
......@@ -125,11 +133,11 @@ func Open(path string) (*File, error) {
// (if the manifest argument is non-empty) or creates a new manifest containing
// the file, returning the resulting manifest hash (the file will then be
// available at bzz:/<hash>/<path>)
func (c *Client) Upload(file *File, manifest string) (string, error) {
func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, error) {
if file.Size <= 0 {
return "", errors.New("file size must be greater than zero")
}
return c.TarUpload(manifest, &FileUploader{file})
return c.TarUpload(manifest, &FileUploader{file}, toEncrypt)
}
// Download downloads a file with the given path from the swarm manifest with
......@@ -159,14 +167,14 @@ func (c *Client) Download(hash, path string) (*File, error) {
// directory will then be available at bzz:/<hash>/path/to/file), with
// the file specified in defaultPath being uploaded to the root of the manifest
// (i.e. bzz:/<hash>/)
func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) {
func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bool) (string, error) {
stat, err := os.Stat(dir)
if err != nil {
return "", err
} else if !stat.IsDir() {
return "", fmt.Errorf("not a directory: %s", dir)
}
return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath})
return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath}, toEncrypt)
}
// DownloadDirectory downloads the files contained in a swarm manifest under
......@@ -228,27 +236,109 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error {
}
}
// DownloadFile downloads a single file into the destination directory
// if the manifest entry does not specify a file name - it will fallback
// to the hash of the file as a filename
func (c *Client) DownloadFile(hash, path, dest string) error {
hasDestinationFilename := false
if stat, err := os.Stat(dest); err == nil {
hasDestinationFilename = !stat.IsDir()
} else {
if os.IsNotExist(err) {
// does not exist - should be created
hasDestinationFilename = true
} else {
return fmt.Errorf("could not stat path: %v", err)
}
}
manifestList, err := c.List(hash, path)
if err != nil {
return fmt.Errorf("could not list manifest: %v", err)
}
switch len(manifestList.Entries) {
case 0:
return fmt.Errorf("could not find path requested at manifest address. make sure the path you've specified is correct")
case 1:
//continue
default:
return fmt.Errorf("got too many matches for this path")
}
uri := c.Gateway + "/bzz:/" + hash + "/" + path
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return err
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected HTTP status: expected 200 OK, got %d", res.StatusCode)
}
filename := ""
if hasDestinationFilename {
filename = dest
} else {
// try to assert
re := regexp.MustCompile("[^/]+$") //everything after last slash
if results := re.FindAllString(path, -1); len(results) > 0 {
filename = results[len(results)-1]
} else {
if entry := manifestList.Entries[0]; entry.Path != "" && entry.Path != "/" {
filename = entry.Path
} else {
// assume hash as name if there's nothing from the command line
filename = hash
}
}
filename = filepath.Join(dest, filename)
}
filePath, err := filepath.Abs(filename)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(filePath), 0777); err != nil {
return err
}
dst, err := os.Create(filename)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, res.Body)
return err
}
// UploadManifest uploads the given manifest to swarm
func (c *Client) UploadManifest(m *api.Manifest) (string, error) {
func (c *Client) UploadManifest(m *api.Manifest, toEncrypt bool) (string, error) {
data, err := json.Marshal(m)
if err != nil {
return "", err
}
return c.UploadRaw(bytes.NewReader(data), int64(len(data)))
return c.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt)
}
// DownloadManifest downloads a swarm manifest
func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
res, err := c.DownloadRaw(hash)
func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) {
res, isEncrypted, err := c.DownloadRaw(hash)
if err != nil {
return nil, err
return nil, isEncrypted, err
}
defer res.Close()
var manifest api.Manifest
if err := json.NewDecoder(res).Decode(&manifest); err != nil {
return nil, err
return nil, isEncrypted, err
}
return &manifest, nil
return &manifest, isEncrypted, nil
}
// List list files in a swarm manifest which have the given prefix, grouping
......@@ -350,10 +440,19 @@ type UploadFn func(file *File) error
// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
// returning the resulting manifest hash
func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) {
func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (string, error) {
reqR, reqW := io.Pipe()
defer reqR.Close()
req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
addr := hash
// If there is a hash already (a manifest), then that manifest will determine if the upload has
// to be encrypted or not. If there is no manifest then the toEncrypt parameter decides if
// there is encryption or not.
if hash == "" && toEncrypt {
// This is the built-in address for the encrypted upload endpoint
addr = "encrypt"
}
req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+addr, reqR)
if err != nil {
return "", err
}
......
swarm/api/client/client_test.go
View file @ eaff8929
......@@ -26,28 +26,43 @@ import (
"testing"
"github.com/ethereum/go-ethereum/swarm/api"
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
func serverFunc(api *api.API) testutil.TestServer {
return swarmhttp.NewServer(api)
}
// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
func TestClientUploadDownloadRaw(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
testClientUploadDownloadRaw(false, t)
}
func TestClientUploadDownloadRawEncrypted(t *testing.T) {
testClientUploadDownloadRaw(true, t)
}
func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc)
defer srv.Close()
client := NewClient(srv.URL)
// upload some raw data
data := []byte("foo123")
hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)))
hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt)
if err != nil {
t.Fatal(err)
}
// check we can download the same data
res, err := client.DownloadRaw(hash)
res, isEncrypted, err := client.DownloadRaw(hash)
if err != nil {
t.Fatal(err)
}
if isEncrypted != toEncrypt {
t.Fatalf("Expected encyption status %v got %v", toEncrypt, isEncrypted)
}
defer res.Close()
gotData, err := ioutil.ReadAll(res)
if err != nil {
......@@ -61,7 +76,15 @@ func TestClientUploadDownloadRaw(t *testing.T) {
// TestClientUploadDownloadFiles test uploading and downloading files to swarm
// manifests
func TestClientUploadDownloadFiles(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
testClientUploadDownloadFiles(false, t)
}
func TestClientUploadDownloadFilesEncrypted(t *testing.T) {
testClientUploadDownloadFiles(true, t)
}
func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc)
defer srv.Close()
client := NewClient(srv.URL)
......@@ -74,7 +97,7 @@ func TestClientUploadDownloadFiles(t *testing.T) {
Size: int64(len(data)),
},
}
hash, err := client.Upload(file, manifest)
hash, err := client.Upload(file, manifest, toEncrypt)
if err != nil {
t.Fatal(err)
}
......@@ -159,7 +182,7 @@ func newTestDirectory(t *testing.T) string {
// TestClientUploadDownloadDirectory tests uploading and downloading a
// directory of files to a swarm manifest
func TestClientUploadDownloadDirectory(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
srv := testutil.NewTestSwarmServer(t, serverFunc)
defer srv.Close()
dir := newTestDirectory(t)
......@@ -168,7 +191,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) {
// upload the directory
client := NewClient(srv.URL)
defaultPath := filepath.Join(dir, testDirFiles[0])
hash, err := client.UploadDirectory(dir, defaultPath, "")
hash, err := client.UploadDirectory(dir, defaultPath, "", false)
if err != nil {
t.Fatalf("error uploading directory: %s", err)
}
......@@ -217,14 +240,22 @@ func TestClientUploadDownloadDirectory(t *testing.T) {
// TestClientFileList tests listing files in a swarm manifest
func TestClientFileList(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
testClientFileList(false, t)
}
func TestClientFileListEncrypted(t *testing.T) {
testClientFileList(true, t)
}
func testClientFileList(toEncrypt bool, t *testing.T) {
srv := testutil.NewTestSwarmServer(t, serverFunc)
defer srv.Close()
dir := newTestDirectory(t)
defer os.RemoveAll(dir)
client := NewClient(srv.URL)
hash, err := client.UploadDirectory(dir, "", "")
hash, err := client.UploadDirectory(dir, "", "", toEncrypt)
if err != nil {
t.Fatalf("error uploading directory: %s", err)
}
......@@ -275,7 +306,7 @@ func TestClientFileList(t *testing.T) {
// TestClientMultipartUpload tests uploading files to swarm using a multipart
// upload
func TestClientMultipartUpload(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
srv := testutil.NewTestSwarmServer(t, serverFunc)
defer srv.Close()
// define an uploader which uploads testDirFiles with some data
......
swarm/api/config.go
View file @ eaff8929
......@@ -21,13 +21,16 @@ import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/contracts/ens"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/pss"
"github.com/ethereum/go-ethereum/swarm/services/swap"
"github.com/ethereum/go-ethereum/swarm/storage"
)
......@@ -41,47 +44,55 @@ const (
// allow several bzz nodes running in parallel
type Config struct {
// serialised/persisted fields
*storage.StoreParams
*storage.ChunkerParams
*storage.FileStoreParams
*storage.LocalStoreParams
*network.HiveParams
Swap *swap.SwapParams
*network.SyncParams
Contract common.Address
EnsRoot common.Address
EnsAPIs []string
Path string
ListenAddr string
Port string
PublicKey string
BzzKey string
NetworkId uint64
SwapEnabled bool
SyncEnabled bool
SwapApi string
Cors string
BzzAccount string
BootNodes string
Swap *swap.LocalProfile
Pss *pss.PssParams
//*network.SyncParams
Contract common.Address
EnsRoot common.Address
EnsAPIs []string
Path string
ListenAddr string
Port string
PublicKey string
BzzKey string
NodeID string
NetworkID uint64
SwapEnabled bool
SyncEnabled bool
DeliverySkipCheck bool
SyncUpdateDelay time.Duration
SwapAPI string
Cors string
BzzAccount string
BootNodes string
privateKey *ecdsa.PrivateKey
}
//create a default config with all parameters to set to defaults
func NewDefaultConfig() (self *Config) {
self = &Config{
StoreParams: storage.NewDefaultStoreParams(),
ChunkerParams: storage.NewChunkerParams(),
HiveParams: network.NewDefaultHiveParams(),
SyncParams: network.NewDefaultSyncParams(),
Swap: swap.NewDefaultSwapParams(),
ListenAddr: DefaultHTTPListenAddr,
Port: DefaultHTTPPort,
Path: node.DefaultDataDir(),
EnsAPIs: nil,
EnsRoot: ens.TestNetAddress,
NetworkId: network.NetworkId,
SwapEnabled: false,
SyncEnabled: true,
SwapApi: "",
BootNodes: "",
func NewConfig() (c *Config) {
c = &Config{
LocalStoreParams: storage.NewDefaultLocalStoreParams(),
FileStoreParams: storage.NewFileStoreParams(),
HiveParams: network.NewHiveParams(),
//SyncParams: network.NewDefaultSyncParams(),
Swap: swap.NewDefaultSwapParams(),
Pss: pss.NewPssParams(),
ListenAddr: DefaultHTTPListenAddr,
Port: DefaultHTTPPort,
Path: node.DefaultDataDir(),
EnsAPIs: nil,
EnsRoot: ens.TestNetAddress,
NetworkID: network.DefaultNetworkID,
SwapEnabled: false,
SyncEnabled: true,
DeliverySkipCheck: false,
SyncUpdateDelay: 15 * time.Second,
SwapAPI: "",
BootNodes: "",
}
return
......@@ -89,11 +100,11 @@ func NewDefaultConfig() (self *Config) {
//some config params need to be initialized after the complete
//config building phase is completed (e.g. due to overriding flags)
func (self *Config) Init(prvKey *ecdsa.PrivateKey) {
func (c *Config) Init(prvKey *ecdsa.PrivateKey) {
address := crypto.PubkeyToAddress(prvKey.PublicKey)
self.Path = filepath.Join(self.Path, "bzz-"+common.Bytes2Hex(address.Bytes()))
err := os.MkdirAll(self.Path, os.ModePerm)
c.Path = filepath.Join(c.Path, "bzz-"+common.Bytes2Hex(address.Bytes()))
err := os.MkdirAll(c.Path, os.ModePerm)
if err != nil {
log.Error(fmt.Sprintf("Error creating root swarm data directory: %v", err))
return
......@@ -103,11 +114,25 @@ func (self *Config) Init(prvKey *ecdsa.PrivateKey) {
pubkeyhex := common.ToHex(pubkey)
keyhex := crypto.Keccak256Hash(pubkey).Hex()
self.PublicKey = pubkeyhex
self.BzzKey = keyhex
c.PublicKey = pubkeyhex
c.BzzKey = keyhex
c.NodeID = discover.PubkeyID(&prvKey.PublicKey).String()
if c.SwapEnabled {
c.Swap.Init(c.Contract, prvKey)
}
c.privateKey = prvKey
c.LocalStoreParams.Init(c.Path)
c.LocalStoreParams.BaseKey = common.FromHex(keyhex)
self.Swap.Init(self.Contract, prvKey)
self.SyncParams.Init(self.Path)
self.HiveParams.Init(self.Path)
self.StoreParams.Init(self.Path)
c.Pss = c.Pss.WithPrivateKey(c.privateKey)
}
func (c *Config) ShiftPrivateKey() (privKey *ecdsa.PrivateKey) {
if c.privateKey != nil {
privKey = c.privateKey
c.privateKey = nil
}
return privKey
}
swarm/api/config_test.go
View file @ eaff8929
......@@ -33,9 +33,10 @@ func TestConfig(t *testing.T) {
t.Fatalf("failed to load private key: %v", err)
}
one := NewDefaultConfig()
two := NewDefaultConfig()
one := NewConfig()
two := NewConfig()
one.LocalStoreParams = two.LocalStoreParams
if equal := reflect.DeepEqual(one, two); !equal {
t.Fatal("Two default configs are not equal")
}
......@@ -49,21 +50,10 @@ func TestConfig(t *testing.T) {
if one.PublicKey == "" {
t.Fatal("Expected PublicKey to be set")
}
//the Init function should append subdirs to the given path
if one.Swap.PayProfile.Beneficiary == (common.Address{}) {
if one.Swap.PayProfile.Beneficiary == (common.Address{}) && one.SwapEnabled {
t.Fatal("Failed to correctly initialize SwapParams")
}
if one.SyncParams.RequestDbPath == one.Path {
t.Fatal("Failed to correctly initialize SyncParams")
}
if one.HiveParams.KadDbPath == one.Path {
t.Fatal("Failed to correctly initialize HiveParams")
}
if one.StoreParams.ChunkDbPath == one.Path {
if one.ChunkDbPath == one.Path {
t.Fatal("Failed to correctly initialize StoreParams")
}
}
swarm/api/filesystem.go
View file @ eaff8929
......@@ -27,26 +27,27 @@ import (
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/storage"
)
const maxParallelFiles = 5
type FileSystem struct {
api *Api
api *API
}
func NewFileSystem(api *Api) *FileSystem {
func NewFileSystem(api *API) *FileSystem {
return &FileSystem{api}
}
// Upload replicates a local directory as a manifest file and uploads it
// using dpa store
// using FileStore store
// This function waits the chunks to be stored.
// TODO: localpath should point to a manifest
//
// DEPRECATED: Use the HTTP API instead
func (self *FileSystem) Upload(lpath, index string) (string, error) {
func (fs *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error) {
var list []*manifestTrieEntry
localpath, err := filepath.Abs(filepath.Clean(lpath))
if err != nil {
......@@ -111,13 +112,13 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
f, err := os.Open(entry.Path)
if err == nil {
stat, _ := f.Stat()
var hash storage.Key
wg := &sync.WaitGroup{}
hash, err = self.api.dpa.Store(f, stat.Size(), wg, nil)
var hash storage.Address
var wait func()
hash, wait, err = fs.api.fileStore.Store(f, stat.Size(), toEncrypt)
if hash != nil {
list[i].Hash = hash.String()
list[i].Hash = hash.Hex()
}
wg.Wait()
wait()
awg.Done()
if err == nil {
first512 := make([]byte, 512)
......@@ -142,7 +143,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
}
trie := &manifestTrie{
dpa: self.api.dpa,
fileStore: fs.api.fileStore,
}
quitC := make(chan bool)
for i, entry := range list {
......@@ -163,7 +164,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
err2 := trie.recalcAndStore()
var hs string
if err2 == nil {
hs = trie.hash.String()
hs = trie.ref.Hex()
}
awg.Wait()
return hs, err2
......@@ -173,7 +174,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
// under localpath
//
// DEPRECATED: Use the HTTP API instead
func (self *FileSystem) Download(bzzpath, localpath string) error {
func (fs *FileSystem) Download(bzzpath, localpath string) error {
lpath, err := filepath.Abs(filepath.Clean(localpath))
if err != nil {
return err
......@@ -188,7 +189,7 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
if err != nil {
return err
}
key, err := self.api.Resolve(uri)
addr, err := fs.api.Resolve(uri)
if err != nil {
return err
}
......@@ -199,14 +200,14 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
}
quitC := make(chan bool)
trie, err := loadManifest(self.api.dpa, key, quitC)
trie, err := loadManifest(fs.api.fileStore, addr, quitC)
if err != nil {
log.Warn(fmt.Sprintf("fs.Download: loadManifestTrie error: %v", err))
return err
}
type downloadListEntry struct {
key storage.Key
addr storage.Address
path string
}
......@@ -217,7 +218,7 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
err = trie.listWithPrefix(path, quitC, func(entry *manifestTrieEntry, suffix string) {
log.Trace(fmt.Sprintf("fs.Download: %#v", entry))
key = common.Hex2Bytes(entry.Hash)
addr = common.Hex2Bytes(entry.Hash)
path := lpath + "/" + suffix
dir := filepath.Dir(path)
if dir != prevPath {
......@@ -225,7 +226,7 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
prevPath = dir
}
if (mde == nil) && (path != dir+"/") {
list = append(list, &downloadListEntry{key: key, path: path})
list = append(list, &downloadListEntry{addr: addr, path: path})
}
})
if err != nil {
......@@ -244,7 +245,7 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
}
go func(i int, entry *downloadListEntry) {
defer wg.Done()
err := retrieveToFile(quitC, self.api.dpa, entry.key, entry.path)
err := retrieveToFile(quitC, fs.api.fileStore, entry.addr, entry.path)
if err != nil {
select {
case errC <- err:
......@@ -267,12 +268,12 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
}
}
func retrieveToFile(quitC chan bool, dpa *storage.DPA, key storage.Key, path string) error {
func retrieveToFile(quitC chan bool, fileStore *storage.FileStore, addr storage.Address, path string) error {
f, err := os.Create(path) // TODO: basePath separators
if err != nil {
return err
}
reader := dpa.Retrieve(key)
reader, _ := fileStore.Retrieve(addr)
writer := bufio.NewWriter(f)
size, err := reader.Size(quitC)
if err != nil {
......
swarm/api/filesystem_test.go
View file @ eaff8929
......@@ -21,7 +21,6 @@ import (
"io/ioutil"
"os"
"path/filepath"
"sync"
"testing"
"github.com/ethereum/go-ethereum/common"
......@@ -30,9 +29,9 @@ import (
var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test")
func testFileSystem(t *testing.T, f func(*FileSystem)) {
testApi(t, func(api *Api) {
f(NewFileSystem(api))
func testFileSystem(t *testing.T, f func(*FileSystem, bool)) {
testAPI(t, func(api *API, toEncrypt bool) {
f(NewFileSystem(api), toEncrypt)
})
}
......@@ -47,9 +46,9 @@ func readPath(t *testing.T, parts ...string) string {
}
func TestApiDirUpload0(t *testing.T) {
testFileSystem(t, func(fs *FileSystem) {
testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "")
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "", toEncrypt)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
......@@ -63,8 +62,8 @@ func TestApiDirUpload0(t *testing.T) {
exp = expResponse(content, "text/css", 0)
checkResponse(t, resp, exp)
key := storage.Key(common.Hex2Bytes(bzzhash))
_, _, _, err = api.Get(key, "")
addr := storage.Address(common.Hex2Bytes(bzzhash))
_, _, _, _, err = api.Get(addr, "")
if err == nil {
t.Fatalf("expected error: %v", err)
}
......@@ -75,27 +74,28 @@ func TestApiDirUpload0(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
newbzzhash, err := fs.Upload(downloadDir, "")
newbzzhash, err := fs.Upload(downloadDir, "", toEncrypt)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if bzzhash != newbzzhash {
// TODO: currently the hash is not deterministic in the encrypted case
if !toEncrypt && bzzhash != newbzzhash {
t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash)
}
})
}
func TestApiDirUploadModify(t *testing.T) {
testFileSystem(t, func(fs *FileSystem) {
testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "")
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "", toEncrypt)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
key := storage.Key(common.Hex2Bytes(bzzhash))
key, err = api.Modify(key, "index.html", "", "")
addr := storage.Address(common.Hex2Bytes(bzzhash))
addr, err = api.Modify(addr, "index.html", "", "")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
......@@ -105,24 +105,23 @@ func TestApiDirUploadModify(t *testing.T) {
t.Errorf("unexpected error: %v", err)
return
}
wg := &sync.WaitGroup{}
hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg)
wg.Wait()
hash, wait, err := api.Store(bytes.NewReader(index), int64(len(index)), toEncrypt)
wait()
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
key, err = api.Modify(key, "index2.html", hash.Hex(), "text/html; charset=utf-8")
addr, err = api.Modify(addr, "index2.html", hash.Hex(), "text/html; charset=utf-8")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
key, err = api.Modify(key, "img/logo.png", hash.Hex(), "text/html; charset=utf-8")
addr, err = api.Modify(addr, "img/logo.png", hash.Hex(), "text/html; charset=utf-8")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash = key.String()
bzzhash = addr.Hex()
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "index2.html")
......@@ -138,7 +137,7 @@ func TestApiDirUploadModify(t *testing.T) {
exp = expResponse(content, "text/css", 0)
checkResponse(t, resp, exp)
_, _, _, err = api.Get(key, "")
_, _, _, _, err = api.Get(addr, "")
if err == nil {
t.Errorf("expected error: %v", err)
}
......@@ -146,9 +145,9 @@ func TestApiDirUploadModify(t *testing.T) {
}
func TestApiDirUploadWithRootFile(t *testing.T) {
testFileSystem(t, func(fs *FileSystem) {
testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "index.html")
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "index.html", toEncrypt)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
......@@ -162,9 +161,9 @@ func TestApiDirUploadWithRootFile(t *testing.T) {
}
func TestApiFileUpload(t *testing.T) {
testFileSystem(t, func(fs *FileSystem) {
testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "")
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "", toEncrypt)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
......@@ -178,9 +177,9 @@ func TestApiFileUpload(t *testing.T) {
}
func TestApiFileUploadWithRootFile(t *testing.T) {
testFileSystem(t, func(fs *FileSystem) {
testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "index.html")
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "index.html", toEncrypt)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
......
swarm/api/http/error.go
View file @ eaff8929
......@@ -31,6 +31,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/api"
l "github.com/ethereum/go-ethereum/swarm/log"
)
//templateMap holds a mapping of an HTTP error code to a template
......@@ -44,7 +45,7 @@ var (
)
//parameters needed for formatting the correct HTML page
type ErrorParams struct {
type ResponseParams struct {
Msg string
Code int
Timestamp string
......@@ -113,45 +114,49 @@ func ValidateCaseErrors(r *Request) string {
//For example, if the user requests bzz:/<hash>/read and that manifest contains entries
//"readme.md" and "readinglist.txt", a HTML page is returned with this two links.
//This only applies if the manifest has no default entry
func ShowMultipleChoices(w http.ResponseWriter, r *Request, list api.ManifestList) {
func ShowMultipleChoices(w http.ResponseWriter, req *Request, list api.ManifestList) {
msg := ""
if list.Entries == nil {
ShowError(w, r, "Could not resolve", http.StatusInternalServerError)
Respond(w, req, "Could not resolve", http.StatusInternalServerError)
return
}
//make links relative
//requestURI comes with the prefix of the ambiguous path, e.g. "read" for "readme.md" and "readinglist.txt"
//to get clickable links, need to remove the ambiguous path, i.e. "read"
idx := strings.LastIndex(r.RequestURI, "/")
idx := strings.LastIndex(req.RequestURI, "/")
if idx == -1 {
ShowError(w, r, "Internal Server Error", http.StatusInternalServerError)
Respond(w, req, "Internal Server Error", http.StatusInternalServerError)
return
}
//remove ambiguous part
base := r.RequestURI[:idx+1]
base := req.RequestURI[:idx+1]
for _, e := range list.Entries {
//create clickable link for each entry
msg += "<a href='" + base + e.Path + "'>" + e.Path + "</a><br/>"
}
respond(w, &r.Request, &ErrorParams{
Code: http.StatusMultipleChoices,
Details: template.HTML(msg),
Timestamp: time.Now().Format(time.RFC1123),
template: getTemplate(http.StatusMultipleChoices),
})
Respond(w, req, msg, http.StatusMultipleChoices)
}
//ShowError is used to show an HTML error page to a client.
//Respond is used to show an HTML page to a client.
//If there is an `Accept` header of `application/json`, JSON will be returned instead
//The function just takes a string message which will be displayed in the error page.
//The code is used to evaluate which template will be displayed
//(and return the correct HTTP status code)
func ShowError(w http.ResponseWriter, r *Request, msg string, code int) {
additionalMessage := ValidateCaseErrors(r)
if code == http.StatusInternalServerError {
log.Error(msg)
func Respond(w http.ResponseWriter, req *Request, msg string, code int) {
additionalMessage := ValidateCaseErrors(req)
switch code {
case http.StatusInternalServerError:
log.Output(msg, log.LvlError, l.CallDepth, "ruid", req.ruid, "code", code)
default:
log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
}
if code >= 400 {
w.Header().Del("Cache-Control") //avoid sending cache headers for errors!
w.Header().Del("ETag")
}
respond(w, &r.Request, &ErrorParams{
respond(w, &req.Request, &ResponseParams{
Code: code,
Msg: msg,
Details: template.HTML(additionalMessage),
......@@ -161,17 +166,17 @@ func ShowError(w http.ResponseWriter, r *Request, msg string, code int) {
}
//evaluate if client accepts html or json response
func respond(w http.ResponseWriter, r *http.Request, params *ErrorParams) {
func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
w.WriteHeader(params.Code)
if r.Header.Get("Accept") == "application/json" {
respondJson(w, params)
respondJSON(w, params)
} else {
respondHtml(w, params)
respondHTML(w, params)
}
}
//return a HTML page
func respondHtml(w http.ResponseWriter, params *ErrorParams) {
func respondHTML(w http.ResponseWriter, params *ResponseParams) {
htmlCounter.Inc(1)
err := params.template.Execute(w, params)
if err != nil {
......@@ -180,7 +185,7 @@ func respondHtml(w http.ResponseWriter, params *ErrorParams) {
}
//return JSON
func respondJson(w http.ResponseWriter, params *ErrorParams) {
func respondJSON(w http.ResponseWriter, params *ResponseParams) {
jsonCounter.Inc(1)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(params)
......@@ -190,7 +195,6 @@ func respondJson(w http.ResponseWriter, params *ErrorParams) {
func getTemplate(code int) *template.Template {
if val, tmpl := templateMap[code]; tmpl {
return val
} else {
return templateMap[0]
}
return templateMap[0]
}
swarm/api/http/error_templates.go
View file @ eaff8929
This diff is collapsed.
swarm/api/http/error_test.go
View file @ eaff8929
......@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http_test
package http
import (
"encoding/json"
......@@ -30,7 +30,7 @@ import (
func TestError(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
srv := testutil.NewTestSwarmServer(t, serverFunc)
defer srv.Close()
var resp *http.Response
......@@ -56,7 +56,7 @@ func TestError(t *testing.T) {
}
func Test404Page(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
srv := testutil.NewTestSwarmServer(t, serverFunc)
defer srv.Close()
var resp *http.Response
......@@ -82,7 +82,7 @@ func Test404Page(t *testing.T) {
}
func Test500Page(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
srv := testutil.NewTestSwarmServer(t, serverFunc)
defer srv.Close()
var resp *http.Response
......@@ -107,7 +107,7 @@ func Test500Page(t *testing.T) {
}
}
func Test500PageWith0xHashPrefix(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
srv := testutil.NewTestSwarmServer(t, serverFunc)
defer srv.Close()
var resp *http.Response
......@@ -137,7 +137,7 @@ func Test500PageWith0xHashPrefix(t *testing.T) {
}
func TestJsonResponse(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
srv := testutil.NewTestSwarmServer(t, serverFunc)
defer srv.Close()
var resp *http.Response
......
swarm/api/http/roundtripper.go
View file @ eaff8929
......@@ -20,7 +20,7 @@ import (
"fmt"
"net/http"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/log"
)
/*
......
swarm/api/http/server.go
View file @ eaff8929
This diff is collapsed.
swarm/api/http/server_test.go
View file @ eaff8929
This diff is collapsed.
swarm/api/http/templates.go
View file @ eaff8929
......@@ -78,7 +78,6 @@ var landingPageTemplate = template.Must(template.New("landingPage").Parse(`
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
<meta http-equiv="X-UA-Compatible" ww="chrome=1">
<meta name="description" content="Ethereum/Swarm Landing page">
<meta property="og:url" content="https://swarm-gateways.net/bzz:/theswarm.eth">
<style>
body, div, header, footer {
......@@ -206,7 +205,7 @@ var landingPageTemplate = template.Must(template.New("landingPage").Parse(`
<footer>
<p>
Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution<br/>
<a href="http://swarm-gateways.net/bzz:/theswarm.eth">Swarm</a>
<a href="/bzz:/theswarm.eth">Swarm</a>
</p>
</footer>
......
swarm/api/manifest.go
View file @ eaff8929
This diff is collapsed.
swarm/api/manifest_test.go
View file @ eaff8929
......@@ -42,7 +42,9 @@ func manifest(paths ...string) (manifestReader storage.LazySectionReader) {
func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...string) *manifestTrie {
quitC := make(chan bool)
trie, err := readManifest(manifest(paths...), nil, nil, quitC)
fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
ref := make([]byte, fileStore.HashSize())
trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC)
if err != nil {
t.Errorf("unexpected error making manifest: %v", err)
}
......@@ -97,7 +99,9 @@ func TestGetEntry(t *testing.T) {
func TestExactMatch(t *testing.T) {
quitC := make(chan bool)
mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map")
trie, err := readManifest(mf, nil, nil, quitC)
fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
ref := make([]byte, fileStore.HashSize())
trie, err := readManifest(mf, ref, fileStore, false, quitC)
if err != nil {
t.Errorf("unexpected error making manifest: %v", err)
}
......@@ -128,7 +132,9 @@ func TestAddFileWithManifestPath(t *testing.T) {
reader := &storage.LazyTestSectionReader{
SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))),
}
trie, err := readManifest(reader, nil, nil, nil)
fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
ref := make([]byte, fileStore.HashSize())
trie, err := readManifest(reader, ref, fileStore, false, nil)
if err != nil {
t.Fatal(err)
}
......@@ -144,3 +150,26 @@ func TestAddFileWithManifestPath(t *testing.T) {
checkEntry(t, "ac", "ac", false, trie)
checkEntry(t, "a", "a", false, trie)
}
// TestReadManifestOverSizeLimit creates a manifest reader with data longer then
// manifestSizeLimit and checks if readManifest function will return the exact error
// message.
// The manifest data is not in json-encoded format, preventing possbile
// successful parsing attempts if limit check fails.
func TestReadManifestOverSizeLimit(t *testing.T) {
manifest := make([]byte, manifestSizeLimit+1)
reader := &storage.LazyTestSectionReader{
SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))),
}
_, err := readManifest(reader, storage.Address{}, nil, false, nil)
if err == nil {
t.Fatal("got no error from readManifest")
}
// Error message is part of the http response body
// which justifies exact string validation.
got := err.Error()
want := fmt.Sprintf("Manifest size of %v bytes exceeds the %v byte limit", len(manifest), manifestSizeLimit)
if got != want {
t.Fatalf("got error mesage %q, expected %q", got, want)
}
}
swarm/api/storage.go
View file @ eaff8929
......@@ -16,7 +16,11 @@
package api
import "path"
import (
"path"
"github.com/ethereum/go-ethereum/swarm/storage"
)
type Response struct {
MimeType string
......@@ -30,10 +34,10 @@ type Response struct {
//
// DEPRECATED: Use the HTTP API instead
type Storage struct {
api *Api
api *API
}
func NewStorage(api *Api) *Storage {
func NewStorage(api *API) *Storage {
return &Storage{api}
}
......@@ -41,12 +45,8 @@ func NewStorage(api *Api) *Storage {
// its content type
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Put(content, contentType string) (string, error) {
key, err := self.api.Put(content, contentType)
if err != nil {
return "", err
}
return key.String(), err
func (s *Storage) Put(content, contentType string, toEncrypt bool) (storage.Address, func(), error) {
return s.api.Put(content, contentType, toEncrypt)
}
// Get retrieves the content from bzzpath and reads the response in full
......@@ -57,16 +57,16 @@ func (self *Storage) Put(content, contentType string) (string, error) {
// size is resp.Size
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Get(bzzpath string) (*Response, error) {
func (s *Storage) Get(bzzpath string) (*Response, error) {
uri, err := Parse(path.Join("bzz:/", bzzpath))
if err != nil {
return nil, err
}
key, err := self.api.Resolve(uri)
addr, err := s.api.Resolve(uri)
if err != nil {
return nil, err
}
reader, mimeType, status, err := self.api.Get(key, uri.Path)
reader, mimeType, status, _, err := s.api.Get(addr, uri.Path)
if err != nil {
return nil, err
}
......@@ -87,18 +87,18 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
// and merge on to it. creating an entry w conentType (mime)
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
func (s *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
uri, err := Parse("bzz:/" + rootHash)
if err != nil {
return "", err
}
key, err := self.api.Resolve(uri)
addr, err := s.api.Resolve(uri)
if err != nil {
return "", err
}
key, err = self.api.Modify(key, path, contentHash, contentType)
addr, err = s.api.Modify(addr, path, contentHash, contentType)
if err != nil {
return "", err
}
return key.String(), nil
return addr.Hex(), nil
}
swarm/api/storage_test.go
View file @ eaff8929
......@@ -20,22 +20,24 @@ import (
"testing"
)
func testStorage(t *testing.T, f func(*Storage)) {
testApi(t, func(api *Api) {
f(NewStorage(api))
func testStorage(t *testing.T, f func(*Storage, bool)) {
testAPI(t, func(api *API, toEncrypt bool) {
f(NewStorage(api), toEncrypt)
})
}
func TestStoragePutGet(t *testing.T) {
testStorage(t, func(api *Storage) {
testStorage(t, func(api *Storage, toEncrypt bool) {
content := "hello"
exp := expResponse(content, "text/plain", 0)
// exp := expResponse([]byte(content), "text/plain", 0)
bzzhash, err := api.Put(content, exp.MimeType)
bzzkey, wait, err := api.Put(content, exp.MimeType, toEncrypt)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// to check put against the Api#Get
wait()
bzzhash := bzzkey.Hex()
// to check put against the API#Get
resp0 := testGet(t, api.api, bzzhash, "")
checkResponse(t, resp0, exp)
......
swarm/api/testapi.go
View file @ eaff8929
......@@ -21,26 +21,26 @@ import (
)
type Control struct {
api *Api
api *API
hive *network.Hive
}
func NewControl(api *Api, hive *network.Hive) *Control {
func NewControl(api *API, hive *network.Hive) *Control {
return &Control{api, hive}
}
func (self *Control) BlockNetworkRead(on bool) {
self.hive.BlockNetworkRead(on)
}
func (self *Control) SyncEnabled(on bool) {
self.hive.SyncEnabled(on)
}
func (self *Control) SwapEnabled(on bool) {
self.hive.SwapEnabled(on)
}
func (self *Control) Hive() string {
return self.hive.String()
//func (self *Control) BlockNetworkRead(on bool) {
// self.hive.BlockNetworkRead(on)
//}
//
//func (self *Control) SyncEnabled(on bool) {
// self.hive.SyncEnabled(on)
//}
//
//func (self *Control) SwapEnabled(on bool) {
// self.hive.SwapEnabled(on)
//}
//
func (c *Control) Hive() string {
return c.hive.String()
}
swarm/api/uri.go
View file @ eaff8929
......@@ -19,9 +19,17 @@ package api
import (
"fmt"
"net/url"
"regexp"
"strings"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/storage"
)
//matches hex swarm hashes
// TODO: this is bad, it should not be hardcoded how long is a hash
var hashMatcher = regexp.MustCompile("^([0-9A-Fa-f]{64})([0-9A-Fa-f]{64})?$")
// URI is a reference to content stored in swarm.
type URI struct {
// Scheme has one of the following values:
......@@ -32,18 +40,15 @@ type URI struct {
// (address is not resolved)
// * bzz-list - list of all files contained in a swarm manifest
//
// Deprecated Schemes:
// * bzzr - raw swarm content
// * bzzi - immutable URI of an entry in a swarm manifest
// (address is not resolved)
// * bzz-hash - hash of swarm content
//
Scheme string
// Addr is either a hexadecimal storage key or it an address which
// resolves to a storage key
// Addr is either a hexadecimal storage address or it an address which
// resolves to a storage address
Addr string
// addr stores the parsed storage address
addr storage.Address
// Path is the path to the content within a swarm manifest
Path string
}
......@@ -59,7 +64,6 @@ type URI struct {
// * <scheme>://<addr>/<path>
//
// with scheme one of bzz, bzz-raw, bzz-immutable, bzz-list or bzz-hash
// or deprecated ones bzzr and bzzi
func Parse(rawuri string) (*URI, error) {
u, err := url.Parse(rawuri)
if err != nil {
......@@ -69,7 +73,7 @@ func Parse(rawuri string) (*URI, error) {
// check the scheme is valid
switch uri.Scheme {
case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi":
case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzz-resource":
default:
return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
}
......@@ -91,6 +95,9 @@ func Parse(rawuri string) (*URI, error) {
}
return uri, nil
}
func (u *URI) Resource() bool {
return u.Scheme == "bzz-resource"
}
func (u *URI) Raw() bool {
return u.Scheme == "bzz-raw"
......@@ -104,14 +111,6 @@ func (u *URI) List() bool {
return u.Scheme == "bzz-list"
}
func (u *URI) DeprecatedRaw() bool {
return u.Scheme == "bzzr"
}
func (u *URI) DeprecatedImmutable() bool {
return u.Scheme == "bzzi"
}
func (u *URI) Hash() bool {
return u.Scheme == "bzz-hash"
}
......@@ -119,3 +118,14 @@ func (u *URI) Hash() bool {
func (u *URI) String() string {
return u.Scheme + ":/" + u.Addr + "/" + u.Path
}
func (u *URI) Address() storage.Address {
if u.addr != nil {
return u.addr
}
if hashMatcher.MatchString(u.Addr) {
u.addr = common.Hex2Bytes(u.Addr)
return u.addr
}
return nil
}
swarm/api/uri_test.go
View file @ eaff8929
This diff is collapsed.
bmt/bmt.go → swarm/bmt/bmt.go
View file @ eaff8929
This diff is collapsed.
bmt/bmt_r.go → swarm/bmt/bmt_r.go
View file @ eaff8929
This diff is collapsed.
bmt/bmt_test.go → swarm/bmt/bmt_test.go
View file @ eaff8929
This diff is collapsed.
swarm/fuse/fuse_dir.go
View file @ eaff8929
This diff is collapsed.
swarm/fuse/fuse_file.go
View file @ eaff8929
This diff is collapsed.
swarm/fuse/swarmfs.go
View file @ eaff8929
......@@ -39,12 +39,12 @@ var (
)
type SwarmFS struct {
swarmApi *api.Api
swarmApi *api.API
activeMounts map[string]*MountInfo
swarmFsLock *sync.RWMutex
}
func NewSwarmFS(api *api.Api) *SwarmFS {
func NewSwarmFS(api *api.API) *SwarmFS {
swarmfsLock.Do(func() {
swarmfs = &SwarmFS{
swarmApi: api,
......
swarm/fuse/swarmfs_test.go
View file @ eaff8929
This diff is collapsed.
swarm/fuse/swarmfs_unix.go
View file @ eaff8929
This diff is collapsed.
swarm/fuse/swarmfs_util.go
View file @ eaff8929
This diff is collapsed.
swarm/grafana_dashboards/ldbstore.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/grafana_dashboards/swarm.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/log/log.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/metrics/flags.go
View file @ eaff8929
......@@ -20,9 +20,9 @@ import (
"time"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/log"
gethmetrics "github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/metrics/influxdb"
"github.com/ethereum/go-ethereum/swarm/log"
"gopkg.in/urfave/cli.v1"
)
......
swarm/multihash/multihash.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/multihash/multihash_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/README.md 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/bitvector/bitvector.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/bitvector/bitvector_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/common.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/depo.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/network/discovery.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/discovery_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/forwarding.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/network/hive.go
View file @ eaff8929
This diff is collapsed.
swarm/network/hive_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/kademlia.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/kademlia/address_test.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/network/kademlia/kaddb.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/network/kademlia/kademlia.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/network/kademlia/kademlia_test.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/network/kademlia_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/messages.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/network/priorityqueue/priorityqueue.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/priorityqueue/priorityqueue_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/protocol.go
View file @ eaff8929
This diff is collapsed.
swarm/network/protocol_test.go
View file @ eaff8929
This diff is collapsed.
swarm/network/simulations/discovery/discovery.go 0 → 100644
View file @ eaff8929
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package discovery
swarm/network/simulations/discovery/discovery_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/simulations/discovery/jsonsnapshot.txt 0 → 100755
View file @ eaff8929
This diff is collapsed.
swarm/network/simulations/overlay.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/simulations/overlay_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/common_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/delivery.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/delivery_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/intervals/dbstore_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/intervals/intervals.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/intervals/intervals_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/intervals/store_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/intervals_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/messages.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/peer.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/snapshot_retrieval_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/snapshot_sync_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/stream.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/streamer_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/syncer.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/syncer_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/testing/snapshot_128.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/testing/snapshot_16.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/testing/snapshot_256.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/testing/snapshot_32.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/testing/snapshot_64.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/stream/testing/testing.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/network/syncdb.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/network/syncdb_test.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/network/syncer.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/network_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pot/doc.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pot/pot.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pot/pot_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/ARCHITECTURE.md 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/README.md 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/api.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/client/client.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/client/client_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/client/doc.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/doc.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/handshake.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/handshake_none.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/handshake_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/ping.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/protocol.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/protocol_none.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/protocol_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/pss.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/pss_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/addpsstodiscoverytestsnapshot.pl 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/addpsstodiscoverytestsnapshot.sh 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/snapshot_128.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/snapshot_16.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/snapshot_2.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/snapshot_256.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/snapshot_3.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/snapshot_32.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/snapshot_4.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/snapshot_64.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/testdata/snapshot_8.json 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/types.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/pss/writeup.md 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/services/swap/swap.go
View file @ eaff8929
This diff is collapsed.
swarm/services/swap/swap/swap.go
View file @ eaff8929
This diff is collapsed.
swarm/services/swap/swap/swap_test.go
View file @ eaff8929
This diff is collapsed.
swarm/state/dbstore.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/state/dbstore_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/state/inmemorystore.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/state/store.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/chunker.go
View file @ eaff8929
This diff is collapsed.
swarm/storage/chunker_test.go
View file @ eaff8929
This diff is collapsed.
swarm/storage/chunkstore.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/common.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/common_test.go
View file @ eaff8929
This diff is collapsed.
swarm/storage/database.go
View file @ eaff8929
This diff is collapsed.
swarm/storage/dbapi.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/dbstore.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/storage/dbstore_test.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/storage/dpa.go deleted 100644 → 0
View file @ d926bf2c
This diff is collapsed.
swarm/storage/encryption/encryption.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/encryption/encryption_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/error.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/filestore.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/hasherstore.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/hasherstore_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/ldbstore.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/ldbstore_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/localstore.go
View file @ eaff8929
This diff is collapsed.
swarm/storage/localstore_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/memstore.go
View file @ eaff8929
This diff is collapsed.
swarm/storage/memstore_test.go
View file @ eaff8929
This diff is collapsed.
swarm/storage/mock/db/db.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mock/db/db_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mock/mem/mem.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mock/mem/mem_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mock/mock.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mock/rpc/rpc.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mock/rpc/rpc_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mock/test/test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mru/error.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mru/resource.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mru/resource_sign.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/mru/resource_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/netstore.go
View file @ eaff8929
This diff is collapsed.
swarm/storage/netstore_test.go 0 → 100644
View file @ eaff8929
This diff is collapsed.
swarm/storage/pyramid.go
View file @ eaff8929
This diff is collapsed.
swarm/storage/swarmhasher.go
View file @ eaff8929
This diff is collapsed.
swarm/storage/types.go
View file @ eaff8929
This diff is collapsed.
swarm/swarm.go
View file @ eaff8929
This diff is collapsed.
swarm/swarm_test.go
View file @ eaff8929
This diff is collapsed.
swarm/testutil/http.go
View file @ eaff8929
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment