integrate blockpool into eth

- remove blockpool code
- remove blockpool integration test (kinda embarrassing)
- remove errors.go

eth/backend.go

@@ -7,6 +7,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/blockpool"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -117,7 +118,7 @@ type Ethereum struct {
 	blockProcessor *core.BlockProcessor
 	txPool         *core.TxPool
 	chainManager   *core.ChainManager
-	blockPool      *BlockPool
+	blockPool      *blockpool.BlockPool
 	whisper        *whisper.Whisper
 
 	net      *p2p.Server
@@ -185,7 +186,7 @@ func New(config *Config) (*Ethereum, error) {
 
 	hasBlock := eth.chainManager.HasBlock
 	insertChain := eth.chainManager.InsertChain
-	eth.blockPool = NewBlockPool(hasBlock, insertChain, ezp.Verify)
+	eth.blockPool = blockpool.New(hasBlock, insertChain, ezp.Verify)
 
 	netprv, err := config.nodeKey()
 	if err != nil {
@@ -220,7 +221,7 @@ func (s *Ethereum) Name() string                         { return s.net.Name }
 func (s *Ethereum) ChainManager() *core.ChainManager     { return s.chainManager }
 func (s *Ethereum) BlockProcessor() *core.BlockProcessor { return s.blockProcessor }
 func (s *Ethereum) TxPool() *core.TxPool                 { return s.txPool }
-func (s *Ethereum) BlockPool() *BlockPool                { return s.blockPool }
+func (s *Ethereum) BlockPool() *blockpool.BlockPool      { return s.blockPool }
 func (s *Ethereum) Whisper() *whisper.Whisper            { return s.whisper }
 func (s *Ethereum) EventMux() *event.TypeMux             { return s.eventMux }
 func (s *Ethereum) Db() ethutil.Database                 { return s.db }

eth/error.go

-package eth
-
-import (
-	"fmt"
-)
-
-const (
-	ErrMsgTooLarge = iota
-	ErrDecode
-	ErrInvalidMsgCode
-	ErrProtocolVersionMismatch
-	ErrNetworkIdMismatch
-	ErrGenesisBlockMismatch
-	ErrNoStatusMsg
-	ErrExtraStatusMsg
-	ErrInvalidBlock
-	ErrInvalidPoW
-	ErrUnrequestedBlock
-	ErrInsufficientChainInfo
-)
-
-var errorToString = map[int]string{
-	ErrMsgTooLarge:             "Message too long",
-	ErrDecode:                  "Invalid message",
-	ErrInvalidMsgCode:          "Invalid message code",
-	ErrProtocolVersionMismatch: "Protocol version mismatch",
-	ErrNetworkIdMismatch:       "NetworkId mismatch",
-	ErrGenesisBlockMismatch:    "Genesis block mismatch",
-	ErrNoStatusMsg:             "No status message",
-	ErrExtraStatusMsg:          "Extra status message",
-	ErrInvalidBlock:            "Invalid block",
-	ErrInvalidPoW:              "Invalid PoW",
-	ErrUnrequestedBlock:        "Unrequested block",
-	ErrInsufficientChainInfo:   "Insufficient chain info",
-}
-
-type protocolError struct {
-	Code    int
-	fatal   bool
-	message string
-	format  string
-	params  []interface{}
-	// size    int
-}
-
-func newProtocolError(code int, format string, params ...interface{}) *protocolError {
-	return &protocolError{Code: code, format: format, params: params}
-}
-
-func ProtocolError(code int, format string, params ...interface{}) (err *protocolError) {
-	err = newProtocolError(code, format, params...)
-	// report(err)
-	return
-}
-
-func (self protocolError) Error() (message string) {
-	if len(message) == 0 {
-		var ok bool
-		self.message, ok = errorToString[self.Code]
-		if !ok {
-			panic("invalid error code")
-		}
-		if self.format != "" {
-			self.message += ": " + fmt.Sprintf(self.format, self.params...)
-		}
-	}
-	return self.message
-}
-
-func (self *protocolError) Fatal() bool {
-	return self.fatal
-}

eth/protocol.go

@@ -7,6 +7,7 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/errs"
 	"github.com/ethereum/go-ethereum/ethutil"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -17,6 +18,8 @@ const (
 	NetworkId          = 0
 	ProtocolLength     = uint64(8)
 	ProtocolMaxMsgSize = 10 * 1024 * 1024
+	maxHashes          = 256
+	maxBlocks          = 64
 )
 
 // eth protocol message codes
@@ -31,6 +34,28 @@ const (
 	NewBlockMsg
 )
 
+const (
+	ErrMsgTooLarge = iota
+	ErrDecode
+	ErrInvalidMsgCode
+	ErrProtocolVersionMismatch
+	ErrNetworkIdMismatch
+	ErrGenesisBlockMismatch
+	ErrNoStatusMsg
+	ErrExtraStatusMsg
+)
+
+var errorToString = map[int]string{
+	ErrMsgTooLarge:             "Message too long",
+	ErrDecode:                  "Invalid message",
+	ErrInvalidMsgCode:          "Invalid message code",
+	ErrProtocolVersionMismatch: "Protocol version mismatch",
+	ErrNetworkIdMismatch:       "NetworkId mismatch",
+	ErrGenesisBlockMismatch:    "Genesis block mismatch",
+	ErrNoStatusMsg:             "No status message",
+	ErrExtraStatusMsg:          "Extra status message",
+}
+
 // ethProtocol represents the ethereum wire protocol
 // instance is running on each peer
 type ethProtocol struct {
@@ -40,6 +65,7 @@ type ethProtocol struct {
 	peer         *p2p.Peer
 	id           string
 	rw           p2p.MsgReadWriter
+	errors       *errs.Errors
 }
 
 // backend is the interface the ethereum protocol backend should implement
@@ -58,7 +84,7 @@ type chainManager interface {
 type blockPool interface {
 	AddBlockHashes(next func() ([]byte, bool), peerId string)
 	AddBlock(block *types.Block, peerId string)
-	AddPeer(td *big.Int, currentBlock []byte, peerId string, requestHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(int, string, ...interface{})) (best bool)
+	AddPeer(td *big.Int, currentBlock []byte, peerId string, requestHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(*errs.Error)) (best bool)
 	RemovePeer(peerId string)
 }
 
@@ -68,8 +94,6 @@ type newBlockMsgData struct {
 	TD    *big.Int
 }
 
-const maxHashes = 255
-
 type getBlockHashesMsgData struct {
 	Hash   []byte
 	Amount uint64
@@ -99,7 +123,11 @@ func runEthProtocol(txPool txPool, chainManager chainManager, blockPool blockPoo
 		blockPool:    blockPool,
 		rw:           rw,
 		peer:         peer,
-		id:           fmt.Sprintf("%x", id[:8]),
+		errors: &errs.Errors{
+			Package: "ETH",
+			Errors:  errorToString,
+		},
+		id: fmt.Sprintf("%x", id[:8]),
 	}
 	err = self.handleStatus()
 	if err == nil {
@@ -145,7 +173,6 @@ func (self *ethProtocol) handle() error {
 			return self.protoError(ErrDecode, "->msg %v: %v", msg, err)
 		}
 
-		//request.Amount = uint64(math.Min(float64(maxHashes), float64(request.Amount)))
 		if request.Amount > maxHashes {
 			request.Amount = maxHashes
 		}
@@ -153,7 +180,6 @@ func (self *ethProtocol) handle() error {
 		return p2p.EncodeMsg(self.rw, BlockHashesMsg, ethutil.ByteSliceToInterface(hashes)...)
 
 	case BlockHashesMsg:
-		// TODO: redo using lazy decode , this way very inefficient on known chains
 		msgStream := rlp.NewStream(msg.Payload)
 		var err error
 		var i int
@@ -191,7 +217,7 @@ func (self *ethProtocol) handle() error {
 			if block != nil {
 				blocks = append(blocks, block)
 			}
-			if i == blockHashesBatchSize {
+			if i == maxBlocks {
 				break
 			}
 		}
@@ -218,7 +244,7 @@ func (self *ethProtocol) handle() error {
 		}
 		hash := request.Block.Hash()
 		// to simplify backend interface adding a new block
-		// uses AddPeer followed by AddHashes, AddBlock only if peer is the best peer
+		// uses AddPeer followed by AddBlock only if peer is the best peer
 		// (or selected as new best peer)
 		if self.blockPool.AddPeer(request.TD, hash, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect) {
 			self.blockPool.AddBlock(request.Block, self.id)
@@ -277,7 +303,7 @@ func (self *ethProtocol) handleStatus() error {
 
 	_, _, genesisBlock := self.chainManager.Status()
 
-	if bytes.Compare(status.GenesisBlock, genesisBlock) != 0 {
+	if !bytes.Equal(status.GenesisBlock, genesisBlock) {
 		return self.protoError(ErrGenesisBlockMismatch, "%x (!= %x)", status.GenesisBlock, genesisBlock)
 	}
 
@@ -297,8 +323,8 @@ func (self *ethProtocol) handleStatus() error {
 }
 
 func (self *ethProtocol) requestBlockHashes(from []byte) error {
-	self.peer.Debugf("fetching hashes (%d) %x...\n", blockHashesBatchSize, from[0:4])
-	return p2p.EncodeMsg(self.rw, GetBlockHashesMsg, interface{}(from), uint64(blockHashesBatchSize))
+	self.peer.Debugf("fetching hashes (%d) %x...\n", maxHashes, from[0:4])
+	return p2p.EncodeMsg(self.rw, GetBlockHashesMsg, interface{}(from), uint64(maxHashes))
 }
 
 func (self *ethProtocol) requestBlocks(hashes [][]byte) error {
@@ -306,26 +332,17 @@ func (self *ethProtocol) requestBlocks(hashes [][]byte) error {
 	return p2p.EncodeMsg(self.rw, GetBlocksMsg, ethutil.ByteSliceToInterface(hashes)...)
 }
 
-func (self *ethProtocol) protoError(code int, format string, params ...interface{}) (err *protocolError) {
-	err = ProtocolError(code, format, params...)
-	if err.Fatal() {
-		self.peer.Errorln("err %v", err)
-		// disconnect
-	} else {
-		self.peer.Debugf("fyi %v", err)
-	}
+func (self *ethProtocol) protoError(code int, format string, params ...interface{}) (err *errs.Error) {
+	err = self.errors.New(code, format, params...)
+	err.Log(self.peer.Logger)
 	return
 }
 
-func (self *ethProtocol) protoErrorDisconnect(code int, format string, params ...interface{}) {
-	err := ProtocolError(code, format, params...)
+func (self *ethProtocol) protoErrorDisconnect(err *errs.Error) {
+	err.Log(self.peer.Logger)
 	if err.Fatal() {
-		self.peer.Errorln("err %v", err)
-		// disconnect
-	} else {
-		self.peer.Debugf("fyi %v", err)
+		self.peer.Disconnect(p2p.DiscSubprotocolError)
 	}
-
 }
 
 func (self *ethProtocol) propagateTxs() {

eth/protocol_test.go

@@ -11,13 +11,23 @@ import (
 
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/errs"
 	"github.com/ethereum/go-ethereum/ethutil"
 	ethlogger "github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 )
 
-var sys = ethlogger.NewStdLogSystem(os.Stdout, log.LstdFlags, ethlogger.LogLevel(ethlogger.DebugDetailLevel))
+var logsys = ethlogger.NewStdLogSystem(os.Stdout, log.LstdFlags, ethlogger.LogLevel(ethlogger.DebugDetailLevel))
+
+var ini = false
+
+func logInit() {
+	if !ini {
+		ethlogger.AddLogSystem(logsys)
+		ini = true
+	}
+}
 
 type testMsgReadWriter struct {
 	in  chan p2p.Msg
@@ -64,7 +74,7 @@ type testChainManager struct {
 type testBlockPool struct {
 	addBlockHashes func(next func() ([]byte, bool), peerId string)
 	addBlock       func(block *types.Block, peerId string) (err error)
-	addPeer        func(td *big.Int, currentBlock []byte, peerId string, requestHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(int, string, ...interface{})) (best bool)
+	addPeer        func(td *big.Int, currentBlock []byte, peerId string, requestHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(*errs.Error)) (best bool)
 	removePeer     func(peerId string)
 }
 
@@ -116,7 +126,7 @@ func (self *testBlockPool) AddBlock(block *types.Block, peerId string) {
 	}
 }
 
-func (self *testBlockPool) AddPeer(td *big.Int, currentBlock []byte, peerId string, requestBlockHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(int, string, ...interface{})) (best bool) {
+func (self *testBlockPool) AddPeer(td *big.Int, currentBlock []byte, peerId string, requestBlockHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(*errs.Error)) (best bool) {
 	if self.addPeer != nil {
 		best = self.addPeer(td, currentBlock, peerId, requestBlockHashes, requestBlocks, peerError)
 	}
@@ -169,7 +179,7 @@ func (self *ethProtocolTester) checkError(expCode int, delay time.Duration) (err
 		self.t.Errorf("no error after %v, expected %v", delay, expCode)
 		return
 	}
-	perr, ok := err.(*protocolError)
+	perr, ok := err.(*errs.Error)
 	if ok && perr != nil {
 		if code := perr.Code; code != expCode {
 			self.t.Errorf("expected protocol error (code %v), got %v (%v)", expCode, code, err)

eth/test/README.md

-= Integration tests for eth protocol and blockpool 
-
-This is a simple suite of tests to fire up a local test node with peers to test blockchain synchronisation and download. 
-The scripts call ethereum (assumed to be compiled in go-ethereum root). 
-
-To run a test:
-
-    . run.sh 00 02
-
-Without arguments, all tests are run. 
-
-Peers are launched with preloaded imported chains. In order to prevent them from synchronizing with each other they are set with `-dial=false` and `-maxpeer 1` options. They log into `/tmp/eth.test/nodes/XX` where XX is the last two digits of their port. 
-
-Chains to import can be bootstrapped by letting nodes mine for some time. This is done with 
-
-    . bootstrap.sh 
-
-Only the relative timing and forks matter so they should work if the bootstrap script is rerun. 
-The reference blockchain of tests are soft links to these import chains and check at the end of a test run. 
-
-Connecting to peers and exporting blockchain is scripted with JS files executed by the JSRE, see `tests/XX.sh`. 
-
-Each test is set with a timeout. This may vary on different computers so adjust sensibly. 
-If you kill a test before it completes, do not forget to kill all the background processes, since they will impact the result. Use:
-
-    killall ethereum
-

eth/test/bootstrap.sh

-#!/bin/bash
-# bootstrap chains - used to regenerate tests/chains/*.chain
-
-mkdir -p chains
-bash ./mine.sh 00 10
-bash ./mine.sh 01 5 00
-bash ./mine.sh 02 10 00
-bash ./mine.sh 03 5 02
-bash ./mine.sh 04 10 02
\ No newline at end of file

eth/test/mine.sh

-#!/bin/bash
-# bash ./mine.sh node_id timeout(sec) [basechain]
-ETH=../../ethereum
-MINE="$ETH -datadir tmp/nodes/$1 -seed=false -port '' -shh=false -id test$1"
-rm -rf tmp/nodes/$1
-echo "Creating chain $1..."
-if [[ "" !=  "$3" ]]; then
-  CHAIN="chains/$3.chain"
-  CHAINARG="-chain $CHAIN"
-  $MINE -mine $CHAINARG -loglevel 3 | grep 'importing'
-fi
-$MINE -mine -loglevel 0 &
-PID=$!
-sleep $2
-kill $PID
-$MINE -loglevel 3 <(echo "eth.export(\"chains/$1.chain\")") > /tmp/eth.test/mine.tmp &
-PID=$!
-sleep 1
-kill $PID
-cat /tmp/eth.test/mine.tmp | grep 'exporting'

eth/test/run.sh

-#!/bin/bash
-# bash run.sh (testid0 testid1 ...)
-# runs tests tests/testid0.sh tests/testid1.sh ...
-# without arguments, it runs all tests
-
-. tests/common.sh
-
-TESTS=
-
-if [ "$#" -eq 0 ]; then
-  for NAME in tests/??.sh; do
-    i=`basename $NAME .sh`
-    TESTS="$TESTS $i"
-  done
-else
-  TESTS=$@
-fi
-
-ETH=../../ethereum
-DIR="/tmp/eth.test/nodes"
-TIMEOUT=10
-
-mkdir -p $DIR/js
-
-echo "running tests $TESTS"
-for NAME in $TESTS; do
-  PIDS=
-  CHAIN="tests/$NAME.chain"
-  JSFILE="$DIR/js/$NAME.js"
-  CHAIN_TEST="$DIR/$NAME/chain"
-
-  echo "RUN: test $NAME"
-  cat tests/common.js > $JSFILE
-  . tests/$NAME.sh
-  sleep $TIMEOUT
-  echo "timeout after $TIMEOUT seconds: killing $PIDS"
-  kill $PIDS
-  if [ -r "$CHAIN" ]; then
-    if diff $CHAIN $CHAIN_TEST >/dev/null ; then
-      echo "chain ok: $CHAIN=$CHAIN_TEST"
-    else
-      echo "FAIL: chains differ: expected $CHAIN ; got $CHAIN_TEST"
-      continue
-    fi
-  fi
-  ERRORS=$DIR/errors
-  if [ -r "$ERRORS" ]; then
-    echo "FAIL: "
-    cat $ERRORS
-  else
-    echo PASS
-  fi
-done
\ No newline at end of file

eth/test/tests/00.chain

-../chains/01.chain
\ No newline at end of file

eth/test/tests/00.sh

-#!/bin/bash
-
-TIMEOUT=4
-
-cat >> $JSFILE <<EOF
-eth.addPeer("localhost:30311");
-sleep(1000)
-eth.export("$CHAIN_TEST");
-EOF
-
-peer 11 01
-test_node $NAME "" -loglevel 5 $JSFILE
-

eth/test/tests/01.chain

-../chains/02.chain
\ No newline at end of file

eth/test/tests/01.sh

-#!/bin/bash
-
-TIMEOUT=5
-
-cat >> $JSFILE <<EOF
-eth.addPeer("localhost:30311");
-log("added peer localhost:30311");
-sleep(1000);
-log("added peer localhost:30312");
-eth.addPeer("localhost:30312");
-sleep(3000);
-eth.export("$CHAIN_TEST");
-EOF
-
-peer 11 01
-peer 12 02
-test_node $NAME "" -loglevel 5 $JSFILE
-

eth/test/tests/02.chain

-../chains/01.chain
\ No newline at end of file

eth/test/tests/02.sh

-#!/bin/bash
-
-TIMEOUT=6
-
-cat >> $JSFILE <<EOF
-eth.addPeer("localhost:30311");
-sleep(200);
-eth.addPeer("localhost:30312");
-sleep(3000);
-eth.export("$CHAIN_TEST");
-EOF
-
-peer 11 01
-peer 12 02
-P12ID=$PID
-test_node $NAME "" -loglevel 5 $JSFILE
-sleep 0.3
-kill $P12ID
-

eth/test/tests/03.chain

-../chains/12k.chain
\ No newline at end of file

eth/test/tests/03.sh

-#!/bin/bash
-
-TIMEOUT=12
-
-cat >> $JSFILE <<EOF
-eth.addPeer("localhost:30311");
-sleep(10000);
-eth.export("$CHAIN_TEST");
-EOF
-
-peer 11 12k
-sleep 2
-test_node $NAME "" -loglevel 5 $JSFILE
-

eth/test/tests/04.sh

-#!/bin/bash
-
-TIMEOUT=15
-
-cat >> $JSFILE <<EOF
-eth.addPeer("localhost:30311");
-sleep(200);
-eth.addPeer("localhost:30312");
-sleep(13000);
-eth.export("$CHAIN_TEST");
-EOF
-
-peer 11 01 -mine
-peer 12 02
-test_node $NAME "" -loglevel 5 $JSFILE
-sleep 6
-cat $DIR/$NAME/debug.log | grep 'best peer'

eth/test/tests/05.sh

-#!/bin/bash
-
-TIMEOUT=60
-
-cat >> $JSFILE <<EOF
-eth.addPeer("localhost:30311");
-sleep(200);
-eth.addPeer("localhost:30312");
-eth.addPeer("localhost:30313");
-eth.addPeer("localhost:30314");
-sleep(3000);
-eth.export("$CHAIN_TEST");
-EOF
-
-peer 11 01 -mine
-peer 12 02 -mine
-peer 13 03
-peer 14 04
-test_node $NAME "" -loglevel 5 $JSFILE
-

eth/test/tests/common.js

-function log(text) {
-  console.log("[JS TEST SCRIPT] " + text);
-}
-
-function sleep(seconds) {
-    var now = new Date().getTime();
-    while(new Date().getTime() < now + seconds){}
-}
-

eth/test/tests/common.sh

-#!/bin/bash
-
-# launched by run.sh
-function test_node {
-  rm -rf $DIR/$1
-  ARGS="-datadir $DIR/$1 -debug debug -seed=false -shh=false -id test$1 -port 303$1"
-  if [ "" != "$2" ]; then
-    chain="chains/$2.chain"
-    echo "import chain $chain"
-    $ETH $ARGS -loglevel 3 -chain $chain | grep CLI |grep import
-  fi
-  echo "starting test node $1 with args $ARGS ${@:3}"
-  $ETH $ARGS  ${@:3} &
-  PID=$!
-  PIDS="$PIDS $PID"
-}
-
-function peer {
-  test_node $@ -loglevel 5 -logfile debug.log -maxpeer 1 -dial=false
-}
\ No newline at end of file