cmd/devp2p: add eth66 test suite (#22363)

Co-authored-by: Martin Holst Swende <martin@swende.se>

cmd/devp2p/README.md

@@ -100,7 +100,17 @@ Then, run the following command, replacing `<enode>` with the enode of the geth
 ```
 
 Repeat the above process (re-initialising the node) in order to run the Eth Protocol test suite again.
- 
+
+#### Eth66 Test Suite
+
+The Eth66 test suite is also a conformance test suite for the eth 66 protocol version specifically. 
+To run the eth66 protocol test suite, initialize a geth node as described above and run the following command,
+replacing `<enode>` with the enode of the geth node:
+
+ ```
+ devp2p rlpx eth66-test <enode> cmd/devp2p/internal/ethtest/testdata/chain.rlp cmd/devp2p/internal/ethtest/testdata/genesis.json
+```
+
 [eth]: https://github.com/ethereum/devp2p/blob/master/caps/eth.md
 [dns-tutorial]: https://geth.ethereum.org/docs/developers/dns-discovery-setup
 [discv4]: https://github.com/ethereum/devp2p/tree/master/discv4.md

cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go

+// Copyright 2021 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethtest
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/eth/protocols/eth"
+	"github.com/ethereum/go-ethereum/internal/utesting"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/stretchr/testify/assert"
+)
+
+func (c *Conn) statusExchange66(t *utesting.T, chain *Chain) Message {
+	status := &Status{
+		ProtocolVersion: uint32(66),
+		NetworkID:       chain.chainConfig.ChainID.Uint64(),
+		TD:              chain.TD(chain.Len()),
+		Head:            chain.blocks[chain.Len()-1].Hash(),
+		Genesis:         chain.blocks[0].Hash(),
+		ForkID:          chain.ForkID(),
+	}
+	return c.statusExchange(t, chain, status)
+}
+
+func (s *Suite) dial66(t *utesting.T) *Conn {
+	conn, err := s.dial()
+	if err != nil {
+		t.Fatalf("could not dial: %v", err)
+	}
+	conn.caps = append(conn.caps, p2p.Cap{Name: "eth", Version: 66})
+	return conn
+}
+
+func (c *Conn) write66(req eth.Packet, code int) error {
+	payload, err := rlp.EncodeToBytes(req)
+	if err != nil {
+		return err
+	}
+	_, err = c.Conn.Write(uint64(code), payload)
+	return err
+}
+
+func (c *Conn) read66() (uint64, Message) {
+	code, rawData, _, err := c.Conn.Read()
+	if err != nil {
+		return 0, errorf("could not read from connection: %v", err)
+	}
+
+	var msg Message
+
+	switch int(code) {
+	case (Hello{}).Code():
+		msg = new(Hello)
+
+	case (Ping{}).Code():
+		msg = new(Ping)
+	case (Pong{}).Code():
+		msg = new(Pong)
+	case (Disconnect{}).Code():
+		msg = new(Disconnect)
+	case (Status{}).Code():
+		msg = new(Status)
+	case (GetBlockHeaders{}).Code():
+		ethMsg := new(eth.GetBlockHeadersPacket66)
+		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return ethMsg.RequestId, GetBlockHeaders(*ethMsg.GetBlockHeadersPacket)
+	case (BlockHeaders{}).Code():
+		ethMsg := new(eth.BlockHeadersPacket66)
+		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return ethMsg.RequestId, BlockHeaders(ethMsg.BlockHeadersPacket)
+	case (GetBlockBodies{}).Code():
+		ethMsg := new(eth.GetBlockBodiesPacket66)
+		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return ethMsg.RequestId, GetBlockBodies(ethMsg.GetBlockBodiesPacket)
+	case (BlockBodies{}).Code():
+		ethMsg := new(eth.BlockBodiesPacket66)
+		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return ethMsg.RequestId, BlockBodies(ethMsg.BlockBodiesPacket)
+	case (NewBlock{}).Code():
+		msg = new(NewBlock)
+	case (NewBlockHashes{}).Code():
+		msg = new(NewBlockHashes)
+	case (Transactions{}).Code():
+		msg = new(Transactions)
+	case (NewPooledTransactionHashes{}).Code():
+		msg = new(NewPooledTransactionHashes)
+	default:
+		msg = errorf("invalid message code: %d", code)
+	}
+
+	if msg != nil {
+		if err := rlp.DecodeBytes(rawData, msg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return 0, msg
+	}
+	return 0, errorf("invalid message: %s", string(rawData))
+}
+
+// ReadAndServe serves GetBlockHeaders requests while waiting
+// on another message from the node.
+func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) {
+	start := time.Now()
+	for time.Since(start) < timeout {
+		timeout := time.Now().Add(10 * time.Second)
+		c.SetReadDeadline(timeout)
+
+		reqID, msg := c.read66()
+
+		switch msg := msg.(type) {
+		case *Ping:
+			c.Write(&Pong{})
+		case *GetBlockHeaders:
+			headers, err := chain.GetHeaders(*msg)
+			if err != nil {
+				return 0, errorf("could not get headers for inbound header request: %v", err)
+			}
+
+			if err := c.Write(headers); err != nil {
+				return 0, errorf("could not write to connection: %v", err)
+			}
+		default:
+			return reqID, msg
+		}
+	}
+	return 0, errorf("no message received within %v", timeout)
+}
+
+func (s *Suite) setupConnection66(t *utesting.T) *Conn {
+	// create conn
+	sendConn := s.dial66(t)
+	sendConn.handshake(t)
+	sendConn.statusExchange66(t, s.chain)
+	return sendConn
+}
+
+func (s *Suite) testAnnounce66(t *utesting.T, sendConn, receiveConn *Conn, blockAnnouncement *NewBlock) {
+	// Announce the block.
+	if err := sendConn.Write(blockAnnouncement); err != nil {
+		t.Fatalf("could not write to connection: %v", err)
+	}
+	s.waitAnnounce66(t, receiveConn, blockAnnouncement)
+}
+
+func (s *Suite) waitAnnounce66(t *utesting.T, conn *Conn, blockAnnouncement *NewBlock) {
+	timeout := 20 * time.Second
+	_, msg := conn.readAndServe66(s.chain, timeout)
+	switch msg := msg.(type) {
+	case *NewBlock:
+		t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block))
+		assert.Equal(t,
+			blockAnnouncement.Block.Header(), msg.Block.Header(),
+			"wrong block header in announcement",
+		)
+		assert.Equal(t,
+			blockAnnouncement.TD, msg.TD,
+			"wrong TD in announcement",
+		)
+	case *NewBlockHashes:
+		blockHashes := *msg
+		t.Logf("received NewBlockHashes message: %s", pretty.Sdump(blockHashes))
+		assert.Equal(t, blockAnnouncement.Block.Hash(), blockHashes[0].Hash,
+			"wrong block hash in announcement",
+		)
+	default:
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
+	}
+}
+
+// waitForBlock66 waits for confirmation from the client that it has
+// imported the given block.
+func (c *Conn) waitForBlock66(block *types.Block) error {
+	defer c.SetReadDeadline(time.Time{})
+
+	timeout := time.Now().Add(20 * time.Second)
+	c.SetReadDeadline(timeout)
+	for {
+		req := eth.GetBlockHeadersPacket66{
+			RequestId: 54,
+			GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+				Origin: eth.HashOrNumber{
+					Hash: block.Hash(),
+				},
+				Amount: 1,
+			},
+		}
+		if err := c.write66(req, GetBlockHeaders{}.Code()); err != nil {
+			return err
+		}
+
+		reqID, msg := c.read66()
+		// check message
+		switch msg := msg.(type) {
+		case BlockHeaders:
+			// check request ID
+			if reqID != req.RequestId {
+				return fmt.Errorf("request ID mismatch: wanted %d, got %d", req.RequestId, reqID)
+			}
+			if len(msg) > 0 {
+				return nil
+			}
+			time.Sleep(100 * time.Millisecond)
+		default:
+			return fmt.Errorf("invalid message: %s", pretty.Sdump(msg))
+		}
+	}
+}
+
+func sendSuccessfulTx66(t *utesting.T, s *Suite, tx *types.Transaction) {
+	sendConn := s.setupConnection66(t)
+	sendSuccessfulTxWithConn(t, s, tx, sendConn)
+}
+
+func sendFailingTx66(t *utesting.T, s *Suite, tx *types.Transaction) {
+	sendConn, recvConn := s.setupConnection66(t), s.setupConnection66(t)
+	sendFailingTxWithConns(t, s, tx, sendConn, recvConn)
+}
+
+func (s *Suite) getBlockHeaders66(t *utesting.T, conn *Conn, req eth.Packet, expectedID uint64) BlockHeaders {
+	if err := conn.write66(req, GetBlockHeaders{}.Code()); err != nil {
+		t.Fatalf("could not write to connection: %v", err)
+	}
+	// check block headers response
+	reqID, msg := conn.readAndServe66(s.chain, timeout)
+
+	switch msg := msg.(type) {
+	case BlockHeaders:
+		if reqID != expectedID {
+			t.Fatalf("request ID mismatch: wanted %d, got %d", expectedID, reqID)
+		}
+		return msg
+	default:
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
+		return nil
+	}
+}
+
+func headersMatch(t *utesting.T, chain *Chain, headers BlockHeaders) {
+	for _, header := range headers {
+		num := header.Number.Uint64()
+		t.Logf("received header (%d): %s", num, pretty.Sdump(header.Hash()))
+		assert.Equal(t, chain.blocks[int(num)].Header(), header)
+	}
+}

cmd/devp2p/internal/ethtest/suite.go

@@ -53,29 +53,47 @@ type Suite struct {
 // NewSuite creates and returns a new eth-test suite that can
 // be used to test the given node against the given blockchain
 // data.
-func NewSuite(dest *enode.Node, chainfile string, genesisfile string) *Suite {
+func NewSuite(dest *enode.Node, chainfile string, genesisfile string) (*Suite, error) {
 	chain, err := loadChain(chainfile, genesisfile)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
 	return &Suite{
 		Dest:      dest,
 		chain:     chain.Shorten(1000),
 		fullChain: chain,
-	}
+	}, nil
 }
 
-func (s *Suite) AllTests() []utesting.Test {
+func (s *Suite) EthTests() []utesting.Test {
 	return []utesting.Test{
+		// status
 		{Name: "Status", Fn: s.TestStatus},
+		{Name: "Status_66", Fn: s.TestStatus_66},
+		// get block headers
 		{Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders},
-		{Name: "Broadcast", Fn: s.TestBroadcast},
+		{Name: "GetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66},
+		{Name: "TestSimultaneousRequests_66", Fn: s.TestSimultaneousRequests_66},
+		{Name: "TestSameRequestID_66", Fn: s.TestSameRequestID_66},
+		{Name: "TestZeroRequestID_66", Fn: s.TestZeroRequestID_66},
+		// get block bodies
 		{Name: "GetBlockBodies", Fn: s.TestGetBlockBodies},
+		{Name: "GetBlockBodies_66", Fn: s.TestGetBlockBodies_66},
+		// broadcast
+		{Name: "Broadcast", Fn: s.TestBroadcast},
+		{Name: "Broadcast_66", Fn: s.TestBroadcast_66},
 		{Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce},
+		{Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66},
+		// malicious handshakes + status
 		{Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake},
 		{Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus},
+		{Name: "TestMaliciousHandshake_66", Fn: s.TestMaliciousHandshake_66},
+		{Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus},
+		// test transactions
 		{Name: "TestTransactions", Fn: s.TestTransaction},
+		{Name: "TestTransactions_66", Fn: s.TestTransaction_66},
 		{Name: "TestMaliciousTransactions", Fn: s.TestMaliciousTx},
+		{Name: "TestMaliciousTransactions_66", Fn: s.TestMaliciousTx_66},
 	}
 }
 
@@ -161,7 +179,7 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
 		headers := *msg
 		for _, header := range headers {
 			num := header.Number.Uint64()
-			t.Logf("received header (%d): %s", num, pretty.Sdump(header))
+			t.Logf("received header (%d): %s", num, pretty.Sdump(header.Hash()))
 			assert.Equal(t, s.chain.blocks[int(num)].Header(), header)
 		}
 	default:
@@ -386,20 +404,23 @@ func (s *Suite) setupConnection(t *utesting.T) *Conn {
 // returning the created Conn if successful.
 func (s *Suite) dial() (*Conn, error) {
 	var conn Conn
-
+	// dial
 	fd, err := net.Dial("tcp", fmt.Sprintf("%v:%d", s.Dest.IP(), s.Dest.TCP()))
 	if err != nil {
 		return nil, err
 	}
 	conn.Conn = rlpx.NewConn(fd, s.Dest.Pubkey())
-
 	// do encHandshake
 	conn.ourKey, _ = crypto.GenerateKey()
 	_, err = conn.Handshake(conn.ourKey)
 	if err != nil {
 		return nil, err
 	}
-
+	// set default p2p capabilities
+	conn.caps = []p2p.Cap{
+		{Name: "eth", Version: 64},
+		{Name: "eth", Version: 65},
+	}
 	return &conn, nil
 }
 

cmd/devp2p/internal/ethtest/transaction.go

@@ -30,6 +30,10 @@ var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c666
 
 func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) {
 	sendConn := s.setupConnection(t)
+	sendSuccessfulTxWithConn(t, s, tx, sendConn)
+}
+
+func sendSuccessfulTxWithConn(t *utesting.T, s *Suite, tx *types.Transaction, sendConn *Conn) {
 	t.Logf("sending tx: %v %v %v\n", tx.Hash().String(), tx.GasPrice(), tx.Gas())
 	// Send the transaction
 	if err := sendConn.Write(&Transactions{tx}); err != nil {
@@ -41,20 +45,21 @@ func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) {
 	switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) {
 	case *Transactions:
 		recTxs := *msg
-		if len(recTxs) < 1 {
-			t.Fatalf("received transactions do not match send: %v", recTxs)
-		}
-		if tx.Hash() != recTxs[len(recTxs)-1].Hash() {
-			t.Fatalf("received transactions do not match send: got %v want %v", recTxs, tx)
+		for _, gotTx := range recTxs {
+			if gotTx.Hash() == tx.Hash() {
+				// Ok
+				return
+			}
 		}
+		t.Fatalf("missing transaction: got %v missing %v", recTxs, tx.Hash())
 	case *NewPooledTransactionHashes:
 		txHashes := *msg
-		if len(txHashes) < 1 {
-			t.Fatalf("received transactions do not match send: %v", txHashes)
-		}
-		if tx.Hash() != txHashes[len(txHashes)-1] {
-			t.Fatalf("wrong announcement received, wanted %v got %v", tx, txHashes)
+		for _, gotHash := range txHashes {
+			if gotHash == tx.Hash() {
+				return
+			}
 		}
+		t.Fatalf("missing transaction announcement: got %v missing %v", txHashes, tx.Hash())
 	default:
 		t.Fatalf("unexpected message in sendSuccessfulTx: %s", pretty.Sdump(msg))
 	}
@@ -62,6 +67,10 @@ func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) {
 
 func sendFailingTx(t *utesting.T, s *Suite, tx *types.Transaction) {
 	sendConn, recvConn := s.setupConnection(t), s.setupConnection(t)
+	sendFailingTxWithConns(t, s, tx, sendConn, recvConn)
+}
+
+func sendFailingTxWithConns(t *utesting.T, s *Suite, tx *types.Transaction, sendConn, recvConn *Conn) {
 	// Wait for a transaction announcement
 	switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) {
 	case *NewPooledTransactionHashes:

cmd/devp2p/internal/ethtest/types.go

@@ -125,6 +125,7 @@ type Conn struct {
 	*rlpx.Conn
 	ourKey             *ecdsa.PrivateKey
 	ethProtocolVersion uint
+	caps               []p2p.Cap
 }
 
 func (c *Conn) Read() Message {
@@ -221,11 +222,8 @@ func (c *Conn) handshake(t *utesting.T) Message {
 	pub0 := crypto.FromECDSAPub(&c.ourKey.PublicKey)[1:]
 	ourHandshake := &Hello{
 		Version: 5,
-		Caps: []p2p.Cap{
-			{Name: "eth", Version: 64},
-			{Name: "eth", Version: 65},
-		},
-		ID: pub0,
+		Caps:    c.caps,
+		ID:      pub0,
 	}
 	if err := c.Write(ourHandshake); err != nil {
 		t.Fatalf("could not write to connection: %v", err)

cmd/devp2p/rlpxcmd.go

@@ -94,6 +94,9 @@ func rlpxEthTest(ctx *cli.Context) error {
 	if ctx.NArg() < 3 {
 		exit("missing path to chain.rlp as command-line argument")
 	}
-	suite := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2])
-	return runTests(ctx, suite.AllTests())
+	suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2])
+	if err != nil {
+		exit(err)
+	}
+	return runTests(ctx, suite.EthTests())
 }

go.mod

@@ -48,6 +48,7 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
 	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
 	golang.org/x/text v0.3.3
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4