core, eth, internal, miner: optimize txpool for quick ops

core/tx_list_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"math/big"
+	"math/rand"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// Tests that transactions can be added to strict lists and list contents and
+// nonce boundaries are correctly maintained.
+func TestStrictTxListAdd(t *testing.T) {
+	// Generate a list of transactions to insert
+	key, _ := crypto.GenerateKey()
+
+	txs := make(types.Transactions, 1024)
+	for i := 0; i < len(txs); i++ {
+		txs[i] = transaction(uint64(i), new(big.Int), key)
+	}
+	// Insert the transactions in a random order
+	list := newTxList(true)
+	for _, v := range rand.Perm(len(txs)) {
+		list.Add(txs[v])
+	}
+	// Verify internal state
+	if list.first != 0 {
+		t.Errorf("lowest nonce mismatch: have %d, want %d", list.first, 0)
+	}
+	if int(list.last) != len(txs)-1 {
+		t.Errorf("highest nonce mismatch: have %d, want %d", list.last, len(txs)-1)
+	}
+	if len(list.items) != len(txs) {
+		t.Errorf("transaction count mismatch: have %d, want %d", len(list.items), len(txs))
+	}
+	for i, tx := range txs {
+		if list.items[tx.Nonce()] != tx {
+			t.Errorf("item %d: transaction mismatch: have %v, want %v", i, list.items[tx.Nonce()], tx)
+		}
+	}
+}

core/types/transaction.go

@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"io"
 	"math/big"
-	"sort"
 	"sync/atomic"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -439,37 +438,29 @@ func (s *TxByPrice) Pop() interface{} {
 // sender accounts and sorts them by nonce. After the account nonce ordering is
 // satisfied, the results are merged back together by price, always comparing only
 // the head transaction from each account. This is done via a heap to keep it fast.
-func SortByPriceAndNonce(txs []*Transaction) {
-	// Separate the transactions by account and sort by nonce
-	byNonce := make(map[common.Address][]*Transaction)
-	for _, tx := range txs {
-		acc, _ := tx.From() // we only sort valid txs so this cannot fail
-		byNonce[acc] = append(byNonce[acc], tx)
-	}
-	for _, accTxs := range byNonce {
-		sort.Sort(TxByNonce(accTxs))
-	}
+func SortByPriceAndNonce(txs map[common.Address]Transactions) Transactions {
 	// Initialize a price based heap with the head transactions
-	byPrice := make(TxByPrice, 0, len(byNonce))
-	for acc, accTxs := range byNonce {
+	byPrice := make(TxByPrice, 0, len(txs))
+	for acc, accTxs := range txs {
 		byPrice = append(byPrice, accTxs[0])
-		byNonce[acc] = accTxs[1:]
+		txs[acc] = accTxs[1:]
 	}
 	heap.Init(&byPrice)
 
 	// Merge by replacing the best with the next from the same account
-	txs = txs[:0]
+	var sorted Transactions
 	for len(byPrice) > 0 {
 		// Retrieve the next best transaction by price
 		best := heap.Pop(&byPrice).(*Transaction)
 
 		// Push in its place the next transaction from the same account
 		acc, _ := best.From() // we only sort valid txs so this cannot fail
-		if accTxs, ok := byNonce[acc]; ok && len(accTxs) > 0 {
+		if accTxs, ok := txs[acc]; ok && len(accTxs) > 0 {
 			heap.Push(&byPrice, accTxs[0])
-			byNonce[acc] = accTxs[1:]
+			txs[acc] = accTxs[1:]
 		}
 		// Accumulate the best priced transaction
-		txs = append(txs, best)
+		sorted = append(sorted, best)
 	}
+	return sorted
 }

core/types/transaction_test.go

@@ -128,15 +128,16 @@ func TestTransactionPriceNonceSort(t *testing.T) {
 		keys[i], _ = crypto.GenerateKey()
 	}
 	// Generate a batch of transactions with overlapping values, but shifted nonces
-	txs := []*Transaction{}
+	groups := map[common.Address]Transactions{}
 	for start, key := range keys {
+		addr := crypto.PubkeyToAddress(key.PublicKey)
 		for i := 0; i < 25; i++ {
 			tx, _ := NewTransaction(uint64(start+i), common.Address{}, big.NewInt(100), big.NewInt(100), big.NewInt(int64(start+i)), nil).SignECDSA(key)
-			txs = append(txs, tx)
+			groups[addr] = append(groups[addr], tx)
 		}
 	}
 	// Sort the transactions and cross check the nonce ordering
-	SortByPriceAndNonce(txs)
+	txs := SortByPriceAndNonce(groups)
 	for i, txi := range txs {
 		fromi, _ := txi.From()
 

eth/api_backend.go

@@ -118,21 +118,25 @@ func (b *EthApiBackend) RemoveTx(txHash common.Hash) {
 	b.eth.txMu.Lock()
 	defer b.eth.txMu.Unlock()
 
-	b.eth.txPool.RemoveTx(txHash)
+	b.eth.txPool.Remove(txHash)
 }
 
 func (b *EthApiBackend) GetPoolTransactions() types.Transactions {
 	b.eth.txMu.Lock()
 	defer b.eth.txMu.Unlock()
 
-	return b.eth.txPool.GetTransactions()
+	var txs types.Transactions
+	for _, batch := range b.eth.txPool.Pending() {
+		txs = append(txs, batch...)
+	}
+	return txs
 }
 
-func (b *EthApiBackend) GetPoolTransaction(txHash common.Hash) *types.Transaction {
+func (b *EthApiBackend) GetPoolTransaction(hash common.Hash) *types.Transaction {
 	b.eth.txMu.Lock()
 	defer b.eth.txMu.Unlock()
 
-	return b.eth.txPool.GetTransaction(txHash)
+	return b.eth.txPool.Get(hash)
 }
 
 func (b *EthApiBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) {
@@ -149,7 +153,7 @@ func (b *EthApiBackend) Stats() (pending int, queued int) {
 	return b.eth.txPool.Stats()
 }
 
-func (b *EthApiBackend) TxPoolContent() (map[common.Address]core.TxList, map[common.Address]core.TxList) {
+func (b *EthApiBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) {
 	b.eth.txMu.Lock()
 	defer b.eth.txMu.Unlock()
 

eth/handler.go

@@ -677,7 +677,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			}
 			p.MarkTransaction(tx.Hash())
 		}
-		pm.txpool.AddTransactions(txs)
+		pm.txpool.AddBatch(txs)
 
 	default:
 		return errResp(ErrInvalidMsgCode, "%v", msg.Code)

eth/helper_test.go

@@ -23,6 +23,7 @@ import (
 	"crypto/ecdsa"
 	"crypto/rand"
 	"math/big"
+	"sort"
 	"sync"
 	"testing"
 
@@ -89,9 +90,9 @@ type testTxPool struct {
 	lock sync.RWMutex // Protects the transaction pool
 }
 
-// AddTransactions appends a batch of transactions to the pool, and notifies any
+// AddBatch appends a batch of transactions to the pool, and notifies any
 // listeners if the addition channel is non nil
-func (p *testTxPool) AddTransactions(txs []*types.Transaction) {
+func (p *testTxPool) AddBatch(txs []*types.Transaction) {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
@@ -101,15 +102,20 @@ func (p *testTxPool) AddTransactions(txs []*types.Transaction) {
 	}
 }
 
-// GetTransactions returns all the transactions known to the pool
-func (p *testTxPool) GetTransactions() types.Transactions {
+// Pending returns all the transactions known to the pool
+func (p *testTxPool) Pending() map[common.Address]types.Transactions {
 	p.lock.RLock()
 	defer p.lock.RUnlock()
 
-	txs := make([]*types.Transaction, len(p.pool))
-	copy(txs, p.pool)
-
-	return txs
+	batches := make(map[common.Address]types.Transactions)
+	for _, tx := range p.pool {
+		from, _ := tx.From()
+		batches[from] = append(batches[from], tx)
+	}
+	for _, batch := range batches {
+		sort.Sort(types.TxByNonce(batch))
+	}
+	return batches
 }
 
 // newTestTransaction create a new dummy transaction.

eth/protocol.go

@@ -97,12 +97,12 @@ var errorToString = map[int]string{
 }
 
 type txPool interface {
-	// AddTransactions should add the given transactions to the pool.
-	AddTransactions([]*types.Transaction)
+	// AddBatch should add the given transactions to the pool.
+	AddBatch([]*types.Transaction)
 
-	// GetTransactions should return pending transactions.
+	// Pending should return pending transactions.
 	// The slice should be modifiable by the caller.
-	GetTransactions() types.Transactions
+	Pending() map[common.Address]types.Transactions
 }
 
 // statusData is the network packet for the status message.

eth/protocol_test.go

@@ -130,7 +130,7 @@ func testSendTransactions(t *testing.T, protocol int) {
 	for nonce := range alltxs {
 		alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), txsize)
 	}
-	pm.txpool.AddTransactions(alltxs)
+	pm.txpool.AddBatch(alltxs)
 
 	// Connect several peers. They should all receive the pending transactions.
 	var wg sync.WaitGroup

eth/sync.go

@@ -45,7 +45,10 @@ type txsync struct {
 
 // syncTransactions starts sending all currently pending transactions to the given peer.
 func (pm *ProtocolManager) syncTransactions(p *peer) {
-	txs := pm.txpool.GetTransactions()
+	var txs types.Transactions
+	for _, batch := range pm.txpool.Pending() {
+		txs = append(txs, batch...)
+	}
 	if len(txs) == 0 {
 		return
 	}

internal/ethapi/backend.go

@@ -58,7 +58,7 @@ type Backend interface {
 	GetPoolTransaction(txHash common.Hash) *types.Transaction
 	GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
 	Stats() (pending int, queued int)
-	TxPoolContent() (map[common.Address]core.TxList, map[common.Address]core.TxList)
+	TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
 }
 
 type State interface {

miner/worker.go

@@ -501,8 +501,7 @@ func (self *worker) commitNewWork() {
 	*/
 
 	//approach 2
-	transactions := self.eth.TxPool().GetTransactions()
-	types.SortByPriceAndNonce(transactions)
+	transactions := types.SortByPriceAndNonce(self.eth.TxPool().Pending())
 
 	/* // approach 3
 	// commit transactions for this run.
@@ -533,8 +532,8 @@ func (self *worker) commitNewWork() {
 
 	work.commitTransactions(self.mux, transactions, self.gasPrice, self.chain)
 
-	self.eth.TxPool().RemoveTransactions(work.lowGasTxs)
-	self.eth.TxPool().RemoveTransactions(work.failedTxs)
+	self.eth.TxPool().RemoveBatch(work.lowGasTxs)
+	self.eth.TxPool().RemoveBatch(work.failedTxs)
 
 	// compute uncles for the new block.
 	var (