Commit a42b7355 authored by Jeffrey Wilcke's avatar Jeffrey Wilcke Committed by GitHub

Merge pull request #2742 from karalabe/tx-spam-protection

Transaction pool optimizations
parents 49227f65 b4a52513
This diff is collapsed.
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package core
import (
"math/big"
"math/rand"
"testing"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
)
// Tests that transactions can be added to strict lists and list contents and
// nonce boundaries are correctly maintained.
func TestStrictTxListAdd(t *testing.T) {
// Generate a list of transactions to insert
key, _ := crypto.GenerateKey()
txs := make(types.Transactions, 1024)
for i := 0; i < len(txs); i++ {
txs[i] = transaction(uint64(i), new(big.Int), key)
}
// Insert the transactions in a random order
list := newTxList(true)
for _, v := range rand.Perm(len(txs)) {
list.Add(txs[v])
}
// Verify internal state
if len(list.txs.items) != len(txs) {
t.Errorf("transaction count mismatch: have %d, want %d", len(list.txs.items), len(txs))
}
for i, tx := range txs {
if list.txs.items[tx.Nonce()] != tx {
t.Errorf("item %d: transaction mismatch: have %v, want %v", i, list.txs.items[tx.Nonce()], tx)
}
}
}
This diff is collapsed.
This diff is collapsed.
......@@ -24,7 +24,6 @@ import (
"fmt"
"io"
"math/big"
"sort"
"sync/atomic"
"github.com/ethereum/go-ethereum/common"
......@@ -427,49 +426,58 @@ func (s *TxByPrice) Pop() interface{} {
return x
}
// SortByPriceAndNonce sorts the transactions by price in such a way that the
// nonce orderings within a single account are maintained.
//
// Note, this is not as trivial as it seems from the first look as there are three
// different criteria that need to be taken into account (price, nonce, account
// match), which cannot be done with any plain sorting method, as certain items
// cannot be compared without context.
// TransactionsByPriceAndNonce represents a set of transactions that can return
// transactions in a profit-maximising sorted order, while supporting removing
// entire batches of transactions for non-executable accounts.
type TransactionsByPriceAndNonce struct {
txs map[common.Address]Transactions // Per account nonce-sorted list of transactions
heads TxByPrice // Next transaction for each unique account (price heap)
}
// NewTransactionsByPriceAndNonce creates a transaction set that can retrieve
// price sorted transactions in a nonce-honouring way.
//
// This method first sorts the separates the list of transactions into individual
// sender accounts and sorts them by nonce. After the account nonce ordering is
// satisfied, the results are merged back together by price, always comparing only
// the head transaction from each account. This is done via a heap to keep it fast.
func SortByPriceAndNonce(txs []*Transaction) {
// Separate the transactions by account and sort by nonce
byNonce := make(map[common.Address][]*Transaction)
for _, tx := range txs {
acc, _ := tx.From() // we only sort valid txs so this cannot fail
byNonce[acc] = append(byNonce[acc], tx)
}
for _, accTxs := range byNonce {
sort.Sort(TxByNonce(accTxs))
}
// Note, the input map is reowned so the caller should not interact any more with
// if after providng it to the constructor.
func NewTransactionsByPriceAndNonce(txs map[common.Address]Transactions) *TransactionsByPriceAndNonce {
// Initialize a price based heap with the head transactions
byPrice := make(TxByPrice, 0, len(byNonce))
for acc, accTxs := range byNonce {
byPrice = append(byPrice, accTxs[0])
byNonce[acc] = accTxs[1:]
heads := make(TxByPrice, 0, len(txs))
for acc, accTxs := range txs {
heads = append(heads, accTxs[0])
txs[acc] = accTxs[1:]
}
heap.Init(&heads)
// Assemble and return the transaction set
return &TransactionsByPriceAndNonce{
txs: txs,
heads: heads,
}
heap.Init(&byPrice)
// Merge by replacing the best with the next from the same account
txs = txs[:0]
for len(byPrice) > 0 {
// Retrieve the next best transaction by price
best := heap.Pop(&byPrice).(*Transaction)
// Push in its place the next transaction from the same account
acc, _ := best.From() // we only sort valid txs so this cannot fail
if accTxs, ok := byNonce[acc]; ok && len(accTxs) > 0 {
heap.Push(&byPrice, accTxs[0])
byNonce[acc] = accTxs[1:]
}
// Peek returns the next transaction by price.
func (t *TransactionsByPriceAndNonce) Peek() *Transaction {
if len(t.heads) == 0 {
return nil
}
// Accumulate the best priced transaction
txs = append(txs, best)
return t.heads[0]
}
// Shift replaces the current best head with the next one from the same account.
func (t *TransactionsByPriceAndNonce) Shift() {
acc, _ := t.heads[0].From() // we only sort valid txs so this cannot fail
if txs, ok := t.txs[acc]; ok && len(txs) > 0 {
t.heads[0], t.txs[acc] = txs[0], txs[1:]
heap.Fix(&t.heads, 0)
} else {
heap.Pop(&t.heads)
}
}
// Pop removes the best transaction, *not* replacing it with the next one from
// the same account. This should be used when a transaction cannot be executed
// and hence all subsequent ones should be discarded from the same account.
func (t *TransactionsByPriceAndNonce) Pop() {
heap.Pop(&t.heads)
}
......@@ -128,15 +128,25 @@ func TestTransactionPriceNonceSort(t *testing.T) {
keys[i], _ = crypto.GenerateKey()
}
// Generate a batch of transactions with overlapping values, but shifted nonces
txs := []*Transaction{}
groups := map[common.Address]Transactions{}
for start, key := range keys {
addr := crypto.PubkeyToAddress(key.PublicKey)
for i := 0; i < 25; i++ {
tx, _ := NewTransaction(uint64(start+i), common.Address{}, big.NewInt(100), big.NewInt(100), big.NewInt(int64(start+i)), nil).SignECDSA(key)
txs = append(txs, tx)
groups[addr] = append(groups[addr], tx)
}
}
// Sort the transactions and cross check the nonce ordering
SortByPriceAndNonce(txs)
txset := NewTransactionsByPriceAndNonce(groups)
txs := Transactions{}
for {
if tx := txset.Peek(); tx != nil {
txs = append(txs, tx)
txset.Shift()
}
break
}
for i, txi := range txs {
fromi, _ := txi.From()
......
......@@ -118,21 +118,25 @@ func (b *EthApiBackend) RemoveTx(txHash common.Hash) {
b.eth.txMu.Lock()
defer b.eth.txMu.Unlock()
b.eth.txPool.RemoveTx(txHash)
b.eth.txPool.Remove(txHash)
}
func (b *EthApiBackend) GetPoolTransactions() types.Transactions {
b.eth.txMu.Lock()
defer b.eth.txMu.Unlock()
return b.eth.txPool.GetTransactions()
var txs types.Transactions
for _, batch := range b.eth.txPool.Pending() {
txs = append(txs, batch...)
}
return txs
}
func (b *EthApiBackend) GetPoolTransaction(txHash common.Hash) *types.Transaction {
func (b *EthApiBackend) GetPoolTransaction(hash common.Hash) *types.Transaction {
b.eth.txMu.Lock()
defer b.eth.txMu.Unlock()
return b.eth.txPool.GetTransaction(txHash)
return b.eth.txPool.Get(hash)
}
func (b *EthApiBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) {
......@@ -149,7 +153,7 @@ func (b *EthApiBackend) Stats() (pending int, queued int) {
return b.eth.txPool.Stats()
}
func (b *EthApiBackend) TxPoolContent() (map[common.Address]map[uint64][]*types.Transaction, map[common.Address]map[uint64][]*types.Transaction) {
func (b *EthApiBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) {
b.eth.txMu.Lock()
defer b.eth.txMu.Unlock()
......
......@@ -677,7 +677,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
p.MarkTransaction(tx.Hash())
}
pm.txpool.AddTransactions(txs)
pm.txpool.AddBatch(txs)
default:
return errResp(ErrInvalidMsgCode, "%v", msg.Code)
......
......@@ -23,6 +23,7 @@ import (
"crypto/ecdsa"
"crypto/rand"
"math/big"
"sort"
"sync"
"testing"
......@@ -89,9 +90,9 @@ type testTxPool struct {
lock sync.RWMutex // Protects the transaction pool
}
// AddTransactions appends a batch of transactions to the pool, and notifies any
// AddBatch appends a batch of transactions to the pool, and notifies any
// listeners if the addition channel is non nil
func (p *testTxPool) AddTransactions(txs []*types.Transaction) {
func (p *testTxPool) AddBatch(txs []*types.Transaction) {
p.lock.Lock()
defer p.lock.Unlock()
......@@ -101,15 +102,20 @@ func (p *testTxPool) AddTransactions(txs []*types.Transaction) {
}
}
// GetTransactions returns all the transactions known to the pool
func (p *testTxPool) GetTransactions() types.Transactions {
// Pending returns all the transactions known to the pool
func (p *testTxPool) Pending() map[common.Address]types.Transactions {
p.lock.RLock()
defer p.lock.RUnlock()
txs := make([]*types.Transaction, len(p.pool))
copy(txs, p.pool)
return txs
batches := make(map[common.Address]types.Transactions)
for _, tx := range p.pool {
from, _ := tx.From()
batches[from] = append(batches[from], tx)
}
for _, batch := range batches {
sort.Sort(types.TxByNonce(batch))
}
return batches
}
// newTestTransaction create a new dummy transaction.
......
......@@ -97,12 +97,12 @@ var errorToString = map[int]string{
}
type txPool interface {
// AddTransactions should add the given transactions to the pool.
AddTransactions([]*types.Transaction)
// AddBatch should add the given transactions to the pool.
AddBatch([]*types.Transaction)
// GetTransactions should return pending transactions.
// Pending should return pending transactions.
// The slice should be modifiable by the caller.
GetTransactions() types.Transactions
Pending() map[common.Address]types.Transactions
}
// statusData is the network packet for the status message.
......
......@@ -130,7 +130,7 @@ func testSendTransactions(t *testing.T, protocol int) {
for nonce := range alltxs {
alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), txsize)
}
pm.txpool.AddTransactions(alltxs)
pm.txpool.AddBatch(alltxs)
// Connect several peers. They should all receive the pending transactions.
var wg sync.WaitGroup
......
......@@ -45,7 +45,10 @@ type txsync struct {
// syncTransactions starts sending all currently pending transactions to the given peer.
func (pm *ProtocolManager) syncTransactions(p *peer) {
txs := pm.txpool.GetTransactions()
var txs types.Transactions
for _, batch := range pm.txpool.Pending() {
txs = append(txs, batch...)
}
if len(txs) == 0 {
return
}
......
......@@ -100,32 +100,26 @@ func NewPublicTxPoolAPI(b Backend) *PublicTxPoolAPI {
}
// Content returns the transactions contained within the transaction pool.
func (s *PublicTxPoolAPI) Content() map[string]map[string]map[string][]*RPCTransaction {
content := map[string]map[string]map[string][]*RPCTransaction{
"pending": make(map[string]map[string][]*RPCTransaction),
"queued": make(map[string]map[string][]*RPCTransaction),
func (s *PublicTxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction {
content := map[string]map[string]map[string]*RPCTransaction{
"pending": make(map[string]map[string]*RPCTransaction),
"queued": make(map[string]map[string]*RPCTransaction),
}
pending, queue := s.b.TxPoolContent()
// Flatten the pending transactions
for account, batches := range pending {
dump := make(map[string][]*RPCTransaction)
for nonce, txs := range batches {
nonce := fmt.Sprintf("%d", nonce)
for _, tx := range txs {
dump[nonce] = append(dump[nonce], newRPCPendingTransaction(tx))
}
for account, txs := range pending {
dump := make(map[string]*RPCTransaction)
for nonce, tx := range txs {
dump[fmt.Sprintf("%d", nonce)] = newRPCPendingTransaction(tx)
}
content["pending"][account.Hex()] = dump
}
// Flatten the queued transactions
for account, batches := range queue {
dump := make(map[string][]*RPCTransaction)
for nonce, txs := range batches {
nonce := fmt.Sprintf("%d", nonce)
for _, tx := range txs {
dump[nonce] = append(dump[nonce], newRPCPendingTransaction(tx))
}
for account, txs := range queue {
dump := make(map[string]*RPCTransaction)
for nonce, tx := range txs {
dump[fmt.Sprintf("%d", nonce)] = newRPCPendingTransaction(tx)
}
content["queued"][account.Hex()] = dump
}
......@@ -143,10 +137,10 @@ func (s *PublicTxPoolAPI) Status() map[string]*rpc.HexNumber {
// Inspect retrieves the content of the transaction pool and flattens it into an
// easily inspectable list.
func (s *PublicTxPoolAPI) Inspect() map[string]map[string]map[string][]string {
content := map[string]map[string]map[string][]string{
"pending": make(map[string]map[string][]string),
"queued": make(map[string]map[string][]string),
func (s *PublicTxPoolAPI) Inspect() map[string]map[string]map[string]string {
content := map[string]map[string]map[string]string{
"pending": make(map[string]map[string]string),
"queued": make(map[string]map[string]string),
}
pending, queue := s.b.TxPoolContent()
......@@ -158,24 +152,18 @@ func (s *PublicTxPoolAPI) Inspect() map[string]map[string]map[string][]string {
return fmt.Sprintf("contract creation: %v wei + %v × %v gas", tx.Value(), tx.Gas(), tx.GasPrice())
}
// Flatten the pending transactions
for account, batches := range pending {
dump := make(map[string][]string)
for nonce, txs := range batches {
nonce := fmt.Sprintf("%d", nonce)
for _, tx := range txs {
dump[nonce] = append(dump[nonce], format(tx))
}
for account, txs := range pending {
dump := make(map[string]string)
for nonce, tx := range txs {
dump[fmt.Sprintf("%d", nonce)] = format(tx)
}
content["pending"][account.Hex()] = dump
}
// Flatten the queued transactions
for account, batches := range queue {
dump := make(map[string][]string)
for nonce, txs := range batches {
nonce := fmt.Sprintf("%d", nonce)
for _, tx := range txs {
dump[nonce] = append(dump[nonce], format(tx))
}
for account, txs := range queue {
dump := make(map[string]string)
for nonce, tx := range txs {
dump[fmt.Sprintf("%d", nonce)] = format(tx)
}
content["queued"][account.Hex()] = dump
}
......
......@@ -58,7 +58,7 @@ type Backend interface {
GetPoolTransaction(txHash common.Hash) *types.Transaction
GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
Stats() (pending int, queued int)
TxPoolContent() (map[common.Address]map[uint64][]*types.Transaction, map[common.Address]map[uint64][]*types.Transaction)
TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
}
type State interface {
......
......@@ -68,12 +68,10 @@ type Work struct {
ancestors *set.Set // ancestor set (used for checking uncle parent validity)
family *set.Set // family set (used for checking uncle invalidity)
uncles *set.Set // uncle set
remove *set.Set // tx which will be removed
tcount int // tx count in cycle
ignoredTransactors *set.Set
lowGasTransactors *set.Set
ownedAccounts *set.Set
lowGasTxs types.Transactions
failedTxs types.Transactions
localMinedBlocks *uint64RingBuffer // the most recent block numbers that were mined locally (used to check block inclusion)
Block *types.Block // the new block
......@@ -236,7 +234,12 @@ func (self *worker) update() {
// Apply transaction to the pending state if we're not mining
if atomic.LoadInt32(&self.mining) == 0 {
self.currentMu.Lock()
self.current.commitTransactions(self.mux, types.Transactions{ev.Tx}, self.gasPrice, self.chain)
acc, _ := ev.Tx.From()
txs := map[common.Address]types.Transactions{acc: types.Transactions{ev.Tx}}
txset := types.NewTransactionsByPriceAndNonce(txs)
self.current.commitTransactions(self.mux, txset, self.gasPrice, self.chain)
self.currentMu.Unlock()
}
}
......@@ -383,10 +386,7 @@ func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error
accounts := self.eth.AccountManager().Accounts()
// Keep track of transactions which return errors so they can be removed
work.remove = set.New()
work.tcount = 0
work.ignoredTransactors = set.New()
work.lowGasTransactors = set.New()
work.ownedAccounts = accountAddressesSet(accounts)
if self.current != nil {
work.localMinedBlocks = self.current.localMinedBlocks
......@@ -495,45 +495,11 @@ func (self *worker) commitNewWork() {
if self.config.DAOForkSupport && self.config.DAOForkBlock != nil && self.config.DAOForkBlock.Cmp(header.Number) == 0 {
core.ApplyDAOHardFork(work.state)
}
txs := types.NewTransactionsByPriceAndNonce(self.eth.TxPool().Pending())
work.commitTransactions(self.mux, txs, self.gasPrice, self.chain)
/* //approach 1
transactions := self.eth.TxPool().GetTransactions()
sort.Sort(types.TxByNonce(transactions))
*/
//approach 2
transactions := self.eth.TxPool().GetTransactions()
types.SortByPriceAndNonce(transactions)
/* // approach 3
// commit transactions for this run.
txPerOwner := make(map[common.Address]types.Transactions)
// Sort transactions by owner
for _, tx := range self.eth.TxPool().GetTransactions() {
from, _ := tx.From() // we can ignore the sender error
txPerOwner[from] = append(txPerOwner[from], tx)
}
var (
singleTxOwner types.Transactions
multiTxOwner types.Transactions
)
// Categorise transactions by
// 1. 1 owner tx per block
// 2. multi txs owner per block
for _, txs := range txPerOwner {
if len(txs) == 1 {
singleTxOwner = append(singleTxOwner, txs[0])
} else {
multiTxOwner = append(multiTxOwner, txs...)
}
}
sort.Sort(types.TxByPrice(singleTxOwner))
sort.Sort(types.TxByNonce(multiTxOwner))
transactions := append(singleTxOwner, multiTxOwner...)
*/
work.commitTransactions(self.mux, transactions, self.gasPrice, self.chain)
self.eth.TxPool().RemoveTransactions(work.lowGasTxs)
self.eth.TxPool().RemoveBatch(work.lowGasTxs)
self.eth.TxPool().RemoveBatch(work.failedTxs)
// compute uncles for the new block.
var (
......@@ -591,65 +557,51 @@ func (self *worker) commitUncle(work *Work, uncle *types.Header) error {
return nil
}
func (env *Work) commitTransactions(mux *event.TypeMux, transactions types.Transactions, gasPrice *big.Int, bc *core.BlockChain) {
func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, gasPrice *big.Int, bc *core.BlockChain) {
gp := new(core.GasPool).AddGas(env.header.GasLimit)
var coalescedLogs vm.Logs
for _, tx := range transactions {
for {
// Retrieve the next transaction and abort if all done
tx := txs.Peek()
if tx == nil {
break
}
// Error may be ignored here. The error has already been checked
// during transaction acceptance is the transaction pool.
from, _ := tx.From()
// Check if it falls within margin. Txs from owned accounts are always processed.
// Ignore any transactions (and accounts subsequently) with low gas limits
if tx.GasPrice().Cmp(gasPrice) < 0 && !env.ownedAccounts.Has(from) {
// ignore the transaction and transactor. We ignore the transactor
// because nonce will fail after ignoring this transaction so there's
// no point
env.lowGasTransactors.Add(from)
glog.V(logger.Info).Infof("transaction(%x) below gas price (tx=%v ask=%v). All sequential txs from this address(%x) will be ignored\n", tx.Hash().Bytes()[:4], common.CurrencyToString(tx.GasPrice()), common.CurrencyToString(gasPrice), from[:4])
}
// Pop the current low-priced transaction without shifting in the next from the account
glog.V(logger.Info).Infof("Transaction (%x) below gas price (tx=%v ask=%v). All sequential txs from this address(%x) will be ignored\n", tx.Hash().Bytes()[:4], common.CurrencyToString(tx.GasPrice()), common.CurrencyToString(gasPrice), from[:4])
// Continue with the next transaction if the transaction sender is included in
// the low gas tx set. This will also remove the tx and all sequential transaction
// from this transactor
if env.lowGasTransactors.Has(from) {
// add tx to the low gas set. This will be removed at the end of the run
// owned accounts are ignored
if !env.ownedAccounts.Has(from) {
env.lowGasTxs = append(env.lowGasTxs, tx)
}
continue
}
txs.Pop()
// Move on to the next transaction when the transactor is in ignored transactions set
// This may occur when a transaction hits the gas limit. When a gas limit is hit and
// the transaction is processed (that could potentially be included in the block) it
// will throw a nonce error because the previous transaction hasn't been processed.
// Therefor we need to ignore any transaction after the ignored one.
if env.ignoredTransactors.Has(from) {
continue
}
// Start executing the transaction
env.state.StartRecord(tx.Hash(), common.Hash{}, 0)
err, logs := env.commitTransaction(tx, bc, gp)
switch {
case core.IsGasLimitErr(err):
// ignore the transactor so no nonce errors will be thrown for this account
// next time the worker is run, they'll be picked up again.
env.ignoredTransactors.Add(from)
// Pop the current out-of-gas transaction without shifting in the next from the account
glog.V(logger.Detail).Infof("Gas limit reached for (%x) in this block. Continue to try smaller txs\n", from[:4])
txs.Pop()
case err != nil:
env.remove.Add(tx.Hash())
// Pop the current failed transaction without shifting in the next from the account
glog.V(logger.Detail).Infof("Transaction (%x) failed, will be removed: %v\n", tx.Hash().Bytes()[:4], err)
env.failedTxs = append(env.failedTxs, tx)
txs.Pop()
if glog.V(logger.Detail) {
glog.Infof("TX (%x) failed, will be removed: %v\n", tx.Hash().Bytes()[:4], err)
}
default:
env.tcount++
// Everything ok, collect the logs and shift in the next transaction from the same account
coalescedLogs = append(coalescedLogs, logs...)
env.tcount++
txs.Shift()
}
}
if len(coalescedLogs) > 0 || env.tcount > 0 {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment