go.mod: use github.com/holiman/bloomfilter/v2 (#22044)

* deps: use improved bloom filter implementation

* eth/handler, trie: use 4 keys for syncbloom + minor fixes

* eth/protocols, trie: revert change on syncbloom method signature

core/state/snapshot/difflayer.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/steakknife/bloomfilter"
+	bloomfilter "github.com/holiman/bloomfilter/v2"
 )
 
 var (

go.mod

@@ -18,7 +18,6 @@ require (
 	github.com/dlclark/regexp2 v1.2.0 // indirect
 	github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf
 	github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498
-	github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813 // indirect
 	github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c
 	github.com/fatih/color v1.3.0
 	github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc
@@ -32,6 +31,7 @@ require (
 	github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989
 	github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277
 	github.com/hashicorp/golang-lru v0.5.4
+	github.com/holiman/bloomfilter/v2 v2.0.3
 	github.com/holiman/uint256 v1.1.1
 	github.com/huin/goupnp v1.0.0
 	github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883
@@ -54,14 +54,11 @@ require (
 	github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect
 	github.com/shirou/gopsutil v2.20.5+incompatible
 	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4
-	github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570
-	github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
 	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
-	golang.org/x/mobile v0.0.0-20200801112145-973feb4309de // indirect
 	golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
 	golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8
 	golang.org/x/text v0.3.3
@@ -69,5 +66,5 @@ require (
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce
 	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6
 	gopkg.in/urfave/cli.v1 v1.20.0
-	gotest.tools v2.2.0+incompatible
+	gotest.tools v2.2.0+incompatible // indirect
 )

trie/sync.go

@@ -410,7 +410,7 @@ func (s *Sync) children(req *request, object node) ([]*request, error) {
 				// Bloom filter says this might be a duplicate, double check.
 				// If database says yes, then at least the trie node is present
 				// and we hold the assumption that it's NOT legacy contract code.
-				if blob := rawdb.ReadTrieNode(s.database, common.BytesToHash(node)); len(blob) > 0 {
+				if blob := rawdb.ReadTrieNode(s.database, hash); len(blob) > 0 {
 					continue
 				}
 				// False positive, bump fault meter

trie/sync_bloom.go

@@ -19,7 +19,6 @@ package trie
 import (
 	"encoding/binary"
 	"fmt"
-	"math"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -29,7 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
-	"github.com/steakknife/bloomfilter"
+	bloomfilter "github.com/holiman/bloomfilter/v2"
 )
 
 var (
@@ -41,18 +40,6 @@ var (
 	bloomErrorGauge = metrics.NewRegisteredGauge("trie/bloom/error", nil)
 )
 
-// syncBloomHasher is a wrapper around a byte blob to satisfy the interface API
-// requirements of the bloom library used. It's used to convert a trie hash or
-// contract code hash into a 64 bit mini hash.
-type syncBloomHasher []byte
-
-func (f syncBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
-func (f syncBloomHasher) Sum(b []byte) []byte               { panic("not implemented") }
-func (f syncBloomHasher) Reset()                            { panic("not implemented") }
-func (f syncBloomHasher) BlockSize() int                    { panic("not implemented") }
-func (f syncBloomHasher) Size() int                         { return 8 }
-func (f syncBloomHasher) Sum64() uint64                     { return binary.BigEndian.Uint64(f) }
-
 // SyncBloom is a bloom filter used during fast sync to quickly decide if a trie
 // node or contract code already exists on disk or not. It self populates from the
 // provided disk database on creation in a background thread and will only start
@@ -69,7 +56,7 @@ type SyncBloom struct {
 // initializes it from the database. The bloom is hard coded to use 3 filters.
 func NewSyncBloom(memory uint64, database ethdb.Iteratee) *SyncBloom {
 	// Create the bloom filter to track known trie nodes
-	bloom, err := bloomfilter.New(memory*1024*1024*8, 3)
+	bloom, err := bloomfilter.New(memory*1024*1024*8, 4)
 	if err != nil {
 		panic(fmt.Sprintf("failed to create bloom: %v", err))
 	}
@@ -110,12 +97,11 @@ func (b *SyncBloom) init(database ethdb.Iteratee) {
 		// If the database entry is a trie node, add it to the bloom
 		key := it.Key()
 		if len(key) == common.HashLength {
-			b.bloom.Add(syncBloomHasher(key))
+			b.bloom.AddHash(binary.BigEndian.Uint64(key))
 			bloomLoadMeter.Mark(1)
-		}
-		// If the database entry is a contract code, add it to the bloom
-		if ok, hash := rawdb.IsCodeKey(key); ok {
-			b.bloom.Add(syncBloomHasher(hash))
+		} else if ok, hash := rawdb.IsCodeKey(key); ok {
+			// If the database entry is a contract code, add it to the bloom
+			b.bloom.AddHash(binary.BigEndian.Uint64(hash))
 			bloomLoadMeter.Mark(1)
 		}
 		// If enough time elapsed since the last iterator swap, restart
@@ -125,14 +111,14 @@ func (b *SyncBloom) init(database ethdb.Iteratee) {
 			it.Release()
 			it = database.NewIterator(nil, key)
 
-			log.Info("Initializing state bloom", "items", b.bloom.N(), "errorrate", b.errorRate(), "elapsed", common.PrettyDuration(time.Since(start)))
+			log.Info("Initializing state bloom", "items", b.bloom.N(), "errorrate", b.bloom.FalsePosititveProbability(), "elapsed", common.PrettyDuration(time.Since(start)))
 			swap = time.Now()
 		}
 	}
 	it.Release()
 
 	// Mark the bloom filter inited and return
-	log.Info("Initialized state bloom", "items", b.bloom.N(), "errorrate", b.errorRate(), "elapsed", common.PrettyDuration(time.Since(start)))
+	log.Info("Initialized state bloom", "items", b.bloom.N(), "errorrate", b.bloom.FalsePosititveProbability(), "elapsed", common.PrettyDuration(time.Since(start)))
 	atomic.StoreUint32(&b.inited, 1)
 }
 
@@ -141,7 +127,7 @@ func (b *SyncBloom) init(database ethdb.Iteratee) {
 func (b *SyncBloom) meter() {
 	for {
 		// Report the current error ration. No floats, lame, scale it up.
-		bloomErrorGauge.Update(int64(b.errorRate() * 100000))
+		bloomErrorGauge.Update(int64(b.bloom.FalsePosititveProbability() * 100000))
 
 		// Wait one second, but check termination more frequently
 		for i := 0; i < 10; i++ {
@@ -162,7 +148,7 @@ func (b *SyncBloom) Close() error {
 		b.pend.Wait()
 
 		// Wipe the bloom, but mark it "uninited" just in case someone attempts an access
-		log.Info("Deallocated state bloom", "items", b.bloom.N(), "errorrate", b.errorRate())
+		log.Info("Deallocated state bloom", "items", b.bloom.N(), "errorrate", b.bloom.FalsePosititveProbability())
 
 		atomic.StoreUint32(&b.inited, 0)
 		b.bloom = nil
@@ -175,7 +161,7 @@ func (b *SyncBloom) Add(hash []byte) {
 	if atomic.LoadUint32(&b.closed) == 1 {
 		return
 	}
-	b.bloom.Add(syncBloomHasher(hash))
+	b.bloom.AddHash(binary.BigEndian.Uint64(hash))
 	bloomAddMeter.Mark(1)
 }
 
@@ -193,22 +179,9 @@ func (b *SyncBloom) Contains(hash []byte) bool {
 		return true
 	}
 	// Bloom initialized, check the real one and report any successful misses
-	maybe := b.bloom.Contains(syncBloomHasher(hash))
+	maybe := b.bloom.ContainsHash(binary.BigEndian.Uint64(hash))
 	if !maybe {
 		bloomMissMeter.Mark(1)
 	}
 	return maybe
 }
-
-// errorRate calculates the probability of a random containment test returning a
-// false positive.
-//
-// We're calculating it ourselves because the bloom library we used missed a
-// parentheses in the formula and calculates it wrong. And it's discontinued...
-func (b *SyncBloom) errorRate() float64 {
-	k := float64(b.bloom.K())
-	n := float64(b.bloom.N())
-	m := float64(b.bloom.M())
-
-	return math.Pow(1.0-math.Exp((-k)*(n+0.5)/(m-1)), k)
-}