Skip to content

G
Geth-Modification
Unverified Commit f585f9ee authored by Péter Szilágyi's avatar Péter Szilágyi
Browse files
Options

core, eth: clean up bloom filtering, add some tests

parent 4ea4d2dc
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1584 additions and 1035 deletions
core/bloombits/fetcher_test.go → core/bloombits/doc.go
View file @ f585f9ee
...@@ -13,89 +13,6 @@ ...@@ -13,89 +13,6 @@
// //
// You should have received a copy of the GNU Lesser General Public License // You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits
import (
"bytes"
"encoding/binary"
"math/rand"
"sync"
"sync/atomic"
"testing"
"time"
)
const testFetcherReqCount = 5000
func fetcherTestVector(b uint, s uint64) []byte {
r := make([]byte, 10)
binary.BigEndian.PutUint16(r[0:2], uint16(b))
binary.BigEndian.PutUint64(r[2:10], s)
return r
}
func TestFetcher(t *testing.T) {
testFetcher(t, 1)
}
func TestFetcherMultipleReaders(t *testing.T) {
testFetcher(t, 10)
}
func testFetcher(t *testing.T, cnt int) { // Package bloombits implements bloom filtering on batches of data.
f := &fetcher{ package bloombits
requestMap: make(map[uint64]fetchRequest),
}
distCh := make(chan distRequest, channelCap)
stop := make(chan struct{})
var reqCount uint32
for i := 0; i < 10; i++ {
go func() {
for {
req, ok := <-distCh
if !ok {
return
}
time.Sleep(time.Duration(rand.Intn(100000)))
atomic.AddUint32(&reqCount, 1)
f.deliver([]uint64{req.sectionIndex}, [][]byte{fetcherTestVector(req.bloomIndex, req.sectionIndex)})
}
}()
}
var wg, wg2 sync.WaitGroup
for cc := 0; cc < cnt; cc++ {
wg.Add(1)
in := make(chan uint64, channelCap)
out := f.fetch(in, distCh, stop, &wg2)
time.Sleep(time.Millisecond * 10 * time.Duration(cc))
go func() {
for i := uint64(0); i < testFetcherReqCount; i++ {
in <- i
}
}()
go func() {
for i := uint64(0); i < testFetcherReqCount; i++ {
bv := <-out
if !bytes.Equal(bv, fetcherTestVector(0, i)) {
if len(bv) != 10 {
t.Errorf("Vector #%d length is %d, expected 10", i, len(bv))
} else {
j := binary.BigEndian.Uint64(bv[2:10])
t.Errorf("Expected vector #%d, fetched #%d", i, j)
}
}
}
wg.Done()
}()
}
wg.Wait()
close(stop)
if reqCount != testFetcherReqCount {
t.Errorf("Request count mismatch: expected %v, got %v", testFetcherReqCount, reqCount)
}
}
core/bloombits/generator.go 0 → 100644
View file @ f585f9ee
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits
import (
"errors"
"github.com/ethereum/go-ethereum/core/types"
)
// errSectionOutOfBounds is returned if the user tried to add more bloom filters
// to the batch than available space, or if tries to retrieve above the capacity,
var errSectionOutOfBounds = errors.New("section out of bounds")
// Generator takes a number of bloom filters and generates the rotated bloom bits
// to be used for batched filtering.
type Generator struct {
blooms [types.BloomBitLength][]byte // Rotated blooms for per-bit matching
sections uint // Number of sections to batch together
nextBit uint // Next bit to set when adding a bloom
}
// NewGenerator creates a rotated bloom generator that can iteratively fill a
// batched bloom filter's bits.
func NewGenerator(sections uint) (*Generator, error) {
if sections%8 != 0 {
return nil, errors.New("section count not multiple of 8")
}
b := &Generator{sections: sections}
for i := 0; i < types.BloomBitLength; i++ {
b.blooms[i] = make([]byte, sections/8)
}
return b, nil
}
// AddBloom takes a single bloom filter and sets the corresponding bit column
// in memory accordingly.
func (b *Generator) AddBloom(bloom types.Bloom) error {
// Make sure we're not adding more bloom filters than our capacity
if b.nextBit >= b.sections {
return errSectionOutOfBounds
}
// Rotate the bloom and insert into our collection
byteMask := b.nextBit / 8
bitMask := byte(1) << byte(7-b.nextBit%8)
for i := 0; i < types.BloomBitLength; i++ {
bloomByteMask := types.BloomByteLength - 1 - i/8
bloomBitMask := byte(1) << byte(i%8)
if (bloom[bloomByteMask] & bloomBitMask) != 0 {
b.blooms[i][byteMask] |= bitMask
}
}
b.nextBit++
return nil
}
// Bitset returns the bit vector belonging to the given bit index after all
// blooms have been added.
func (b *Generator) Bitset(idx uint) ([]byte, error) {
if b.nextBit != b.sections {
return nil, errors.New("bloom not fully generated yet")
}
if idx >= b.sections {
return nil, errSectionOutOfBounds
}
return b.blooms[idx], nil
}
core/bloombits/utils.go → core/bloombits/generator_test.go
View file @ f585f9ee
...@@ -13,51 +13,48 @@ ...@@ -13,51 +13,48 @@
// //
// You should have received a copy of the GNU Lesser General Public License // You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits package bloombits
import ( import (
"bytes"
"math/rand"
"testing"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
) )
const BloomLength = 2048 // Tests that batched bloom bits are correctly rotated from the input bloom
// filters.
func TestGenerator(t *testing.T) {
// Generate the input and the rotated output
var input, output [types.BloomBitLength][types.BloomByteLength]byte
// BloomBitsCreator takes SectionSize number of header bloom filters and calculates the bloomBits vectors of the section for i := 0; i < types.BloomBitLength; i++ {
type BloomBitsCreator struct { for j := 0; j < types.BloomBitLength; j++ {
blooms [BloomLength][]byte bit := byte(rand.Int() % 2)
sectionSize, bitIndex uint64
}
func NewBloomBitsCreator(sectionSize uint64) *BloomBitsCreator { input[i][j/8] |= bit << byte(7-j%8)
b := &BloomBitsCreator{sectionSize: sectionSize} output[types.BloomBitLength-1-j][i/8] |= bit << byte(7-i%8)
for i, _ := range b.blooms { }
b.blooms[i] = make([]byte, sectionSize/8)
} }
return b // Crunch the input through the generator and verify the result
} gen, err := NewGenerator(types.BloomBitLength)
if err != nil {
// AddHeaderBloom takes a single bloom filter and sets the corresponding bit column in memory accordingly t.Fatalf("failed to create bloombit generator: %v", err)
func (b *BloomBitsCreator) AddHeaderBloom(bloom types.Bloom) {
if b.bitIndex >= b.sectionSize {
panic("too many header blooms added")
} }
for i, bloom := range input {
byteIdx := b.bitIndex / 8 if err := gen.AddBloom(bloom); err != nil {
bitMask := byte(1) << byte(7-b.bitIndex%8) t.Fatalf("bloom %d: failed to add: %v", i, err)
for bloomBitIdx, _ := range b.blooms {
bloomByteIdx := BloomLength/8 - 1 - bloomBitIdx/8
bloomBitMask := byte(1) << byte(bloomBitIdx%8)
if (bloom[bloomByteIdx] & bloomBitMask) != 0 {
b.blooms[bloomBitIdx][byteIdx] |= bitMask
} }
} }
b.bitIndex++ for i, want := range output {
} have, err := gen.Bitset(uint(i))
if err != nil {
// GetBitVector returns the bit vector belonging to the given bit index after header blooms have been added t.Fatalf("output %d: failed to retrieve bits: %v", i, err)
func (b *BloomBitsCreator) GetBitVector(idx uint) []byte { }
if b.bitIndex != b.sectionSize { if !bytes.Equal(have, want[:]) {
panic("not enough header blooms added") t.Errorf("output %d: bit vector mismatch have %x, want %x", i, have, want)
}
} }
return b.blooms[idx][:]
} }
core/bloombits/matcher.go
View file @ f585f9ee
This diff is collapsed.
core/bloombits/matcher_test.go
View file @ f585f9ee
This diff is collapsed.
core/bloombits/scheduler.go 0 → 100644
View file @ f585f9ee
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits
import (
"sync"
)
// request represents a bloom retrieval task to prioritize and pull from the local
// database or remotely from the network.
type request struct {
section uint64 // Section index to retrieve the a bit-vector from
bit uint // Bit index within the section to retrieve the vector of
}
// response represents the state of a requested bit-vector through a scheduler.
type response struct {
cached []byte // Cached bits to dedup multiple requests
done chan struct{} // Channel to allow waiting for completion
}
// scheduler handles the scheduling of bloom-filter retrieval operations for
// entire section-batches belonging to a single bloom bit. Beside scheduling the
// retrieval operations, this struct also deduplicates the requests and caches
// the results to minimize network/database overhead even in complex filtering
// scenarios.
type scheduler struct {
bit uint // Index of the bit in the bloom filter this scheduler is responsible for
responses map[uint64]*response // Currently pending retrieval requests or already cached responses
lock sync.Mutex // Lock protecting the responses from concurrent access
}
// newScheduler creates a new bloom-filter retrieval scheduler for a specific
// bit index.
func newScheduler(idx uint) *scheduler {
return &scheduler{
bit: idx,
responses: make(map[uint64]*response),
}
}
// run creates a retrieval pipeline, receiving section indexes from sections and
// returning the results in the same order through the done channel. Concurrent
// runs of the same scheduler are allowed, leading to retrieval task deduplication.
func (s *scheduler) run(sections chan uint64, dist chan *request, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) {
// Create a forwarder channel between requests and responses of the same size as
// the distribution channel (since that will block the pipeline anyway).
pend := make(chan uint64, cap(dist))
// Start the pipeline schedulers to forward between user -> distributor -> user
wg.Add(2)
go s.scheduleRequests(sections, dist, pend, quit, wg)
go s.scheduleDeliveries(pend, done, quit, wg)
}
// reset cleans up any leftovers from previous runs. This is required before a
// restart to ensure the no previously requested but never delivered state will
// cause a lockup.
func (s *scheduler) reset() {
s.lock.Lock()
defer s.lock.Unlock()
for section, res := range s.responses {
if res.cached == nil {
delete(s.responses, section)
}
}
}
// scheduleRequests reads section retrieval requests from the input channel,
// deduplicates the stream and pushes unique retrieval tasks into the distribution
// channel for a database or network layer to honour.
func (s *scheduler) scheduleRequests(reqs chan uint64, dist chan *request, pend chan uint64, quit chan struct{}, wg *sync.WaitGroup) {
// Clean up the goroutine and pipeline when done
defer wg.Done()
defer close(pend)
// Keep reading and scheduling section requests
for {
select {
case <-quit:
return
case section, ok := <-reqs:
// New section retrieval requested
if !ok {
return
}
// Deduplicate retrieval requests
unique := false
s.lock.Lock()
if s.responses[section] == nil {
s.responses[section] = &response{
done: make(chan struct{}),
}
unique = true
}
s.lock.Unlock()
// Schedule the section for retrieval and notify the deliverer to expect this section
if unique {
select {
case <-quit:
return
case dist <- &request{bit: s.bit, section: section}:
}
}
select {
case <-quit:
return
case pend <- section:
}
}
}
}
// scheduleDeliveries reads section acceptance notifications and waits for them
// to be delivered, pushing them into the output data buffer.
func (s *scheduler) scheduleDeliveries(pend chan uint64, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) {
// Clean up the goroutine and pipeline when done
defer wg.Done()
defer close(done)
// Keep reading notifications and scheduling deliveries
for {
select {
case <-quit:
return
case idx, ok := <-pend:
// New section retrieval pending
if !ok {
return
}
// Wait until the request is honoured
s.lock.Lock()
res := s.responses[idx]
s.lock.Unlock()
select {
case <-quit:
return
case <-res.done:
}
// Deliver the result
select {
case <-quit:
return
case done <- res.cached:
}
}
}
}
// deliver is called by the request distributor when a reply to a request arrives.
func (s *scheduler) deliver(sections []uint64, data [][]byte) {
s.lock.Lock()
defer s.lock.Unlock()
for i, section := range sections {
if res := s.responses[section]; res != nil && res.cached == nil { // Avoid non-requests and double deliveries
res.cached = data[i]
close(res.done)
}
}
}
core/bloombits/scheduler_test.go 0 → 100644
View file @ f585f9ee
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits
import (
"bytes"
"math/big"
"math/rand"
"sync"
"sync/atomic"
"testing"
"time"
)
// Tests that the scheduler can deduplicate and forward retrieval requests to
// underlying fetchers and serve responses back, irrelevant of the concurrency
// of the requesting clients or serving data fetchers.
func TestSchedulerSingleClientSingleFetcher(t *testing.T) { testScheduler(t, 1, 1, 5000) }
func TestSchedulerSingleClientMultiFetcher(t *testing.T) { testScheduler(t, 1, 10, 5000) }
func TestSchedulerMultiClientSingleFetcher(t *testing.T) { testScheduler(t, 10, 1, 5000) }
func TestSchedulerMultiClientMultiFetcher(t *testing.T) { testScheduler(t, 10, 10, 5000) }
func testScheduler(t *testing.T, clients int, fetchers int, requests int) {
f := newScheduler(0)
// Create a batch of handler goroutines that respond to bloom bit requests and
// deliver them to the scheduler.
var fetchPend sync.WaitGroup
fetchPend.Add(fetchers)
defer fetchPend.Wait()
fetch := make(chan *request, 16)
defer close(fetch)
var delivered uint32
for i := 0; i < fetchers; i++ {
go func() {
defer fetchPend.Done()
for req := range fetch {
time.Sleep(time.Duration(rand.Intn(int(100 * time.Microsecond))))
atomic.AddUint32(&delivered, 1)
f.deliver([]uint64{
req.section + uint64(requests), // Non-requested data (ensure it doesn't go out of bounds)
req.section, // Requested data
req.section, // Duplicated data (ensure it doesn't double close anything)
}, [][]byte{
[]byte{},
new(big.Int).SetUint64(req.section).Bytes(),
new(big.Int).SetUint64(req.section).Bytes(),
})
}
}()
}
// Start a batch of goroutines to concurrently run scheduling tasks
quit := make(chan struct{})
var pend sync.WaitGroup
pend.Add(clients)
for i := 0; i < clients; i++ {
go func() {
defer pend.Done()
in := make(chan uint64, 16)
out := make(chan []byte, 16)
f.run(in, fetch, out, quit, &pend)
go func() {
for j := 0; j < requests; j++ {
in <- uint64(j)
}
close(in)
}()
for j := 0; j < requests; j++ {
bits := <-out
if want := new(big.Int).SetUint64(uint64(j)).Bytes(); !bytes.Equal(bits, want) {
t.Errorf("vector %d: delivered content mismatch: have %x, want %x", j, bits, want)
}
}
}()
}
pend.Wait()
if have := atomic.LoadUint32(&delivered); int(have) != requests {
t.Errorf("request count mismatch: have %v, want %v", have, requests)
}
}
core/chain_indexer.go
View file @ f585f9ee
...@@ -36,14 +36,13 @@ import ( ...@@ -36,14 +36,13 @@ import (
type ChainIndexerBackend interface { type ChainIndexerBackend interface {
// Reset initiates the processing of a new chain segment, potentially terminating // Reset initiates the processing of a new chain segment, potentially terminating
// any partially completed operations (in case of a reorg). // any partially completed operations (in case of a reorg).
Reset(section uint64, lastSectionHead common.Hash) Reset(section uint64)
// Process crunches through the next header in the chain segment. The caller // Process crunches through the next header in the chain segment. The caller
// will ensure a sequential order of headers. // will ensure a sequential order of headers.
Process(header *types.Header) Process(header *types.Header)
// Commit finalizes the section metadata and stores it into the database. This // Commit finalizes the section metadata and stores it into the database.
// interface will usually be a batch writer.
Commit() error Commit() error
} }
...@@ -101,34 +100,11 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken ...@@ -101,34 +100,11 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken
return c return c
} }
// AddKnownSectionHead marks a new section head as known/processed if it is newer
// than the already known best section head
func (c *ChainIndexer) AddKnownSectionHead(section uint64, shead common.Hash) {
c.lock.Lock()
defer c.lock.Unlock()
if section < c.storedSections {
return
}
c.setSectionHead(section, shead)
c.setValidSections(section + 1)
}
// IndexerChain interface is used for connecting the indexer to a blockchain
type IndexerChain interface {
CurrentHeader() *types.Header
SubscribeChainEvent(ch chan<- ChainEvent) event.Subscription
}
// Start creates a goroutine to feed chain head events into the indexer for // Start creates a goroutine to feed chain head events into the indexer for
// cascading background processing. Children do not need to be started, they // cascading background processing. Children do not need to be started, they
// are notified about new events by their parents. // are notified about new events by their parents.
func (c *ChainIndexer) Start(chain IndexerChain) { func (c *ChainIndexer) Start(currentHeader *types.Header, chainEventer func(ch chan<- ChainEvent) event.Subscription) {
ch := make(chan ChainEvent, 10) go c.eventLoop(currentHeader, chainEventer)
sub := chain.SubscribeChainEvent(ch)
currentHeader := chain.CurrentHeader()
go c.eventLoop(currentHeader, ch, sub)
} }
// Close tears down all goroutines belonging to the indexer and returns any error // Close tears down all goroutines belonging to the indexer and returns any error
...@@ -149,14 +125,12 @@ func (c *ChainIndexer) Close() error { ...@@ -149,14 +125,12 @@ func (c *ChainIndexer) Close() error {
errs = append(errs, err) errs = append(errs, err)
} }
} }
// Close all children // Close all children
for _, child := range c.children { for _, child := range c.children {
if err := child.Close(); err != nil { if err := child.Close(); err != nil {
errs = append(errs, err) errs = append(errs, err)
} }
} }
// Return any failures // Return any failures
switch { switch {
case len(errs) == 0: case len(errs) == 0:
...@@ -173,10 +147,12 @@ func (c *ChainIndexer) Close() error { ...@@ -173,10 +147,12 @@ func (c *ChainIndexer) Close() error {
// eventLoop is a secondary - optional - event loop of the indexer which is only // eventLoop is a secondary - optional - event loop of the indexer which is only
// started for the outermost indexer to push chain head events into a processing // started for the outermost indexer to push chain head events into a processing
// queue. // queue.
func (c *ChainIndexer) eventLoop(currentHeader *types.Header, ch chan ChainEvent, sub event.Subscription) { func (c *ChainIndexer) eventLoop(currentHeader *types.Header, chainEventer func(ch chan<- ChainEvent) event.Subscription) {
// Mark the chain indexer as active, requiring an additional teardown // Mark the chain indexer as active, requiring an additional teardown
atomic.StoreUint32(&c.active, 1) atomic.StoreUint32(&c.active, 1)
events := make(chan ChainEvent, 10)
sub := chainEventer(events)
defer sub.Unsubscribe() defer sub.Unsubscribe()
// Fire the initial new head event to start any outstanding processing // Fire the initial new head event to start any outstanding processing
...@@ -193,7 +169,7 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, ch chan ChainEvent ...@@ -193,7 +169,7 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, ch chan ChainEvent
errc <- nil errc <- nil
return return
case ev, ok := <-ch: case ev, ok := <-events:
// Received a new event, ensure it's not nil (closing) and update // Received a new event, ensure it's not nil (closing) and update
if !ok { if !ok {
errc := <-c.quit errc := <-c.quit
...@@ -257,10 +233,9 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) { ...@@ -257,10 +233,9 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) {
// down into the processing backend. // down into the processing backend.
func (c *ChainIndexer) updateLoop() { func (c *ChainIndexer) updateLoop() {
var ( var (
updated time.Time updating bool
updateMsg bool updated time.Time
) )
for { for {
select { select {
case errc := <-c.quit: case errc := <-c.quit:
...@@ -275,7 +250,7 @@ func (c *ChainIndexer) updateLoop() { ...@@ -275,7 +250,7 @@ func (c *ChainIndexer) updateLoop() {
// Periodically print an upgrade log message to the user // Periodically print an upgrade log message to the user
if time.Since(updated) > 8*time.Second { if time.Since(updated) > 8*time.Second {
if c.knownSections > c.storedSections+1 { if c.knownSections > c.storedSections+1 {
updateMsg = true updating = true
c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections) c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections)
} }
updated = time.Now() updated = time.Now()
...@@ -284,7 +259,7 @@ func (c *ChainIndexer) updateLoop() { ...@@ -284,7 +259,7 @@ func (c *ChainIndexer) updateLoop() {
section := c.storedSections section := c.storedSections
var oldHead common.Hash var oldHead common.Hash
if section > 0 { if section > 0 {
oldHead = c.SectionHead(section - 1) oldHead = c.sectionHead(section - 1)
} }
// Process the newly defined section in the background // Process the newly defined section in the background
c.lock.Unlock() c.lock.Unlock()
...@@ -295,11 +270,11 @@ func (c *ChainIndexer) updateLoop() { ...@@ -295,11 +270,11 @@ func (c *ChainIndexer) updateLoop() {
c.lock.Lock() c.lock.Lock()
// If processing succeeded and no reorgs occcurred, mark the section completed // If processing succeeded and no reorgs occcurred, mark the section completed
if err == nil && oldHead == c.SectionHead(section-1) { if err == nil && oldHead == c.sectionHead(section-1) {
c.setSectionHead(section, newHead) c.setSectionHead(section, newHead)
c.setValidSections(section + 1) c.setValidSections(section + 1)
if c.storedSections == c.knownSections && updateMsg { if c.storedSections == c.knownSections && updating {
updateMsg = false updating = false
c.log.Info("Finished upgrading chain index") c.log.Info("Finished upgrading chain index")
} }
...@@ -336,7 +311,7 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com ...@@ -336,7 +311,7 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com
c.log.Trace("Processing new chain section", "section", section) c.log.Trace("Processing new chain section", "section", section)
// Reset and partial processing // Reset and partial processing
c.backend.Reset(section, lastHead) c.backend.Reset(section)
for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ { for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ {
hash := GetCanonicalHash(c.chainDb, number) hash := GetCanonicalHash(c.chainDb, number)
...@@ -366,7 +341,7 @@ func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) { ...@@ -366,7 +341,7 @@ func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) {
c.lock.Lock() c.lock.Lock()
defer c.lock.Unlock() defer c.lock.Unlock()
return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1) return c.storedSections, c.storedSections*c.sectionSize - 1, c.sectionHead(c.storedSections - 1)
} }
// AddChildIndexer adds a child ChainIndexer that can use the output of this one // AddChildIndexer adds a child ChainIndexer that can use the output of this one
...@@ -408,7 +383,7 @@ func (c *ChainIndexer) setValidSections(sections uint64) { ...@@ -408,7 +383,7 @@ func (c *ChainIndexer) setValidSections(sections uint64) {
// sectionHead retrieves the last block hash of a processed section from the // sectionHead retrieves the last block hash of a processed section from the
// index database. // index database.
func (c *ChainIndexer) SectionHead(section uint64) common.Hash { func (c *ChainIndexer) sectionHead(section uint64) common.Hash {
var data [8]byte var data [8]byte
binary.BigEndian.PutUint64(data[:], section) binary.BigEndian.PutUint64(data[:], section)
......
core/chain_indexer_test.go
View file @ f585f9ee
...@@ -23,7 +23,6 @@ import ( ...@@ -23,7 +23,6 @@ import (
"testing" "testing"
"time" "time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
) )
...@@ -209,7 +208,7 @@ func (b *testChainIndexBackend) reorg(headNum uint64) uint64 { ...@@ -209,7 +208,7 @@ func (b *testChainIndexBackend) reorg(headNum uint64) uint64 {
return b.stored * b.indexer.sectionSize return b.stored * b.indexer.sectionSize
} }
func (b *testChainIndexBackend) Reset(section uint64, lastSectionHead common.Hash) { func (b *testChainIndexBackend) Reset(section uint64) {
b.section = section b.section = section
b.headerCnt = 0 b.headerCnt = 0
} }
......
core/database_util.go
View file @ f585f9ee
This diff is collapsed.
core/types/bloom9.go
View file @ f585f9ee
...@@ -28,10 +28,16 @@ type bytesBacked interface { ...@@ -28,10 +28,16 @@ type bytesBacked interface {
Bytes() []byte Bytes() []byte
} }
const bloomLength = 256 const (
// BloomByteLength represents the number of bytes used in a header log bloom.
BloomByteLength = 256
// Bloom represents a 256 bit bloom filter. // BloomBitLength represents the number of bits used in a header log bloom.
type Bloom [bloomLength]byte BloomBitLength = 8 * BloomByteLength
)
// Bloom represents a 2048 bit bloom filter.
type Bloom [BloomByteLength]byte
// BytesToBloom converts a byte slice to a bloom filter. // BytesToBloom converts a byte slice to a bloom filter.
// It panics if b is not of suitable size. // It panics if b is not of suitable size.
...@@ -47,7 +53,7 @@ func (b *Bloom) SetBytes(d []byte) { ...@@ -47,7 +53,7 @@ func (b *Bloom) SetBytes(d []byte) {
if len(b) < len(d) { if len(b) < len(d) {
panic(fmt.Sprintf("bloom bytes too big %d %d", len(b), len(d))) panic(fmt.Sprintf("bloom bytes too big %d %d", len(b), len(d)))
} }
copy(b[bloomLength-len(d):], d) copy(b[BloomByteLength-len(d):], d)
} }
// Add adds d to the filter. Future calls of Test(d) will return true. // Add adds d to the filter. Future calls of Test(d) will return true.
...@@ -106,20 +112,6 @@ func LogsBloom(logs []*Log) *big.Int { ...@@ -106,20 +112,6 @@ func LogsBloom(logs []*Log) *big.Int {
return bin return bin
} }
type BloomIndexList [3]uint
// BloomIndexes returns the bloom filter bit indexes belonging to the given key
func BloomIndexes(b []byte) BloomIndexList {
b = crypto.Keccak256(b[:])
var r [3]uint
for i, _ := range r {
r[i] = (uint(b[i+i+1]) + (uint(b[i+i]) << 8)) & 2047
}
return r
}
func bloom9(b []byte) *big.Int { func bloom9(b []byte) *big.Int {
b = crypto.Keccak256(b[:]) b = crypto.Keccak256(b[:])
......
eth/api_backend.go
View file @ f585f9ee
...@@ -24,11 +24,11 @@ import ( ...@@ -24,11 +24,11 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/filters"
"github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/gasprice"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/event"
...@@ -196,28 +196,13 @@ func (b *EthApiBackend) AccountManager() *accounts.Manager { ...@@ -196,28 +196,13 @@ func (b *EthApiBackend) AccountManager() *accounts.Manager {
return b.eth.AccountManager() return b.eth.AccountManager()
} }
func (b *EthApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) { func (b *EthApiBackend) BloomStatus() (uint64, uint64) {
results := make([][]byte, len(sectionIdxList)) sections, _, _ := b.eth.bloomIndexer.Sections()
var err error return params.BloomBitsBlocks, sections
for i, sectionIdx := range sectionIdxList {
sectionHead := core.GetCanonicalHash(b.eth.chainDb, (sectionIdx+1)*bloomBitsSection-1)
results[i], err = core.GetBloomBits(b.eth.chainDb, bitIdx, sectionIdx, sectionHead)
if err != nil {
return nil, err
}
}
return results, nil
}
func (b *EthApiBackend) BloomBitsSections() uint64 {
sections, _, _ := b.eth.bbIndexer.Sections()
return sections
} }
func (b *EthApiBackend) BloomBitsConfig() filters.BloomConfig { func (b *EthApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {
return filters.BloomConfig{ for i := 0; i < bloomFilterThreads; i++ {
SectionSize: bloomBitsSection, go session.Multiplex(bloomRetrievalBatch, bloomRetrievalWait, b.eth.bloomRequests)
MaxRequestLen: 16,
MaxRequestWait: 0,
} }
} }
eth/backend.go
View file @ f585f9ee
...@@ -32,6 +32,7 @@ import ( ...@@ -32,6 +32,7 @@ import (
"github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/clique"
"github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/downloader"
...@@ -77,7 +78,8 @@ type Ethereum struct { ...@@ -77,7 +78,8 @@ type Ethereum struct {
engine consensus.Engine engine consensus.Engine
accountManager *accounts.Manager accountManager *accounts.Manager
bbIndexer *core.ChainIndexer bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests
bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports
ApiBackend *EthApiBackend ApiBackend *EthApiBackend
...@@ -127,7 +129,8 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { ...@@ -127,7 +129,8 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
networkId: config.NetworkId, networkId: config.NetworkId,
gasPrice: config.GasPrice, gasPrice: config.GasPrice,
etherbase: config.Etherbase, etherbase: config.Etherbase,
bbIndexer: NewBloomBitsProcessor(chainDb, bloomBitsSection), bloomRequests: make(chan chan *bloombits.Retrieval),
bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks),
} }
log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId)
...@@ -151,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { ...@@ -151,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
eth.blockchain.SetHead(compat.RewindTo) eth.blockchain.SetHead(compat.RewindTo)
core.WriteChainConfig(chainDb, genesisHash, chainConfig) core.WriteChainConfig(chainDb, genesisHash, chainConfig)
} }
eth.bbIndexer.Start(eth.blockchain) eth.bloomIndexer.Start(eth.blockchain.CurrentHeader(), eth.blockchain.SubscribeChainEvent)
if config.TxPool.Journal != "" { if config.TxPool.Journal != "" {
config.TxPool.Journal = ctx.ResolvePath(config.TxPool.Journal) config.TxPool.Journal = ctx.ResolvePath(config.TxPool.Journal)
...@@ -261,7 +264,7 @@ func (s *Ethereum) APIs() []rpc.API { ...@@ -261,7 +264,7 @@ func (s *Ethereum) APIs() []rpc.API {
}, { }, {
Namespace: "eth", Namespace: "eth",
Version: "1.0", Version: "1.0",
Service: filters.NewPublicFilterAPI(s.ApiBackend, false, bloomBitsSection), Service: filters.NewPublicFilterAPI(s.ApiBackend, false),
Public: true, Public: true,
}, { }, {
Namespace: "admin", Namespace: "admin",
...@@ -359,14 +362,17 @@ func (s *Ethereum) Downloader() *downloader.Downloader { return s.protocolManage ...@@ -359,14 +362,17 @@ func (s *Ethereum) Downloader() *downloader.Downloader { return s.protocolManage
func (s *Ethereum) Protocols() []p2p.Protocol { func (s *Ethereum) Protocols() []p2p.Protocol {
if s.lesServer == nil { if s.lesServer == nil {
return s.protocolManager.SubProtocols return s.protocolManager.SubProtocols
} else {
return append(s.protocolManager.SubProtocols, s.lesServer.Protocols()...)
} }
return append(s.protocolManager.SubProtocols, s.lesServer.Protocols()...)
} }
// Start implements node.Service, starting all internal goroutines needed by the // Start implements node.Service, starting all internal goroutines needed by the
// Ethereum protocol implementation. // Ethereum protocol implementation.
func (s *Ethereum) Start(srvr *p2p.Server) error { func (s *Ethereum) Start(srvr *p2p.Server) error {
// Start the bloom bits servicing goroutines
s.startBloomHandlers()
// Start the RPC service
s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.NetVersion()) s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.NetVersion())
// Figure out a max peers count based on the server limits // Figure out a max peers count based on the server limits
...@@ -377,6 +383,7 @@ func (s *Ethereum) Start(srvr *p2p.Server) error { ...@@ -377,6 +383,7 @@ func (s *Ethereum) Start(srvr *p2p.Server) error {
maxPeers = srvr.MaxPeers / 2 maxPeers = srvr.MaxPeers / 2
} }
} }
// Start the networking layer and the light server if requested
s.protocolManager.Start(maxPeers) s.protocolManager.Start(maxPeers)
if s.lesServer != nil { if s.lesServer != nil {
s.lesServer.Start(srvr) s.lesServer.Start(srvr)
...@@ -390,7 +397,7 @@ func (s *Ethereum) Stop() error { ...@@ -390,7 +397,7 @@ func (s *Ethereum) Stop() error {
if s.stopDbUpgrade != nil { if s.stopDbUpgrade != nil {
s.stopDbUpgrade() s.stopDbUpgrade()
} }
s.bbIndexer.Close() s.bloomIndexer.Close()
s.blockchain.Stop() s.blockchain.Stop()
s.protocolManager.Stop() s.protocolManager.Stop()
if s.lesServer != nil { if s.lesServer != nil {
......
eth/bloombits.go 0 → 100644
View file @ f585f9ee
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package eth
import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/bitutil"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
)
const (
// bloomServiceThreads is the number of goroutines used globally by an Ethereum
// instance to service bloombits lookups for all running filters.
bloomServiceThreads = 16
// bloomFilterThreads is the number of goroutines used locally per filter to
// multiplex requests onto the global servicing goroutines.
bloomFilterThreads = 3
// bloomRetrievalBatch is the maximum number of bloom bit retrievals to service
// in a single batch.
bloomRetrievalBatch = 16
// bloomRetrievalWait is the maximum time to wait for enough bloom bit requests
// to accumulate request an entire batch (avoiding hysteresis).
bloomRetrievalWait = time.Duration(0)
)
// startBloomHandlers starts a batch of goroutines to accept bloom bit database
// retrievals from possibly a range of filters and serving the data to satisfy.
func (eth *Ethereum) startBloomHandlers() {
for i := 0; i < bloomServiceThreads; i++ {
go func() {
for {
select {
case <-eth.shutdownChan:
return
case request := <-eth.bloomRequests:
task := <-request
task.Bitsets = make([][]byte, len(task.Sections))
for i, section := range task.Sections {
head := core.GetCanonicalHash(eth.chainDb, (section+1)*params.BloomBitsBlocks-1)
blob, err := bitutil.DecompressBytes(core.GetBloomBits(eth.chainDb, task.Bit, section, head), int(params.BloomBitsBlocks)/8)
if err != nil {
panic(err)
}
task.Bitsets[i] = blob
}
request <- task
}
}
}()
}
}
const (
// bloomConfirms is the number of confirmation blocks before a bloom section is
// considered probably final and its rotated bits are calculated.
bloomConfirms = 256
// bloomThrottling is the time to wait between processing two consecutive index
// sections. It's useful during chain upgrades to prevent disk overload.
bloomThrottling = 100 * time.Millisecond
)
// BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index
// for the Ethereum header bloom filters, permitting blazing fast filtering.
type BloomIndexer struct {
size uint64 // section size to generate bloombits for
db ethdb.Database // database instance to write index data and metadata into
gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index
section uint64 // Section is the section number being processed currently
head common.Hash // Head is the hash of the last header processed
}
// NewBloomIndexer returns a chain indexer that generates bloom bits data for the
// canonical chain for fast logs filtering.
func NewBloomIndexer(db ethdb.Database, size uint64) *core.ChainIndexer {
backend := &BloomIndexer{
db: db,
size: size,
}
table := ethdb.NewTable(db, string(core.BloomBitsIndexPrefix))
return core.NewChainIndexer(db, table, backend, size, bloomConfirms, bloomThrottling, "bloombits")
}
// Reset implements core.ChainIndexerBackend, starting a new bloombits index
// section.
func (b *BloomIndexer) Reset(section uint64) {
gen, err := bloombits.NewGenerator(uint(b.size))
if err != nil {
panic(err)
}
b.gen, b.section, b.head = gen, section, common.Hash{}
}
// Process implements core.ChainIndexerBackend, adding a new header's bloom into
// the index.
func (b *BloomIndexer) Process(header *types.Header) {
b.gen.AddBloom(header.Bloom)
b.head = header.Hash()
}
// Commit implements core.ChainIndexerBackend, finalizing the bloom section and
// writing it out into the database.
func (b *BloomIndexer) Commit() error {
batch := b.db.NewBatch()
for i := 0; i < types.BloomBitLength; i++ {
bits, err := b.gen.Bitset(uint(i))
if err != nil {
return err
}
core.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits))
}
return batch.Write()
}
eth/db_upgrade.go
View file @ f585f9ee
...@@ -22,10 +22,7 @@ import ( ...@@ -22,10 +22,7 @@ import (
"time" "time"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/bitutil"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
...@@ -136,38 +133,3 @@ func upgradeDeduplicateData(db ethdb.Database) func() error { ...@@ -136,38 +133,3 @@ func upgradeDeduplicateData(db ethdb.Database) func() error {
return <-errc return <-errc
} }
} }
// BloomBitsIndex implements ChainIndex
type BloomBitsIndex struct {
db ethdb.Database
bc *bloombits.BloomBitsCreator
section, sectionSize uint64
sectionHead common.Hash
}
// number of confirmation blocks before a section is considered probably final and its bloom bits are calculated
const bloomBitsConfirmations = 256
// NewBloomBitsProcessor returns a chain processor that generates bloom bits data for the canonical chain
func NewBloomBitsProcessor(db ethdb.Database, sectionSize uint64) *core.ChainIndexer {
backend := &BloomBitsIndex{db: db, sectionSize: sectionSize}
return core.NewChainIndexer(db, ethdb.NewTable(db, "bbIndex-"), backend, sectionSize, bloomBitsConfirmations, time.Millisecond*100, "bloombits")
}
func (b *BloomBitsIndex) Reset(section uint64, lastSectionHead common.Hash) {
b.bc = bloombits.NewBloomBitsCreator(b.sectionSize)
b.section = section
}
func (b *BloomBitsIndex) Process(header *types.Header) {
b.bc.AddHeaderBloom(header.Bloom)
b.sectionHead = header.Hash()
}
func (b *BloomBitsIndex) Commit() error {
for i := 0; i < bloombits.BloomLength; i++ {
compVector := bitutil.CompressBytes(b.bc.GetBitVector(uint(i)))
core.StoreBloomBits(b.db, uint64(i), b.section, b.sectionHead, compVector)
}
return nil
}
eth/filters/api.go
View file @ f585f9ee
...@@ -51,27 +51,24 @@ type filter struct { ...@@ -51,27 +51,24 @@ type filter struct {
// PublicFilterAPI offers support to create and manage filters. This will allow external clients to retrieve various // PublicFilterAPI offers support to create and manage filters. This will allow external clients to retrieve various
// information related to the Ethereum protocol such als blocks, transactions and logs. // information related to the Ethereum protocol such als blocks, transactions and logs.
type PublicFilterAPI struct { type PublicFilterAPI struct {
backend Backend backend Backend
bloomBitsSection uint64 mux *event.TypeMux
mux *event.TypeMux quit chan struct{}
quit chan struct{} chainDb ethdb.Database
chainDb ethdb.Database events *EventSystem
events *EventSystem filtersMu sync.Mutex
filtersMu sync.Mutex filters map[rpc.ID]*filter
filters map[rpc.ID]*filter
} }
// NewPublicFilterAPI returns a new PublicFilterAPI instance. // NewPublicFilterAPI returns a new PublicFilterAPI instance.
func NewPublicFilterAPI(backend Backend, lightMode bool, bloomBitsSection uint64) *PublicFilterAPI { func NewPublicFilterAPI(backend Backend, lightMode bool) *PublicFilterAPI {
api := &PublicFilterAPI{ api := &PublicFilterAPI{
backend: backend, backend: backend,
bloomBitsSection: bloomBitsSection, mux: backend.EventMux(),
mux: backend.EventMux(), chainDb: backend.ChainDb(),
chainDb: backend.ChainDb(), events: NewEventSystem(backend.EventMux(), backend, lightMode),
events: NewEventSystem(backend.EventMux(), backend, lightMode), filters: make(map[rpc.ID]*filter),
filters: make(map[rpc.ID]*filter),
} }
go api.timeoutLoop() go api.timeoutLoop()
return api return api
...@@ -326,16 +323,20 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { ...@@ -326,16 +323,20 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) {
// //
// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs
func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) { func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) {
// Convert the RPC block numbers into internal representations
if crit.FromBlock == nil { if crit.FromBlock == nil {
crit.FromBlock = big.NewInt(rpc.LatestBlockNumber.Int64()) crit.FromBlock = big.NewInt(rpc.LatestBlockNumber.Int64())
} }
if crit.ToBlock == nil { if crit.ToBlock == nil {
crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64()) crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64())
} }
// Create and run the filter to get all the logs
filter := New(api.backend, crit.FromBlock.Int64(), crit.ToBlock.Int64(), crit.Addresses, crit.Topics) filter := New(api.backend, crit.FromBlock.Int64(), crit.ToBlock.Int64(), crit.Addresses, crit.Topics)
logs, err := filter.Find(ctx) logs, err := filter.Logs(ctx)
if err != nil {
return nil, err
}
return returnLogs(logs), err return returnLogs(logs), err
} }
...@@ -369,20 +370,18 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty ...@@ -369,20 +370,18 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty
return nil, fmt.Errorf("filter not found") return nil, fmt.Errorf("filter not found")
} }
var begin, end int64 begin := rpc.LatestBlockNumber.Int64()
if f.crit.FromBlock != nil { if f.crit.FromBlock != nil {
begin = f.crit.FromBlock.Int64() begin = f.crit.FromBlock.Int64()
} else {
begin = rpc.LatestBlockNumber.Int64()
} }
end := rpc.LatestBlockNumber.Int64()
if f.crit.ToBlock != nil { if f.crit.ToBlock != nil {
end = f.crit.ToBlock.Int64() end = f.crit.ToBlock.Int64()
} else {
end = rpc.LatestBlockNumber.Int64()
} }
// Create and run the filter to get all the logs
filter := New(api.backend, begin, end, f.crit.Addresses, f.crit.Topics) filter := New(api.backend, begin, end, f.crit.Addresses, f.crit.Topics)
logs, err := filter.Find(ctx) logs, err := filter.Logs(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -390,7 +389,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty ...@@ -390,7 +389,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty
} }
// GetFilterChanges returns the logs for the filter with the given id since // GetFilterChanges returns the logs for the filter with the given id since
// last time is was called. This can be used for polling. // last time it was called. This can be used for polling.
// //
// For pending transaction and block filters the result is []common.Hash. // For pending transaction and block filters the result is []common.Hash.
// (pending)Log filters return []Log. // (pending)Log filters return []Log.
......
eth/filters/bench_test.go
View file @ f585f9ee
...@@ -31,82 +31,41 @@ import ( ...@@ -31,82 +31,41 @@ import (
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/node"
"github.com/golang/snappy"
) )
func BenchmarkBloomBits512(b *testing.B) { func BenchmarkBloomBits512(b *testing.B) {
benchmarkBloomBitsForSize(b, 512) benchmarkBloomBits(b, 512)
} }
func BenchmarkBloomBits1k(b *testing.B) { func BenchmarkBloomBits1k(b *testing.B) {
benchmarkBloomBitsForSize(b, 1024) benchmarkBloomBits(b, 1024)
} }
func BenchmarkBloomBits2k(b *testing.B) { func BenchmarkBloomBits2k(b *testing.B) {
benchmarkBloomBitsForSize(b, 2048) benchmarkBloomBits(b, 2048)
} }
func BenchmarkBloomBits4k(b *testing.B) { func BenchmarkBloomBits4k(b *testing.B) {
benchmarkBloomBitsForSize(b, 4096) benchmarkBloomBits(b, 4096)
} }
func BenchmarkBloomBits8k(b *testing.B) { func BenchmarkBloomBits8k(b *testing.B) {
benchmarkBloomBitsForSize(b, 8192) benchmarkBloomBits(b, 8192)
} }
func BenchmarkBloomBits16k(b *testing.B) { func BenchmarkBloomBits16k(b *testing.B) {
benchmarkBloomBitsForSize(b, 16384) benchmarkBloomBits(b, 16384)
} }
func BenchmarkBloomBits32k(b *testing.B) { func BenchmarkBloomBits32k(b *testing.B) {
benchmarkBloomBitsForSize(b, 32768) benchmarkBloomBits(b, 32768)
}
func benchmarkBloomBitsForSize(b *testing.B, sectionSize uint64) {
benchmarkBloomBits(b, sectionSize, 0)
benchmarkBloomBits(b, sectionSize, 1)
benchmarkBloomBits(b, sectionSize, 2)
} }
const benchFilterCnt = 2000 const benchFilterCnt = 2000
func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) { func benchmarkBloomBits(b *testing.B, sectionSize uint64) {
benchDataDir := node.DefaultDataDir() + "/geth/chaindata" benchDataDir := node.DefaultDataDir() + "/geth/chaindata"
fmt.Println("Running bloombits benchmark section size:", sectionSize, " compression method:", comp) fmt.Println("Running bloombits benchmark section size:", sectionSize)
var (
compressFn func([]byte) []byte
decompressFn func([]byte, int) ([]byte, error)
)
switch comp {
case 0:
// no compression
compressFn = func(data []byte) []byte {
return data
}
decompressFn = func(data []byte, target int) ([]byte, error) {
if len(data) != target {
panic(nil)
}
return data, nil
}
case 1:
// bitutil/compress.go
compressFn = bitutil.CompressBytes
decompressFn = bitutil.DecompressBytes
case 2:
// go snappy
compressFn = func(data []byte) []byte {
return snappy.Encode(nil, data)
}
decompressFn = func(data []byte, target int) ([]byte, error) {
decomp, err := snappy.Decode(nil, data)
if err != nil || len(decomp) != target {
panic(err)
}
return decomp, nil
}
}
db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024) db, err := ethdb.NewLDBDatabase(benchDataDir, 128, 1024)
if err != nil { if err != nil {
...@@ -128,7 +87,10 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) { ...@@ -128,7 +87,10 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) {
cnt := (headNum - 512) / sectionSize cnt := (headNum - 512) / sectionSize
var dataSize, compSize uint64 var dataSize, compSize uint64
for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ { for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ {
bc := bloombits.NewBloomBitsCreator(sectionSize) bc, err := bloombits.NewGenerator(uint(sectionSize))
if err != nil {
b.Fatalf("failed to create generator: %v", err)
}
var header *types.Header var header *types.Header
for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ {
hash := core.GetCanonicalHash(db, i) hash := core.GetCanonicalHash(db, i)
...@@ -136,15 +98,18 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) { ...@@ -136,15 +98,18 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) {
if header == nil { if header == nil {
b.Fatalf("Error creating bloomBits data") b.Fatalf("Error creating bloomBits data")
} }
bc.AddHeaderBloom(header.Bloom) bc.AddBloom(header.Bloom)
} }
sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*sectionSize-1) sectionHead := core.GetCanonicalHash(db, (sectionIdx+1)*sectionSize-1)
for i := 0; i < bloombits.BloomLength; i++ { for i := 0; i < types.BloomBitLength; i++ {
data := bc.GetBitVector(uint(i)) data, err := bc.Bitset(uint(i))
comp := compressFn(data) if err != nil {
b.Fatalf("failed to retrieve bitset: %v", err)
}
comp := bitutil.CompressBytes(data)
dataSize += uint64(len(data)) dataSize += uint64(len(data))
compSize += uint64(len(comp)) compSize += uint64(len(comp))
core.StoreBloomBits(db, uint64(i), sectionIdx, sectionHead, comp) core.WriteBloomBits(db, uint(i), sectionIdx, sectionHead, comp)
} }
//if sectionIdx%50 == 0 { //if sectionIdx%50 == 0 {
// fmt.Println(" section", sectionIdx, "/", cnt) // fmt.Println(" section", sectionIdx, "/", cnt)
...@@ -171,8 +136,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) { ...@@ -171,8 +136,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64, comp int) {
addr[0] = byte(i) addr[0] = byte(i)
addr[1] = byte(i / 256) addr[1] = byte(i / 256)
filter := New(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) filter := New(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil)
filter.decompress = decompressFn if _, err := filter.Logs(context.Background()); err != nil {
if _, err := filter.Find(context.Background()); err != nil {
b.Error("filter.Find error:", err) b.Error("filter.Find error:", err)
} }
} }
...@@ -229,7 +193,7 @@ func BenchmarkNoBloomBits(b *testing.B) { ...@@ -229,7 +193,7 @@ func BenchmarkNoBloomBits(b *testing.B) {
mux := new(event.TypeMux) mux := new(event.TypeMux)
backend := &testBackend{mux, db, 0, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} backend := &testBackend{mux, db, 0, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)}
filter := New(backend, 0, int64(headNum), []common.Address{common.Address{}}, nil) filter := New(backend, 0, int64(headNum), []common.Address{common.Address{}}, nil)
filter.Find(context.Background()) filter.Logs(context.Background())
d := time.Since(start) d := time.Since(start)
fmt.Println("Finished running filter benchmarks") fmt.Println("Finished running filter benchmarks")
fmt.Println(" ", d, "total ", d*time.Duration(1000000)/time.Duration(headNum+1), "per million blocks") fmt.Println(" ", d, "total ", d*time.Duration(1000000)/time.Duration(headNum+1), "per million blocks")
......
eth/filters/filter.go
View file @ f585f9ee
...@@ -19,11 +19,9 @@ package filters ...@@ -19,11 +19,9 @@ package filters
import ( import (
"context" "context"
"math/big" "math/big"
"sync"
"time" "time"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/bitutil"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
...@@ -37,140 +35,143 @@ type Backend interface { ...@@ -37,140 +35,143 @@ type Backend interface {
EventMux() *event.TypeMux EventMux() *event.TypeMux
HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error)
GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
BloomBitsSections() uint64
BloomBitsConfig() BloomConfig
SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription
SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error)
}
type BloomConfig struct { BloomStatus() (uint64, uint64)
SectionSize uint64 ServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
MaxRequestLen int
MaxRequestWait time.Duration
} }
// Filter can be used to retrieve and filter logs. // Filter can be used to retrieve and filter logs.
type Filter struct { type Filter struct {
backend Backend backend Backend
bloomBitsConfig BloomConfig
db ethdb.Database db ethdb.Database
begin, end int64 begin, end int64
addresses []common.Address addresses []common.Address
topics [][]common.Hash topics [][]common.Hash
decompress func([]byte, int) ([]byte, error) matcher *bloombits.Matcher
matcher *bloombits.Matcher
} }
// New creates a new filter which uses a bloom filter on blocks to figure out whether // New creates a new filter which uses a bloom filter on blocks to figure out whether
// a particular block is interesting or not. // a particular block is interesting or not.
func New(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { func New(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter {
size, _ := backend.BloomStatus()
return &Filter{ return &Filter{
backend: backend, backend: backend,
begin: begin, begin: begin,
end: end, end: end,
addresses: addresses, addresses: addresses,
topics: topics, topics: topics,
bloomBitsConfig: backend.BloomBitsConfig(), db: backend.ChainDb(),
db: backend.ChainDb(), matcher: bloombits.NewMatcher(size, addresses, topics),
matcher: bloombits.NewMatcher(backend.BloomBitsConfig().SectionSize, addresses, topics),
decompress: bitutil.DecompressBytes,
} }
} }
// FindOnce searches the blockchain for matching log entries, returning // Logs searches the blockchain for matching log entries, returning all from the
// all matching entries from the first block that contains matches, // first block that contains matches, updating the start of the filter accordingly.
// updating the start point of the filter accordingly. If no results are func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) {
// found, a nil slice is returned. // Figure out the limits of the filter range
func (f *Filter) FindOnce(ctx context.Context) ([]*types.Log, error) { header, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber)
head, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) if header == nil {
if head == nil {
return nil, nil return nil, nil
} }
headBlockNumber := head.Number.Uint64() head := header.Number.Uint64()
var beginBlockNo uint64 = uint64(f.begin)
if f.begin == -1 { if f.begin == -1 {
beginBlockNo = headBlockNumber f.begin = int64(head)
} }
var endBlockNo uint64 = uint64(f.end) end := uint64(f.end)
if f.end == -1 { if f.end == -1 {
endBlockNo = headBlockNumber end = head
} }
// Gather all indexed logs, and finish with non indexed ones
logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo) var (
f.begin = int64(blockNumber + 1) logs []*types.Log
return logs, err err error
} )
size, sections := f.backend.BloomStatus()
// Run filters logs with the current parameters set if indexed := sections * size; indexed > uint64(f.begin) {
func (f *Filter) Find(ctx context.Context) (logs []*types.Log, err error) { if indexed > end {
for { logs, err = f.indexedLogs(ctx, end)
newLogs, err := f.FindOnce(ctx) } else {
if len(newLogs) == 0 || err != nil { logs, err = f.indexedLogs(ctx, indexed-1)
}
if err != nil {
return logs, err return logs, err
} }
logs = append(logs, newLogs...)
} }
rest, err := f.unindexedLogs(ctx, end)
logs = append(logs, rest...)
return logs, err
} }
// nextRequest returns the next request to retrieve for the bloombits matcher // indexedLogs returns the logs matching the filter criteria based on the bloom
func (f *Filter) nextRequest() (bloombits uint, sections []uint64) { // bits indexed available locally or via the network.
bloomIndex, ok := f.matcher.AllocSectionQueue() func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, error) {
if !ok { // Create a matcher session and request servicing from the backend
return 0, nil matches := make(chan uint64, 64)
session, err := f.matcher.Start(uint64(f.begin), end, matches)
if err != nil {
return nil, err
} }
if f.bloomBitsConfig.MaxRequestWait > 0 && defer session.Close(time.Second)
(f.bloomBitsConfig.MaxRequestLen <= 1 || // SectionCount is always greater than zero after a successful alloc
f.matcher.SectionCount(bloomIndex) < f.bloomBitsConfig.MaxRequestLen) { f.backend.ServiceFilter(ctx, session)
time.Sleep(f.bloomBitsConfig.MaxRequestWait)
// Iterate over the matches until exhausted or context closed
var logs []*types.Log
for {
select {
case number, ok := <-matches:
// Abort if all matches have been fulfilled
if !ok {
f.begin = int64(end) + 1
return logs, nil
}
// Retrieve the suggested block and pull any truly matching logs
header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(number))
if header == nil || err != nil {
return logs, err
}
found, err := f.checkMatches(ctx, header)
if err != nil {
return logs, err
}
logs = append(logs, found...)
case <-ctx.Done():
return logs, ctx.Err()
}
} }
return bloomIndex, f.matcher.FetchSections(bloomIndex, f.bloomBitsConfig.MaxRequestLen)
} }
// serveMatcher serves the bloombits matcher by fetching the requested vectors // indexedLogs returns the logs matching the filter criteria based on raw block
// through the filter backend // iteration and bloom matching.
func (f *Filter) serveMatcher(ctx context.Context, stop chan struct{}, wg *sync.WaitGroup) chan error { func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, error) {
errChn := make(chan error, 1) var logs []*types.Log
wg.Add(10)
for i := 0; i < 10; i++ { for ; f.begin <= int64(end); f.begin++ {
go func(i int) { header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin))
defer wg.Done() if header == nil || err != nil {
return logs, err
for { }
b, s := f.nextRequest() if bloomFilter(header.Bloom, f.addresses, f.topics) {
if s == nil { found, err := f.checkMatches(ctx, header)
return if err != nil {
} return logs, err
data, err := f.backend.GetBloomBits(ctx, uint64(b), s)
if err != nil {
select {
case errChn <- err:
case <-stop:
}
return
}
decomp := make([][]byte, len(data))
for i, d := range data {
var err error
if decomp[i], err = f.decompress(d, int(f.bloomBitsConfig.SectionSize/8)); err != nil {
select {
case errChn <- err:
case <-stop:
}
return
}
}
f.matcher.Deliver(b, s, decomp)
} }
}(i) logs = append(logs, found...)
}
} }
return logs, nil
return errChn
} }
// checkMatches checks if the receipts belonging to the given header contain any log events that // checkMatches checks if the receipts belonging to the given header contain any log events that
...@@ -192,83 +193,6 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs [ ...@@ -192,83 +193,6 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs [
return nil, nil return nil, nil
} }
func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types.Log, blockNumber uint64, err error) {
haveBloomBitsBefore := f.backend.BloomBitsSections() * f.bloomBitsConfig.SectionSize
if haveBloomBitsBefore > start {
e := end
if haveBloomBitsBefore <= e {
e = haveBloomBitsBefore - 1
}
stop := make(chan struct{})
var wg sync.WaitGroup
matches := f.matcher.Start(start, e)
errChn := f.serveMatcher(ctx, stop, &wg)
defer func() {
f.matcher.Stop()
close(stop)
wg.Wait()
}()
loop:
for {
select {
case i, ok := <-matches:
if !ok {
break loop
}
blockNumber := rpc.BlockNumber(i)
header, err := f.backend.HeaderByNumber(ctx, blockNumber)
if header == nil || err != nil {
return logs, end, err
}
logs, err := f.checkMatches(ctx, header)
if err != nil {
return nil, end, err
}
if logs != nil {
return logs, i, nil
}
case err := <-errChn:
return logs, end, err
case <-ctx.Done():
return nil, end, ctx.Err()
}
}
if end < haveBloomBitsBefore {
return logs, end, nil
}
start = haveBloomBitsBefore
}
// search the rest with regular block-by-block bloom filtering
for i := start; i <= end; i++ {
blockNumber := rpc.BlockNumber(i)
header, err := f.backend.HeaderByNumber(ctx, blockNumber)
if header == nil || err != nil {
return logs, end, err
}
// Use bloom filtering to see if this block is interesting given the
// current parameters
if f.bloomFilter(header.Bloom) {
logs, err := f.checkMatches(ctx, header)
if err != nil {
return nil, end, err
}
if logs != nil {
return logs, i, nil
}
}
}
return logs, end, nil
}
func includes(addresses []common.Address, a common.Address) bool { func includes(addresses []common.Address, a common.Address) bool {
for _, addr := range addresses { for _, addr := range addresses {
if addr == a { if addr == a {
...@@ -323,10 +247,6 @@ Logs: ...@@ -323,10 +247,6 @@ Logs:
return ret return ret
} }
func (f *Filter) bloomFilter(bloom types.Bloom) bool {
return bloomFilter(bloom, f.addresses, f.topics)
}
func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]common.Hash) bool { func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]common.Hash) bool {
if len(addresses) > 0 { if len(addresses) > 0 {
var included bool var included bool
......
eth/filters/filter_system_test.go
View file @ f585f9ee
...@@ -20,12 +20,14 @@ import ( ...@@ -20,12 +20,14 @@ import (
"context" "context"
"fmt" "fmt"
"math/big" "math/big"
"math/rand"
"reflect" "reflect"
"testing" "testing"
"time" "time"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/event"
...@@ -85,29 +87,35 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc ...@@ -85,29 +87,35 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc
return b.chainFeed.Subscribe(ch) return b.chainFeed.Subscribe(ch)
} }
func (b *testBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) { func (b *testBackend) BloomStatus() (uint64, uint64) {
results := make([][]byte, len(sectionIdxList)) return params.BloomBitsBlocks, b.sections
var err error
for i, sectionIdx := range sectionIdxList {
sectionHead := core.GetCanonicalHash(b.db, (sectionIdx+1)*testBloomBitsSection-1)
results[i], err = core.GetBloomBits(b.db, bitIdx, sectionIdx, sectionHead)
if err != nil {
return nil, err
}
}
return results, nil
} }
func (b *testBackend) BloomBitsSections() uint64 { func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {
return b.sections requests := make(chan chan *bloombits.Retrieval)
}
func (b *testBackend) BloomBitsConfig() BloomConfig { go session.Multiplex(16, 0, requests)
return BloomConfig{ go func() {
SectionSize: testBloomBitsSection, for {
MaxRequestLen: 16, // Wait for a service request or a shutdown
MaxRequestWait: 0, select {
} case <-ctx.Done():
return
case request := <-requests:
task := <-request
task.Bitsets = make([][]byte, len(task.Sections))
for i, section := range task.Sections {
if rand.Int()%4 != 0 { // Handle occasional missing deliveries
head := core.GetCanonicalHash(b.db, (section+1)*params.BloomBitsBlocks-1)
task.Bitsets[i] = core.GetBloomBits(b.db, task.Bit, section, head)
}
}
request <- task
}
}
}()
} }
// TestBlockSubscription tests if a block subscription returns block hashes for posted chain events. // TestBlockSubscription tests if a block subscription returns block hashes for posted chain events.
...@@ -126,7 +134,7 @@ func TestBlockSubscription(t *testing.T) { ...@@ -126,7 +134,7 @@ func TestBlockSubscription(t *testing.T) {
logsFeed = new(event.Feed) logsFeed = new(event.Feed)
chainFeed = new(event.Feed) chainFeed = new(event.Feed)
backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed}
api = NewPublicFilterAPI(backend, false, 0) api = NewPublicFilterAPI(backend, false)
genesis = new(core.Genesis).MustCommit(db) genesis = new(core.Genesis).MustCommit(db)
chain, _ = core.GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *core.BlockGen) {}) chain, _ = core.GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *core.BlockGen) {})
chainEvents = []core.ChainEvent{} chainEvents = []core.ChainEvent{}
...@@ -183,7 +191,7 @@ func TestPendingTxFilter(t *testing.T) { ...@@ -183,7 +191,7 @@ func TestPendingTxFilter(t *testing.T) {
logsFeed = new(event.Feed) logsFeed = new(event.Feed)
chainFeed = new(event.Feed) chainFeed = new(event.Feed)
backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed}
api = NewPublicFilterAPI(backend, false, 0) api = NewPublicFilterAPI(backend, false)
transactions = []*types.Transaction{ transactions = []*types.Transaction{
types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), new(big.Int), new(big.Int), nil), types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), new(big.Int), new(big.Int), nil),
...@@ -246,7 +254,7 @@ func TestLogFilterCreation(t *testing.T) { ...@@ -246,7 +254,7 @@ func TestLogFilterCreation(t *testing.T) {
logsFeed = new(event.Feed) logsFeed = new(event.Feed)
chainFeed = new(event.Feed) chainFeed = new(event.Feed)
backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed}
api = NewPublicFilterAPI(backend, false, 0) api = NewPublicFilterAPI(backend, false)
testCases = []struct { testCases = []struct {
crit FilterCriteria crit FilterCriteria
...@@ -295,7 +303,7 @@ func TestInvalidLogFilterCreation(t *testing.T) { ...@@ -295,7 +303,7 @@ func TestInvalidLogFilterCreation(t *testing.T) {
logsFeed = new(event.Feed) logsFeed = new(event.Feed)
chainFeed = new(event.Feed) chainFeed = new(event.Feed)
backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed}
api = NewPublicFilterAPI(backend, false, 0) api = NewPublicFilterAPI(backend, false)
) )
// different situations where log filter creation should fail. // different situations where log filter creation should fail.
...@@ -325,7 +333,7 @@ func TestLogFilter(t *testing.T) { ...@@ -325,7 +333,7 @@ func TestLogFilter(t *testing.T) {
logsFeed = new(event.Feed) logsFeed = new(event.Feed)
chainFeed = new(event.Feed) chainFeed = new(event.Feed)
backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed}
api = NewPublicFilterAPI(backend, false, 0) api = NewPublicFilterAPI(backend, false)
firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111")
secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222")
...@@ -442,7 +450,7 @@ func TestPendingLogsSubscription(t *testing.T) { ...@@ -442,7 +450,7 @@ func TestPendingLogsSubscription(t *testing.T) {
logsFeed = new(event.Feed) logsFeed = new(event.Feed)
chainFeed = new(event.Feed) chainFeed = new(event.Feed)
backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed}
api = NewPublicFilterAPI(backend, false, 0) api = NewPublicFilterAPI(backend, false)
firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111")
secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222")
......
eth/filters/filter_test.go
View file @ f585f9ee
...@@ -32,8 +32,6 @@ import ( ...@@ -32,8 +32,6 @@ import (
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
) )
const testBloomBitsSection = 4096
func makeReceipt(addr common.Address) *types.Receipt { func makeReceipt(addr common.Address) *types.Receipt {
receipt := types.NewReceipt(nil, false, new(big.Int)) receipt := types.NewReceipt(nil, false, new(big.Int))
receipt.Logs = []*types.Log{ receipt.Logs = []*types.Log{
...@@ -101,7 +99,7 @@ func BenchmarkFilters(b *testing.B) { ...@@ -101,7 +99,7 @@ func BenchmarkFilters(b *testing.B) {
filter := New(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) filter := New(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil)
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
logs, _ := filter.Find(context.Background()) logs, _ := filter.Logs(context.Background())
if len(logs) != 4 { if len(logs) != 4 {
b.Fatal("expected 4 logs, got", len(logs)) b.Fatal("expected 4 logs, got", len(logs))
} }
...@@ -189,13 +187,13 @@ func TestFilters(t *testing.T) { ...@@ -189,13 +187,13 @@ func TestFilters(t *testing.T) {
filter := New(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) filter := New(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}})
logs, _ := filter.Find(context.Background()) logs, _ := filter.Logs(context.Background())
if len(logs) != 4 { if len(logs) != 4 {
t.Error("expected 4 log, got", len(logs)) t.Error("expected 4 log, got", len(logs))
} }
filter = New(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) filter = New(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}})
logs, _ = filter.Find(context.Background()) logs, _ = filter.Logs(context.Background())
if len(logs) != 1 { if len(logs) != 1 {
t.Error("expected 1 log, got", len(logs)) t.Error("expected 1 log, got", len(logs))
} }
...@@ -204,7 +202,7 @@ func TestFilters(t *testing.T) { ...@@ -204,7 +202,7 @@ func TestFilters(t *testing.T) {
} }
filter = New(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) filter = New(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}})
logs, _ = filter.Find(context.Background()) logs, _ = filter.Logs(context.Background())
if len(logs) != 1 { if len(logs) != 1 {
t.Error("expected 1 log, got", len(logs)) t.Error("expected 1 log, got", len(logs))
} }
...@@ -214,7 +212,7 @@ func TestFilters(t *testing.T) { ...@@ -214,7 +212,7 @@ func TestFilters(t *testing.T) {
filter = New(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}}) filter = New(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}})
logs, _ = filter.Find(context.Background()) logs, _ = filter.Logs(context.Background())
if len(logs) != 2 { if len(logs) != 2 {
t.Error("expected 2 log, got", len(logs)) t.Error("expected 2 log, got", len(logs))
} }
...@@ -222,7 +220,7 @@ func TestFilters(t *testing.T) { ...@@ -222,7 +220,7 @@ func TestFilters(t *testing.T) {
failHash := common.BytesToHash([]byte("fail")) failHash := common.BytesToHash([]byte("fail"))
filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}}) filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}})
logs, _ = filter.Find(context.Background()) logs, _ = filter.Logs(context.Background())
if len(logs) != 0 { if len(logs) != 0 {
t.Error("expected 0 log, got", len(logs)) t.Error("expected 0 log, got", len(logs))
} }
...@@ -230,14 +228,14 @@ func TestFilters(t *testing.T) { ...@@ -230,14 +228,14 @@ func TestFilters(t *testing.T) {
failAddr := common.BytesToAddress([]byte("failmenow")) failAddr := common.BytesToAddress([]byte("failmenow"))
filter = New(backend, 0, -1, []common.Address{failAddr}, nil) filter = New(backend, 0, -1, []common.Address{failAddr}, nil)
logs, _ = filter.Find(context.Background()) logs, _ = filter.Logs(context.Background())
if len(logs) != 0 { if len(logs) != 0 {
t.Error("expected 0 log, got", len(logs)) t.Error("expected 0 log, got", len(logs))
} }
filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) filter = New(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}})
logs, _ = filter.Find(context.Background()) logs, _ = filter.Logs(context.Background())
if len(logs) != 0 { if len(logs) != 0 {
t.Error("expected 0 log, got", len(logs)) t.Error("expected 0 log, got", len(logs))
} }
......
eth/handler.go
View file @ f585f9ee
...@@ -49,8 +49,6 @@ const ( ...@@ -49,8 +49,6 @@ const (
// txChanSize is the size of channel listening to TxPreEvent. // txChanSize is the size of channel listening to TxPreEvent.
// The number is referenced from the size of tx pool. // The number is referenced from the size of tx pool.
txChanSize = 4096 txChanSize = 4096
bloomBitsSection = 4096
) )
var ( var (
...@@ -94,8 +92,6 @@ type ProtocolManager struct { ...@@ -94,8 +92,6 @@ type ProtocolManager struct {
quitSync chan struct{} quitSync chan struct{}
noMorePeers chan struct{} noMorePeers chan struct{}
lesServer LesServer
// wait group is used for graceful shutdowns during downloading // wait group is used for graceful shutdowns during downloading
// and processing // and processing
wg sync.WaitGroup wg sync.WaitGroup
...@@ -118,7 +114,6 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne ...@@ -118,7 +114,6 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne
txsyncCh: make(chan *txsync), txsyncCh: make(chan *txsync),
quitSync: make(chan struct{}), quitSync: make(chan struct{}),
} }
// Figure out whether to allow fast sync or not // Figure out whether to allow fast sync or not
if mode == downloader.FastSync && blockchain.CurrentBlock().NumberU64() > 0 { if mode == downloader.FastSync && blockchain.CurrentBlock().NumberU64() > 0 {
log.Warn("Blockchain not empty, fast sync disabled") log.Warn("Blockchain not empty, fast sync disabled")
......
les/api_backend.go
View file @ f585f9ee
...@@ -19,17 +19,16 @@ package les ...@@ -19,17 +19,16 @@ package les
import ( import (
"context" "context"
"math/big" "math/big"
"time"
"github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/filters"
"github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/gasprice"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/event"
...@@ -174,18 +173,9 @@ func (b *LesApiBackend) AccountManager() *accounts.Manager { ...@@ -174,18 +173,9 @@ func (b *LesApiBackend) AccountManager() *accounts.Manager {
return b.eth.accountManager return b.eth.accountManager
} }
func (b *LesApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) { func (b *LesApiBackend) BloomStatus() (uint64, uint64) {
return nil, nil // implemented in a subsequent PR return params.BloomBitsBlocks, 0
} }
func (b *LesApiBackend) BloomBitsSections() uint64 { func (b *LesApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {
return 0
}
func (b *LesApiBackend) BloomBitsConfig() filters.BloomConfig {
return filters.BloomConfig{
SectionSize: 32768,
MaxRequestLen: 16,
MaxRequestWait: time.Microsecond * 100,
}
} }
les/backend.go
View file @ f585f9ee
...@@ -169,7 +169,7 @@ func (s *LightEthereum) APIs() []rpc.API { ...@@ -169,7 +169,7 @@ func (s *LightEthereum) APIs() []rpc.API {
}, { }, {
Namespace: "eth", Namespace: "eth",
Version: "1.0", Version: "1.0",
Service: filters.NewPublicFilterAPI(s.ApiBackend, true, 0), Service: filters.NewPublicFilterAPI(s.ApiBackend, true),
Public: true, Public: true,
}, { }, {
Namespace: "net", Namespace: "net",
......
params/network_params.go 0 → 100644
View file @ f585f9ee
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package params
// These are network parameters that need to be constant between clients, but
// aren't necesarilly consensus related.
const (
// BloomBitsBlocks is the number of blocks a single bloom bit section vector
// contains.
BloomBitsBlocks uint64 = 4096
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment