Skip to content

G
Geth-Modification
Unverified Commit e5676754 authored by Péter Szilágyi's avatar Péter Szilágyi
Browse files
Options

core/state/snapshot: move iterator out into its own files

parent 7e389963
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 838 additions and 652 deletions
core/state/snapshot/difflayer.go
View file @ e5676754
...@@ -18,7 +18,6 @@ package snapshot ...@@ -18,7 +18,6 @@ package snapshot
import ( import (
"encoding/binary" "encoding/binary"
"bytes"
"fmt" "fmt"
"math" "math"
"math/rand" "math/rand"
...@@ -476,291 +475,3 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash { ...@@ -476,291 +475,3 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash {
dl.storageList[accountHash] = accountStorageList dl.storageList[accountHash] = accountStorageList
return accountStorageList return accountStorageList
} }
type Iterator interface {
// Next steps the iterator forward one element, and returns false if
// the iterator is exhausted
Next() bool
// Key returns the current key
Key() common.Hash
// Seek steps the iterator forward as many elements as needed, so that after
// calling Next(), the iterator will be at a key higher than the given hash
Seek(common.Hash)
}
func (dl *diffLayer) newIterator() Iterator {
dl.AccountList()
return &dlIterator{dl, -1}
}
type dlIterator struct {
layer *diffLayer
index int
}
func (it *dlIterator) Next() bool {
if it.index < len(it.layer.accountList) {
it.index++
}
return it.index < len(it.layer.accountList)
}
func (it *dlIterator) Key() common.Hash {
if it.index < len(it.layer.accountList) {
return it.layer.accountList[it.index]
}
return common.Hash{}
}
func (it *dlIterator) Seek(key common.Hash) {
// Search uses binary search to find and return the smallest index i
// in [0, n) at which f(i) is true
size := len(it.layer.accountList)
index := sort.Search(size,
func(i int) bool {
v := it.layer.accountList[i]
return bytes.Compare(key[:], v[:]) < 0
})
it.index = index - 1
}
type binaryIterator struct {
a Iterator
b Iterator
aDone bool
bDone bool
k common.Hash
}
func (dl *diffLayer) newBinaryIterator() Iterator {
parent, ok := dl.parent.(*diffLayer)
if !ok {
// parent is the disk layer
return dl.newIterator()
}
l := &binaryIterator{
a: dl.newIterator(),
b: parent.newBinaryIterator()}
l.aDone = !l.a.Next()
l.bDone = !l.b.Next()
return l
}
func (it *binaryIterator) Next() bool {
if it.aDone && it.bDone {
return false
}
nextB := it.b.Key()
first:
nextA := it.a.Key()
if it.aDone {
it.bDone = !it.b.Next()
it.k = nextB
return true
}
if it.bDone {
it.aDone = !it.a.Next()
it.k = nextA
return true
}
if diff := bytes.Compare(nextA[:], nextB[:]); diff < 0 {
it.aDone = !it.a.Next()
it.k = nextA
return true
} else if diff == 0 {
// Now we need to advance one of them
it.aDone = !it.a.Next()
goto first
}
it.bDone = !it.b.Next()
it.k = nextB
return true
}
func (it *binaryIterator) Key() common.Hash {
return it.k
}
func (it *binaryIterator) Seek(key common.Hash) {
panic("todo: implement")
}
func (dl *diffLayer) iterators() []Iterator {
if parent, ok := dl.parent.(*diffLayer); ok {
iterators := parent.iterators()
return append(iterators, dl.newIterator())
}
return []Iterator{dl.newIterator()}
}
// fastIterator is a more optimized multi-layer iterator which maintains a
// direct mapping of all iterators leading down to the bottom layer
type fastIterator struct {
iterators []Iterator
initiated bool
}
// Len returns the number of active iterators
func (fi *fastIterator) Len() int {
return len(fi.iterators)
}
// Less implements sort.Interface
func (fi *fastIterator) Less(i, j int) bool {
a := fi.iterators[i].Key()
b := fi.iterators[j].Key()
return bytes.Compare(a[:], b[:]) < 0
}
// Swap implements sort.Interface
func (fi *fastIterator) Swap(i, j int) {
fi.iterators[i], fi.iterators[j] = fi.iterators[j], fi.iterators[i]
}
// Next implements the Iterator interface. It returns false if no more elemnts
// can be retrieved (false == exhausted)
func (fi *fastIterator) Next() bool {
if len(fi.iterators) == 0 {
return false
}
if !fi.initiated {
// Don't forward first time -- we had to 'Next' once in order to
// do the sorting already
fi.initiated = true
return true
}
return fi.innerNext(0)
}
// innerNext handles the next operation internally,
// and should be invoked when we know that two elements in the list may have
// the same value.
// For example, if the list becomes [2,3,5,5,8,9,10], then we should invoke
// innerNext(3), which will call Next on elem 3 (the second '5'). It will continue
// along the list and apply the same operation if needed
func (fi *fastIterator) innerNext(pos int) bool {
if !fi.iterators[pos].Next() {
//Exhausted, remove this iterator
fi.remove(pos)
if len(fi.iterators) == 0 {
return false
}
return true
}
if pos == len(fi.iterators)-1 {
// Only one iterator left
return true
}
// We next:ed the elem at 'pos'. Now we may have to re-sort that elem
val, neighbour := fi.iterators[pos].Key(), fi.iterators[pos+1].Key()
diff := bytes.Compare(val[:], neighbour[:])
if diff < 0 {
// It is still in correct place
return true
}
if diff == 0 {
// It has same value as the neighbour. So still in correct place, but
// we need to iterate on the neighbour
fi.innerNext(pos + 1)
return true
}
// At this point, the elem is in the wrong location, but the
// remaining list is sorted. Find out where to move the elem
iterationNeeded := false
index := sort.Search(len(fi.iterators), func(n int) bool {
if n <= pos {
// No need to search 'behind' us
return false
}
if n == len(fi.iterators)-1 {
// Can always place an elem last
return true
}
neighbour := fi.iterators[n+1].Key()
diff := bytes.Compare(val[:], neighbour[:])
if diff == 0 {
// The elem we're placing it next to has the same value,
// so it's going to need further iteration
iterationNeeded = true
}
return diff < 0
})
fi.move(pos, index)
if iterationNeeded {
fi.innerNext(index)
}
return true
}
// move moves an iterator to another position in the list
func (fi *fastIterator) move(index, newpos int) {
if newpos > len(fi.iterators)-1 {
newpos = len(fi.iterators) - 1
}
var (
elem = fi.iterators[index]
middle = fi.iterators[index+1 : newpos+1]
suffix []Iterator
)
if newpos < len(fi.iterators)-1 {
suffix = fi.iterators[newpos+1:]
}
fi.iterators = append(fi.iterators[:index], middle...)
fi.iterators = append(fi.iterators, elem)
fi.iterators = append(fi.iterators, suffix...)
}
// remove drops an iterator from the list
func (fi *fastIterator) remove(index int) {
fi.iterators = append(fi.iterators[:index], fi.iterators[index+1:]...)
}
// Key returns the current key
func (fi *fastIterator) Key() common.Hash {
return fi.iterators[0].Key()
}
func (fi *fastIterator) Seek(key common.Hash) {
// We need to apply this across all iterators
var seen = make(map[common.Hash]struct{})
length := len(fi.iterators)
for i, it := range fi.iterators {
it.Seek(key)
for {
if !it.Next() {
// To be removed
// swap it to the last position for now
fi.iterators[i], fi.iterators[length-1] = fi.iterators[length-1], fi.iterators[i]
length--
break
}
v := it.Key()
if _, exist := seen[v]; !exist {
seen[v] = struct{}{}
break
}
}
}
// Now remove those that were placed in the end
fi.iterators = fi.iterators[:length]
// The list is now totally unsorted, need to re-sort the entire list
sort.Sort(fi)
fi.initiated = false
}
// The fast iterator does not query parents as much.
func (dl *diffLayer) newFastIterator() Iterator {
f := &fastIterator{dl.iterators(), false}
f.Seek(common.Hash{})
return f
}
// Debug is a convencience helper during testing
func (fi *fastIterator) Debug() {
for _, it := range fi.iterators {
fmt.Printf(" %v ", it.Key()[31])
}
fmt.Println()
}
core/state/snapshot/difflayer_test.go
View file @ e5676754
This diff is collapsed.
core/state/snapshot/iterator.go 0 → 100644
View file @ e5676754
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"sort"
"github.com/ethereum/go-ethereum/common"
)
// AccountIterator is an iterator to step over all the accounts in a snapshot,
// which may or may npt be composed of multiple layers.
type AccountIterator interface {
// Seek steps the iterator forward as many elements as needed, so that after
// calling Next(), the iterator will be at a key higher than the given hash.
Seek(hash common.Hash)
// Next steps the iterator forward one element, returning false if exhausted,
// or an error if iteration failed for some reason (e.g. root being iterated
// becomes stale and garbage collected).
Next() bool
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit (e.g. snapshot stack becoming stale).
Error() error
// Key returns the hash of the account the iterator is currently at.
Key() common.Hash
// Value returns the RLP encoded slim account the iterator is currently at.
// An error will be returned if the iterator becomes invalid (e.g. snaph
Value() []byte
}
// diffAccountIterator is an account iterator that steps over the accounts (both
// live and deleted) contained within a single
type diffAccountIterator struct {
layer *diffLayer
index int
}
func (dl *diffLayer) newAccountIterator() *diffAccountIterator {
dl.AccountList()
return &diffAccountIterator{layer: dl, index: -1}
}
// Seek steps the iterator forward as many elements as needed, so that after
// calling Next(), the iterator will be at a key higher than the given hash.
func (it *diffAccountIterator) Seek(key common.Hash) {
// Search uses binary search to find and return the smallest index i
// in [0, n) at which f(i) is true
index := sort.Search(len(it.layer.accountList), func(i int) bool {
return bytes.Compare(key[:], it.layer.accountList[i][:]) < 0
})
it.index = index - 1
}
// Next steps the iterator forward one element, returning false if exhausted.
func (it *diffAccountIterator) Next() bool {
if it.index < len(it.layer.accountList) {
it.index++
}
return it.index < len(it.layer.accountList)
}
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit (e.g. snapshot stack becoming stale).
//
// A diff layer is immutable after creation content wise and can always be fully
// iterated without error, so this method always returns nil.
func (it *diffAccountIterator) Error() error {
return nil
}
// Key returns the hash of the account the iterator is currently at.
func (it *diffAccountIterator) Key() common.Hash {
if it.index < len(it.layer.accountList) {
return it.layer.accountList[it.index]
}
return common.Hash{}
}
// Value returns the RLP encoded slim account the iterator is currently at.
func (it *diffAccountIterator) Value() []byte {
it.layer.lock.RLock()
defer it.layer.lock.RUnlock()
hash := it.layer.accountList[it.index]
if data, ok := it.layer.accountData[hash]; ok {
return data
}
panic("iterator references non-existent layer account")
}
func (dl *diffLayer) iterators() []AccountIterator {
if parent, ok := dl.parent.(*diffLayer); ok {
iterators := parent.iterators()
return append(iterators, dl.newAccountIterator())
}
return []AccountIterator{dl.newAccountIterator()}
}
core/state/snapshot/iterator_binary.go 0 → 100644
View file @ e5676754
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"github.com/ethereum/go-ethereum/common"
)
// binaryAccountIterator is a simplistic iterator to step over the accounts in
// a snapshot, which may or may npt be composed of multiple layers. Performance
// wise this iterator is slow, it's meant for cross validating the fast one,
type binaryAccountIterator struct {
a *diffAccountIterator
b AccountIterator
aDone bool
bDone bool
k common.Hash
fail error
}
// newBinaryAccountIterator creates a simplistic account iterator to step over
// all the accounts in a slow, but eaily verifyable way.
func (dl *diffLayer) newBinaryAccountIterator() AccountIterator {
parent, ok := dl.parent.(*diffLayer)
if !ok {
// parent is the disk layer
return dl.newAccountIterator()
}
l := &binaryAccountIterator{
a: dl.newAccountIterator(),
b: parent.newBinaryAccountIterator(),
}
l.aDone = !l.a.Next()
l.bDone = !l.b.Next()
return l
}
// Seek steps the iterator forward as many elements as needed, so that after
// calling Next(), the iterator will be at a key higher than the given hash.
func (it *binaryAccountIterator) Seek(key common.Hash) {
panic("todo: implement")
}
// Next steps the iterator forward one element, returning false if exhausted,
// or an error if iteration failed for some reason (e.g. root being iterated
// becomes stale and garbage collected).
func (it *binaryAccountIterator) Next() bool {
if it.aDone && it.bDone {
return false
}
nextB := it.b.Key()
first:
nextA := it.a.Key()
if it.aDone {
it.bDone = !it.b.Next()
it.k = nextB
return true
}
if it.bDone {
it.aDone = !it.a.Next()
it.k = nextA
return true
}
if diff := bytes.Compare(nextA[:], nextB[:]); diff < 0 {
it.aDone = !it.a.Next()
it.k = nextA
return true
} else if diff == 0 {
// Now we need to advance one of them
it.aDone = !it.a.Next()
goto first
}
it.bDone = !it.b.Next()
it.k = nextB
return true
}
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit (e.g. snapshot stack becoming stale).
func (it *binaryAccountIterator) Error() error {
return it.fail
}
// Key returns the hash of the account the iterator is currently at.
func (it *binaryAccountIterator) Key() common.Hash {
return it.k
}
// Value returns the RLP encoded slim account the iterator is currently at, or
// nil if the iterated snapshot stack became stale (you can check Error after
// to see if it failed or not).
func (it *binaryAccountIterator) Value() []byte {
blob, err := it.a.layer.AccountRLP(it.k)
if err != nil {
it.fail = err
return nil
}
return blob
}
core/state/snapshot/iterator_fast.go 0 → 100644
View file @ e5676754
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"fmt"
"sort"
"github.com/ethereum/go-ethereum/common"
)
// fastAccountIterator is a more optimized multi-layer iterator which maintains a
// direct mapping of all iterators leading down to the bottom layer
type fastAccountIterator struct {
iterators []AccountIterator
initiated bool
fail error
}
// The fast iterator does not query parents as much.
func (dl *diffLayer) newFastAccountIterator() AccountIterator {
f := &fastAccountIterator{
iterators: dl.iterators(),
initiated: false,
}
f.Seek(common.Hash{})
return f
}
// Len returns the number of active iterators
func (fi *fastAccountIterator) Len() int {
return len(fi.iterators)
}
// Less implements sort.Interface
func (fi *fastAccountIterator) Less(i, j int) bool {
a := fi.iterators[i].Key()
b := fi.iterators[j].Key()
return bytes.Compare(a[:], b[:]) < 0
}
// Swap implements sort.Interface
func (fi *fastAccountIterator) Swap(i, j int) {
fi.iterators[i], fi.iterators[j] = fi.iterators[j], fi.iterators[i]
}
func (fi *fastAccountIterator) Seek(key common.Hash) {
// We need to apply this across all iterators
var seen = make(map[common.Hash]struct{})
length := len(fi.iterators)
for i, it := range fi.iterators {
it.Seek(key)
for {
if !it.Next() {
// To be removed
// swap it to the last position for now
fi.iterators[i], fi.iterators[length-1] = fi.iterators[length-1], fi.iterators[i]
length--
break
}
v := it.Key()
if _, exist := seen[v]; !exist {
seen[v] = struct{}{}
break
}
}
}
// Now remove those that were placed in the end
fi.iterators = fi.iterators[:length]
// The list is now totally unsorted, need to re-sort the entire list
sort.Sort(fi)
fi.initiated = false
}
// Next implements the Iterator interface. It returns false if no more elemnts
// can be retrieved (false == exhausted)
func (fi *fastAccountIterator) Next() bool {
if len(fi.iterators) == 0 {
return false
}
if !fi.initiated {
// Don't forward first time -- we had to 'Next' once in order to
// do the sorting already
fi.initiated = true
return true
}
return fi.innerNext(0)
}
// innerNext handles the next operation internally,
// and should be invoked when we know that two elements in the list may have
// the same value.
// For example, if the list becomes [2,3,5,5,8,9,10], then we should invoke
// innerNext(3), which will call Next on elem 3 (the second '5'). It will continue
// along the list and apply the same operation if needed
func (fi *fastAccountIterator) innerNext(pos int) bool {
if !fi.iterators[pos].Next() {
//Exhausted, remove this iterator
fi.remove(pos)
if len(fi.iterators) == 0 {
return false
}
return true
}
if pos == len(fi.iterators)-1 {
// Only one iterator left
return true
}
// We next:ed the elem at 'pos'. Now we may have to re-sort that elem
val, neighbour := fi.iterators[pos].Key(), fi.iterators[pos+1].Key()
diff := bytes.Compare(val[:], neighbour[:])
if diff < 0 {
// It is still in correct place
return true
}
if diff == 0 {
// It has same value as the neighbour. So still in correct place, but
// we need to iterate on the neighbour
fi.innerNext(pos + 1)
return true
}
// At this point, the elem is in the wrong location, but the
// remaining list is sorted. Find out where to move the elem
iterationNeeded := false
index := sort.Search(len(fi.iterators), func(n int) bool {
if n <= pos {
// No need to search 'behind' us
return false
}
if n == len(fi.iterators)-1 {
// Can always place an elem last
return true
}
neighbour := fi.iterators[n+1].Key()
diff := bytes.Compare(val[:], neighbour[:])
if diff == 0 {
// The elem we're placing it next to has the same value,
// so it's going to need further iteration
iterationNeeded = true
}
return diff < 0
})
fi.move(pos, index)
if iterationNeeded {
fi.innerNext(index)
}
return true
}
// move moves an iterator to another position in the list
func (fi *fastAccountIterator) move(index, newpos int) {
if newpos > len(fi.iterators)-1 {
newpos = len(fi.iterators) - 1
}
var (
elem = fi.iterators[index]
middle = fi.iterators[index+1 : newpos+1]
suffix []AccountIterator
)
if newpos < len(fi.iterators)-1 {
suffix = fi.iterators[newpos+1:]
}
fi.iterators = append(fi.iterators[:index], middle...)
fi.iterators = append(fi.iterators, elem)
fi.iterators = append(fi.iterators, suffix...)
}
// remove drops an iterator from the list
func (fi *fastAccountIterator) remove(index int) {
fi.iterators = append(fi.iterators[:index], fi.iterators[index+1:]...)
}
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit (e.g. snapshot stack becoming stale).
func (fi *fastAccountIterator) Error() error {
return fi.fail
}
// Key returns the current key
func (fi *fastAccountIterator) Key() common.Hash {
return fi.iterators[0].Key()
}
// Value returns the current key
func (fi *fastAccountIterator) Value() []byte {
panic("todo")
}
// Debug is a convencience helper during testing
func (fi *fastAccountIterator) Debug() {
for _, it := range fi.iterators {
fmt.Printf(" %v ", it.Key()[31])
}
fmt.Println()
}
core/state/snapshot/iterator_test.go 0 → 100644
View file @ e5676754
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment