core/state/snapshot: unlink snapshots from blocks, quad->linear cleanup

parent cdf3f016
...@@ -140,10 +140,10 @@ type BlockChain struct { ...@@ -140,10 +140,10 @@ type BlockChain struct {
chainConfig *params.ChainConfig // Chain & network configuration chainConfig *params.ChainConfig // Chain & network configuration
cacheConfig *CacheConfig // Cache configuration for pruning cacheConfig *CacheConfig // Cache configuration for pruning
db ethdb.Database // Low level persistent database to store final content in db ethdb.Database // Low level persistent database to store final content in
snaps *snapshot.SnapshotTree // Snapshot tree for fast trie leaf access snaps *snapshot.Tree // Snapshot tree for fast trie leaf access
triegc *prque.Prque // Priority queue mapping block numbers to tries to gc triegc *prque.Prque // Priority queue mapping block numbers to tries to gc
gcproc time.Duration // Accumulates canonical block processing for trie dumping gcproc time.Duration // Accumulates canonical block processing for trie dumping
hc *HeaderChain hc *HeaderChain
rmLogsFeed event.Feed rmLogsFeed event.Feed
...@@ -301,7 +301,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par ...@@ -301,7 +301,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
} }
// Load any existing snapshot, regenerating it if loading failed // Load any existing snapshot, regenerating it if loading failed
head := bc.CurrentBlock() head := bc.CurrentBlock()
if bc.snaps, err = snapshot.New(bc.db, "snapshot.rlp", head.NumberU64(), head.Root()); err != nil { if bc.snaps, err = snapshot.New(bc.db, "snapshot.rlp", head.Root()); err != nil {
return nil, err return nil, err
} }
// Take ownership of this particular state // Take ownership of this particular state
......
...@@ -17,38 +17,36 @@ ...@@ -17,38 +17,36 @@
package rawdb package rawdb
import ( import (
"encoding/binary"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
) )
// ReadSnapshotBlock retrieves the number and root of the block whose state is // ReadSnapshotRoot retrieves the root of the block whose state is contained in
// contained in the persisted snapshot. // the persisted snapshot.
func ReadSnapshotBlock(db ethdb.KeyValueReader) (uint64, common.Hash) { func ReadSnapshotRoot(db ethdb.KeyValueReader) common.Hash {
data, _ := db.Get(snapshotBlockKey) data, _ := db.Get(snapshotRootKey)
if len(data) != 8+common.HashLength { if len(data) != common.HashLength {
return 0, common.Hash{} return common.Hash{}
} }
return binary.BigEndian.Uint64(data[:8]), common.BytesToHash(data[8:]) return common.BytesToHash(data)
} }
// WriteSnapshotBlock stores the number and root of the block whose state is // WriteSnapshotRoot stores the root of the block whose state is contained in
// contained in the persisted snapshot. // the persisted snapshot.
func WriteSnapshotBlock(db ethdb.KeyValueWriter, number uint64, root common.Hash) { func WriteSnapshotRoot(db ethdb.KeyValueWriter, root common.Hash) {
if err := db.Put(snapshotBlockKey, append(encodeBlockNumber(number), root.Bytes()...)); err != nil { if err := db.Put(snapshotRootKey, root[:]); err != nil {
log.Crit("Failed to store snapsnot block's number and root", "err", err) log.Crit("Failed to store snapshot root", "err", err)
} }
} }
// DeleteSnapshotBlock deletes the number and hash of the block whose state is // DeleteSnapshotRoot deletes the hash of the block whose state is contained in
// contained in the persisted snapshot. Since snapshots are not immutable, this // the persisted snapshot. Since snapshots are not immutable, this method can
// method can be used during updates, so a crash or failure will mark the entire // be used during updates, so a crash or failure will mark the entire snapshot
// snapshot invalid. // invalid.
func DeleteSnapshotBlock(db ethdb.KeyValueWriter) { func DeleteSnapshotRoot(db ethdb.KeyValueWriter) {
if err := db.Delete(snapshotBlockKey); err != nil { if err := db.Delete(snapshotRootKey); err != nil {
log.Crit("Failed to remove snapsnot block's number and hash", "err", err) log.Crit("Failed to remove snapshot root", "err", err)
} }
} }
......
...@@ -41,8 +41,8 @@ var ( ...@@ -41,8 +41,8 @@ var (
// fastTrieProgressKey tracks the number of trie entries imported during fast sync. // fastTrieProgressKey tracks the number of trie entries imported during fast sync.
fastTrieProgressKey = []byte("TrieSync") fastTrieProgressKey = []byte("TrieSync")
// snapshotBlockKey tracks the number and hash of the last snapshot. // snapshotRootKey tracks the number and hash of the last snapshot.
snapshotBlockKey = []byte("SnapshotBlock") snapshotRootKey = []byte("SnapshotRoot")
// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
......
...@@ -36,9 +36,8 @@ type diffLayer struct { ...@@ -36,9 +36,8 @@ type diffLayer struct {
parent snapshot // Parent snapshot modified by this one, never nil parent snapshot // Parent snapshot modified by this one, never nil
memory uint64 // Approximate guess as to how much memory we use memory uint64 // Approximate guess as to how much memory we use
number uint64 // Block number to which this snapshot diff belongs to root common.Hash // Root hash to which this snapshot diff belongs to
root common.Hash // Root hash to which this snapshot diff belongs to stale bool // Signals that the layer became stale (state progressed)
stale bool // Signals that the layer became stale (state progressed)
accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil
accountData map[common.Hash][]byte // Keyed accounts for direct retrival (nil means deleted) accountData map[common.Hash][]byte // Keyed accounts for direct retrival (nil means deleted)
...@@ -50,11 +49,10 @@ type diffLayer struct { ...@@ -50,11 +49,10 @@ type diffLayer struct {
// newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low // newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low
// level persistent database or a hierarchical diff already. // level persistent database or a hierarchical diff already.
func newDiffLayer(parent snapshot, number uint64, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { func newDiffLayer(parent snapshot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
// Create the new layer with some pre-allocated data segments // Create the new layer with some pre-allocated data segments
dl := &diffLayer{ dl := &diffLayer{
parent: parent, parent: parent,
number: number,
root: root, root: root,
accountData: accounts, accountData: accounts,
storageData: storage, storageData: storage,
...@@ -63,7 +61,6 @@ func newDiffLayer(parent snapshot, number uint64, root common.Hash, accounts map ...@@ -63,7 +61,6 @@ func newDiffLayer(parent snapshot, number uint64, root common.Hash, accounts map
for _, data := range accounts { for _, data := range accounts {
dl.memory += uint64(len(data)) dl.memory += uint64(len(data))
} }
// Fill the storage hashes and sort them for the iterator // Fill the storage hashes and sort them for the iterator
dl.storageList = make(map[common.Hash][]common.Hash) dl.storageList = make(map[common.Hash][]common.Hash)
...@@ -93,9 +90,18 @@ func newDiffLayer(parent snapshot, number uint64, root common.Hash, accounts map ...@@ -93,9 +90,18 @@ func newDiffLayer(parent snapshot, number uint64, root common.Hash, accounts map
return dl return dl
} }
// Info returns the block number and root hash for which this snapshot was made. // Root returns the root hash for which this snapshot was made.
func (dl *diffLayer) Info() (uint64, common.Hash) { func (dl *diffLayer) Root() common.Hash {
return dl.number, dl.root return dl.root
}
// Stale return whether this layer has become stale (was flattened across) or if
// it's still live.
func (dl *diffLayer) Stale() bool {
dl.lock.RLock()
defer dl.lock.RUnlock()
return dl.stale
} }
// Account directly retrieves the account associated with a particular hash in // Account directly retrieves the account associated with a particular hash in
...@@ -164,7 +170,7 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro ...@@ -164,7 +170,7 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro
// Update creates a new layer on top of the existing snapshot diff tree with // Update creates a new layer on top of the existing snapshot diff tree with
// the specified data items. // the specified data items.
func (dl *diffLayer) Update(blockRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { func (dl *diffLayer) Update(blockRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
return newDiffLayer(dl, dl.number+1, blockRoot, accounts, storage) return newDiffLayer(dl, blockRoot, accounts, storage)
} }
// flatten pushes all data from this point downwards, flattening everything into // flatten pushes all data from this point downwards, flattening everything into
...@@ -213,7 +219,6 @@ func (dl *diffLayer) flatten() snapshot { ...@@ -213,7 +219,6 @@ func (dl *diffLayer) flatten() snapshot {
// Return the combo parent // Return the combo parent
return &diffLayer{ return &diffLayer{
parent: parent.parent, parent: parent.parent,
number: dl.number,
root: dl.root, root: dl.root,
storageList: parent.storageList, storageList: parent.storageList,
storageData: parent.storageData, storageData: parent.storageData,
......
...@@ -43,18 +43,12 @@ type journalStorage struct { ...@@ -43,18 +43,12 @@ type journalStorage struct {
// diff and verifying that it can be linked to the requested parent. // diff and verifying that it can be linked to the requested parent.
func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) { func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) {
// Read the next diff journal entry // Read the next diff journal entry
var ( var root common.Hash
number uint64 if err := r.Decode(&root); err != nil {
root common.Hash
)
if err := r.Decode(&number); err != nil {
// The first read may fail with EOF, marking the end of the journal // The first read may fail with EOF, marking the end of the journal
if err == io.EOF { if err == io.EOF {
return parent, nil return parent, nil
} }
return nil, fmt.Errorf("load diff number: %v", err)
}
if err := r.Decode(&root); err != nil {
return nil, fmt.Errorf("load diff root: %v", err) return nil, fmt.Errorf("load diff root: %v", err)
} }
var accounts []journalAccount var accounts []journalAccount
...@@ -77,13 +71,7 @@ func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) { ...@@ -77,13 +71,7 @@ func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) {
} }
storageData[entry.Hash] = slots storageData[entry.Hash] = slots
} }
// Validate the block number to avoid state corruption return loadDiffLayer(newDiffLayer(parent, root, accountData, storageData), r)
if parent, ok := parent.(*diffLayer); ok {
if number != parent.number+1 {
return nil, fmt.Errorf("snapshot chain broken: block #%d after #%d", number, parent.number)
}
}
return loadDiffLayer(newDiffLayer(parent, number, root, accountData, storageData), r)
} }
// journal is the internal version of Journal that also returns the journal file // journal is the internal version of Journal that also returns the journal file
...@@ -113,13 +101,8 @@ func (dl *diffLayer) journal() (io.WriteCloser, error) { ...@@ -113,13 +101,8 @@ func (dl *diffLayer) journal() (io.WriteCloser, error) {
writer.Close() writer.Close()
return nil, ErrSnapshotStale return nil, ErrSnapshotStale
} }
buf := bufio.NewWriter(writer)
// Everything below was journalled, persist this layer too // Everything below was journalled, persist this layer too
if err := rlp.Encode(buf, dl.number); err != nil { buf := bufio.NewWriter(writer)
buf.Flush()
writer.Close()
return nil, err
}
if err := rlp.Encode(buf, dl.root); err != nil { if err := rlp.Encode(buf, dl.root); err != nil {
buf.Flush() buf.Flush()
writer.Close() writer.Close()
......
...@@ -61,11 +61,11 @@ func TestMergeBasics(t *testing.T) { ...@@ -61,11 +61,11 @@ func TestMergeBasics(t *testing.T) {
} }
} }
// Add some (identical) layers on top // Add some (identical) layers on top
parent := newDiffLayer(emptyLayer{}, 1, common.Hash{}, accounts, storage) parent := newDiffLayer(emptyLayer{}, common.Hash{}, accounts, storage)
child := newDiffLayer(parent, 1, common.Hash{}, accounts, storage) child := newDiffLayer(parent, common.Hash{}, accounts, storage)
child = newDiffLayer(child, 1, common.Hash{}, accounts, storage) child = newDiffLayer(child, common.Hash{}, accounts, storage)
child = newDiffLayer(child, 1, common.Hash{}, accounts, storage) child = newDiffLayer(child, common.Hash{}, accounts, storage)
child = newDiffLayer(child, 1, common.Hash{}, accounts, storage) child = newDiffLayer(child, common.Hash{}, accounts, storage)
// And flatten // And flatten
merged := (child.flatten()).(*diffLayer) merged := (child.flatten()).(*diffLayer)
...@@ -122,7 +122,7 @@ func TestMergeDelete(t *testing.T) { ...@@ -122,7 +122,7 @@ func TestMergeDelete(t *testing.T) {
} }
// Add some flip-flopping layers on top // Add some flip-flopping layers on top
parent := newDiffLayer(emptyLayer{}, 1, common.Hash{}, flip(), storage) parent := newDiffLayer(emptyLayer{}, common.Hash{}, flip(), storage)
child := parent.Update(common.Hash{}, flop(), storage) child := parent.Update(common.Hash{}, flop(), storage)
child = child.Update(common.Hash{}, flip(), storage) child = child.Update(common.Hash{}, flip(), storage)
child = child.Update(common.Hash{}, flop(), storage) child = child.Update(common.Hash{}, flop(), storage)
...@@ -139,10 +139,6 @@ func TestMergeDelete(t *testing.T) { ...@@ -139,10 +139,6 @@ func TestMergeDelete(t *testing.T) {
// And flatten // And flatten
merged := (child.flatten()).(*diffLayer) merged := (child.flatten()).(*diffLayer)
// check number
if got, exp := merged.number, child.number; got != exp {
t.Errorf("merged layer: wrong number - exp %d got %d", exp, got)
}
if data, _ := merged.Account(h1); data == nil { if data, _ := merged.Account(h1); data == nil {
t.Errorf("merged layer: expected %x to be non-nil", h1) t.Errorf("merged layer: expected %x to be non-nil", h1)
} }
...@@ -169,7 +165,7 @@ func TestInsertAndMerge(t *testing.T) { ...@@ -169,7 +165,7 @@ func TestInsertAndMerge(t *testing.T) {
{ {
var accounts = make(map[common.Hash][]byte) var accounts = make(map[common.Hash][]byte)
var storage = make(map[common.Hash]map[common.Hash][]byte) var storage = make(map[common.Hash]map[common.Hash][]byte)
parent = newDiffLayer(emptyLayer{}, 1, common.Hash{}, accounts, storage) parent = newDiffLayer(emptyLayer{}, common.Hash{}, accounts, storage)
} }
{ {
var accounts = make(map[common.Hash][]byte) var accounts = make(map[common.Hash][]byte)
...@@ -178,7 +174,7 @@ func TestInsertAndMerge(t *testing.T) { ...@@ -178,7 +174,7 @@ func TestInsertAndMerge(t *testing.T) {
accstorage := make(map[common.Hash][]byte) accstorage := make(map[common.Hash][]byte)
storage[acc] = accstorage storage[acc] = accstorage
storage[acc][slot] = []byte{0x01} storage[acc][slot] = []byte{0x01}
child = newDiffLayer(parent, 2, common.Hash{}, accounts, storage) child = newDiffLayer(parent, common.Hash{}, accounts, storage)
} }
// And flatten // And flatten
merged := (child.flatten()).(*diffLayer) merged := (child.flatten()).(*diffLayer)
...@@ -200,11 +196,12 @@ func (emptyLayer) Journal() error { ...@@ -200,11 +196,12 @@ func (emptyLayer) Journal() error {
panic("implement me") panic("implement me")
} }
func (emptyLayer) Info() (uint64, common.Hash) { func (emptyLayer) Stale() bool {
return 0, common.Hash{} panic("implement me")
} }
func (emptyLayer) Number() uint64 {
return 0 func (emptyLayer) Root() common.Hash {
return common.Hash{}
} }
func (emptyLayer) Account(hash common.Hash) (*Account, error) { func (emptyLayer) Account(hash common.Hash) (*Account, error) {
...@@ -227,8 +224,6 @@ func (emptyLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error) ...@@ -227,8 +224,6 @@ func (emptyLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error)
// BenchmarkSearch-6 500000 3723 ns/op (10k per layer, only top-level RLock() // BenchmarkSearch-6 500000 3723 ns/op (10k per layer, only top-level RLock()
func BenchmarkSearch(b *testing.B) { func BenchmarkSearch(b *testing.B) {
// First, we set up 128 diff layers, with 1K items each // First, we set up 128 diff layers, with 1K items each
blocknum := uint64(0)
fill := func(parent snapshot) *diffLayer { fill := func(parent snapshot) *diffLayer {
accounts := make(map[common.Hash][]byte) accounts := make(map[common.Hash][]byte)
storage := make(map[common.Hash]map[common.Hash][]byte) storage := make(map[common.Hash]map[common.Hash][]byte)
...@@ -236,10 +231,8 @@ func BenchmarkSearch(b *testing.B) { ...@@ -236,10 +231,8 @@ func BenchmarkSearch(b *testing.B) {
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
accounts[randomHash()] = randomAccount() accounts[randomHash()] = randomAccount()
} }
blocknum++ return newDiffLayer(parent, common.Hash{}, accounts, storage)
return newDiffLayer(parent, blocknum, common.Hash{}, accounts, storage)
} }
var layer snapshot var layer snapshot
layer = emptyLayer{} layer = emptyLayer{}
for i := 0; i < 128; i++ { for i := 0; i < 128; i++ {
...@@ -261,8 +254,6 @@ func BenchmarkSearch(b *testing.B) { ...@@ -261,8 +254,6 @@ func BenchmarkSearch(b *testing.B) {
// BenchmarkSearchSlot-6 100000 14551 ns/op (when checking parent number using atomic) // BenchmarkSearchSlot-6 100000 14551 ns/op (when checking parent number using atomic)
func BenchmarkSearchSlot(b *testing.B) { func BenchmarkSearchSlot(b *testing.B) {
// First, we set up 128 diff layers, with 1K items each // First, we set up 128 diff layers, with 1K items each
blocknum := uint64(0)
accountKey := common.Hash{} accountKey := common.Hash{}
storageKey := common.HexToHash("0x1337") storageKey := common.HexToHash("0x1337")
accountRLP := randomAccount() accountRLP := randomAccount()
...@@ -278,16 +269,13 @@ func BenchmarkSearchSlot(b *testing.B) { ...@@ -278,16 +269,13 @@ func BenchmarkSearchSlot(b *testing.B) {
accStorage[randomHash()] = value accStorage[randomHash()] = value
storage[accountKey] = accStorage storage[accountKey] = accStorage
} }
blocknum++ return newDiffLayer(parent, common.Hash{}, accounts, storage)
return newDiffLayer(parent, blocknum, common.Hash{}, accounts, storage)
} }
var layer snapshot var layer snapshot
layer = emptyLayer{} layer = emptyLayer{}
for i := 0; i < 128; i++ { for i := 0; i < 128; i++ {
layer = fill(layer) layer = fill(layer)
} }
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
layer.Storage(accountKey, storageKey) layer.Storage(accountKey, storageKey)
...@@ -300,7 +288,7 @@ func BenchmarkSearchSlot(b *testing.B) { ...@@ -300,7 +288,7 @@ func BenchmarkSearchSlot(b *testing.B) {
// Without sorting and tracking accountlist // Without sorting and tracking accountlist
// BenchmarkFlatten-6 300 5511511 ns/op // BenchmarkFlatten-6 300 5511511 ns/op
func BenchmarkFlatten(b *testing.B) { func BenchmarkFlatten(b *testing.B) {
fill := func(parent snapshot, blocknum int) *diffLayer { fill := func(parent snapshot) *diffLayer {
accounts := make(map[common.Hash][]byte) accounts := make(map[common.Hash][]byte)
storage := make(map[common.Hash]map[common.Hash][]byte) storage := make(map[common.Hash]map[common.Hash][]byte)
...@@ -317,7 +305,7 @@ func BenchmarkFlatten(b *testing.B) { ...@@ -317,7 +305,7 @@ func BenchmarkFlatten(b *testing.B) {
} }
storage[accountKey] = accStorage storage[accountKey] = accStorage
} }
return newDiffLayer(parent, uint64(blocknum), common.Hash{}, accounts, storage) return newDiffLayer(parent, common.Hash{}, accounts, storage)
} }
b.ResetTimer() b.ResetTimer()
...@@ -327,7 +315,7 @@ func BenchmarkFlatten(b *testing.B) { ...@@ -327,7 +315,7 @@ func BenchmarkFlatten(b *testing.B) {
var layer snapshot var layer snapshot
layer = emptyLayer{} layer = emptyLayer{}
for i := 1; i < 128; i++ { for i := 1; i < 128; i++ {
layer = fill(layer, i) layer = fill(layer)
} }
b.StartTimer() b.StartTimer()
...@@ -336,7 +324,6 @@ func BenchmarkFlatten(b *testing.B) { ...@@ -336,7 +324,6 @@ func BenchmarkFlatten(b *testing.B) {
if !ok { if !ok {
break break
} }
layer = dl.flatten() layer = dl.flatten()
} }
b.StopTimer() b.StopTimer()
...@@ -351,7 +338,7 @@ func BenchmarkFlatten(b *testing.B) { ...@@ -351,7 +338,7 @@ func BenchmarkFlatten(b *testing.B) {
// BenchmarkJournal-6 1 1471373923 ns/ops // BenchmarkJournal-6 1 1471373923 ns/ops
// BenchmarkJournal-6 1 1208083335 ns/op // bufio writer // BenchmarkJournal-6 1 1208083335 ns/op // bufio writer
func BenchmarkJournal(b *testing.B) { func BenchmarkJournal(b *testing.B) {
fill := func(parent snapshot, blocknum int) *diffLayer { fill := func(parent snapshot) *diffLayer {
accounts := make(map[common.Hash][]byte) accounts := make(map[common.Hash][]byte)
storage := make(map[common.Hash]map[common.Hash][]byte) storage := make(map[common.Hash]map[common.Hash][]byte)
...@@ -368,15 +355,14 @@ func BenchmarkJournal(b *testing.B) { ...@@ -368,15 +355,14 @@ func BenchmarkJournal(b *testing.B) {
} }
storage[accountKey] = accStorage storage[accountKey] = accStorage
} }
return newDiffLayer(parent, uint64(blocknum), common.Hash{}, accounts, storage) return newDiffLayer(parent, common.Hash{}, accounts, storage)
} }
var layer snapshot var layer snapshot
layer = &diskLayer{ layer = &diskLayer{
journal: path.Join(os.TempDir(), "difflayer_journal.tmp"), journal: path.Join(os.TempDir(), "difflayer_journal.tmp"),
} }
for i := 1; i < 128; i++ { for i := 1; i < 128; i++ {
layer = fill(layer, i) layer = fill(layer)
} }
b.ResetTimer() b.ResetTimer()
......
...@@ -32,16 +32,24 @@ type diskLayer struct { ...@@ -32,16 +32,24 @@ type diskLayer struct {
db ethdb.KeyValueStore // Key-value store containing the base snapshot db ethdb.KeyValueStore // Key-value store containing the base snapshot
cache *bigcache.BigCache // Cache to avoid hitting the disk for direct access cache *bigcache.BigCache // Cache to avoid hitting the disk for direct access
number uint64 // Block number of the base snapshot root common.Hash // Root hash of the base snapshot
root common.Hash // Root hash of the base snapshot stale bool // Signals that the layer became stale (state progressed)
stale bool // Signals that the layer became stale (state progressed)
lock sync.RWMutex lock sync.RWMutex
} }
// Info returns the block number and root hash for which this snapshot was made. // Root returns root hash for which this snapshot was made.
func (dl *diskLayer) Info() (uint64, common.Hash) { func (dl *diskLayer) Root() common.Hash {
return dl.number, dl.root return dl.root
}
// Stale return whether this layer has become stale (was flattened across) or if
// it's still live.
func (dl *diskLayer) Stale() bool {
dl.lock.RLock()
defer dl.lock.RUnlock()
return dl.stale
} }
// Account directly retrieves the account associated with a particular hash in // Account directly retrieves the account associated with a particular hash in
...@@ -123,7 +131,7 @@ func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro ...@@ -123,7 +131,7 @@ func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro
// the specified data items. Note, the maps are retained by the method to avoid // the specified data items. Note, the maps are retained by the method to avoid
// copying everything. // copying everything.
func (dl *diskLayer) Update(blockHash common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { func (dl *diskLayer) Update(blockHash common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
return newDiffLayer(dl, dl.number+1, blockHash, accounts, storage) return newDiffLayer(dl, blockHash, accounts, storage)
} }
// Journal commits an entire diff hierarchy to disk into a single journal file. // Journal commits an entire diff hierarchy to disk into a single journal file.
......
...@@ -85,7 +85,7 @@ func wipeSnapshot(db ethdb.KeyValueStore) error { ...@@ -85,7 +85,7 @@ func wipeSnapshot(db ethdb.KeyValueStore) error {
} }
it.Release() it.Release()
rawdb.DeleteSnapshotBlock(batch) rawdb.DeleteSnapshotRoot(batch)
if err := batch.Write(); err != nil { if err := batch.Write(); err != nil {
return err return err
} }
...@@ -107,7 +107,7 @@ func wipeSnapshot(db ethdb.KeyValueStore) error { ...@@ -107,7 +107,7 @@ func wipeSnapshot(db ethdb.KeyValueStore) error {
} }
// generateSnapshot regenerates a brand new snapshot based on an existing state database and head block. // generateSnapshot regenerates a brand new snapshot based on an existing state database and head block.
func generateSnapshot(db ethdb.KeyValueStore, journal string, headNumber uint64, headRoot common.Hash) (snapshot, error) { func generateSnapshot(db ethdb.KeyValueStore, journal string, root common.Hash) (snapshot, error) {
// Wipe any previously existing snapshot from the database // Wipe any previously existing snapshot from the database
if err := wipeSnapshot(db); err != nil { if err := wipeSnapshot(db); err != nil {
return nil, err return nil, err
...@@ -124,7 +124,7 @@ func generateSnapshot(db ethdb.KeyValueStore, journal string, headNumber uint64, ...@@ -124,7 +124,7 @@ func generateSnapshot(db ethdb.KeyValueStore, journal string, headNumber uint64,
batch := db.NewBatch() batch := db.NewBatch()
triedb := trie.NewDatabase(db) triedb := trie.NewDatabase(db)
accTrie, err := trie.NewSecure(headRoot, triedb) accTrie, err := trie.NewSecure(root, triedb)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -186,7 +186,7 @@ func generateSnapshot(db ethdb.KeyValueStore, journal string, headNumber uint64, ...@@ -186,7 +186,7 @@ func generateSnapshot(db ethdb.KeyValueStore, journal string, headNumber uint64,
fmt.Printf("Totals: %9s (%d accs, %d nodes) + %9s (%d slots, %d nodes)\n", accountSize.TerminalString(), accountCount, accIt.Nodes, storageSize.TerminalString(), storageCount, storageNodes) fmt.Printf("Totals: %9s (%d accs, %d nodes) + %9s (%d slots, %d nodes)\n", accountSize.TerminalString(), accountCount, accIt.Nodes, storageSize.TerminalString(), storageCount, storageNodes)
// Update the snapshot block marker and write any remainder data // Update the snapshot block marker and write any remainder data
rawdb.WriteSnapshotBlock(batch, headNumber, headRoot) rawdb.WriteSnapshotRoot(batch, root)
batch.Write() batch.Write()
batch.Reset() batch.Reset()
...@@ -207,7 +207,6 @@ func generateSnapshot(db ethdb.KeyValueStore, journal string, headNumber uint64, ...@@ -207,7 +207,6 @@ func generateSnapshot(db ethdb.KeyValueStore, journal string, headNumber uint64,
journal: journal, journal: journal,
db: db, db: db,
cache: cache, cache: cache,
number: headNumber, root: root,
root: headRoot,
}, nil }, nil
} }
...@@ -47,7 +47,7 @@ func TestWipe(t *testing.T) { ...@@ -47,7 +47,7 @@ func TestWipe(t *testing.T) {
rawdb.WriteStorageSnapshot(db, account, randomHash(), randomHash().Bytes()) rawdb.WriteStorageSnapshot(db, account, randomHash(), randomHash().Bytes())
} }
} }
rawdb.WriteSnapshotBlock(db, 123, randomHash()) rawdb.WriteSnapshotRoot(db, randomHash())
// Add some random non-snapshot data too to make wiping harder // Add some random non-snapshot data too to make wiping harder
for i := 0; i < 65536; i++ { for i := 0; i < 65536; i++ {
...@@ -76,8 +76,8 @@ func TestWipe(t *testing.T) { ...@@ -76,8 +76,8 @@ func TestWipe(t *testing.T) {
if items != 128+128*1024 { if items != 128+128*1024 {
t.Fatalf("snapshot size mismatch: have %d, want %d", items, 128+128*1024) t.Fatalf("snapshot size mismatch: have %d, want %d", items, 128+128*1024)
} }
if number, hash := rawdb.ReadSnapshotBlock(db); number != 123 || hash == (common.Hash{}) { if hash := rawdb.ReadSnapshotRoot(db); hash == (common.Hash{}) {
t.Errorf("snapshot block marker mismatch: have #%d [%#x], want #%d [<not-nil>]", number, hash, 123) t.Errorf("snapshot block marker mismatch: have %#x, want <not-nil>", hash)
} }
// Wipe all snapshot entries from the database // Wipe all snapshot entries from the database
if err := wipeSnapshot(db); err != nil { if err := wipeSnapshot(db); err != nil {
...@@ -93,8 +93,8 @@ func TestWipe(t *testing.T) { ...@@ -93,8 +93,8 @@ func TestWipe(t *testing.T) {
t.Errorf("snapshot entry remained after wipe: %x", key) t.Errorf("snapshot entry remained after wipe: %x", key)
} }
} }
if number, hash := rawdb.ReadSnapshotBlock(db); number != 0 || hash != (common.Hash{}) { if hash := rawdb.ReadSnapshotRoot(db); hash != (common.Hash{}) {
t.Errorf("snapshot block marker remained after wipe: #%d [%#x]", number, hash) t.Errorf("snapshot block marker remained after wipe: %#x", hash)
} }
// Iterate over the database and ensure miscellaneous items are present // Iterate over the database and ensure miscellaneous items are present
items = 0 items = 0
......
This diff is collapsed.
...@@ -37,7 +37,7 @@ func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) { ...@@ -37,7 +37,7 @@ func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) {
root: common.HexToHash("0x01"), root: common.HexToHash("0x01"),
cache: cache, cache: cache,
} }
snaps := &SnapshotTree{ snaps := &Tree{
layers: map[common.Hash]snapshot{ layers: map[common.Hash]snapshot{
base.root: base, base.root: base,
}, },
...@@ -83,7 +83,7 @@ func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) { ...@@ -83,7 +83,7 @@ func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) {
root: common.HexToHash("0x01"), root: common.HexToHash("0x01"),
cache: cache, cache: cache,
} }
snaps := &SnapshotTree{ snaps := &Tree{
layers: map[common.Hash]snapshot{ layers: map[common.Hash]snapshot{
base.root: base, base.root: base,
}, },
...@@ -132,7 +132,7 @@ func TestDiffLayerExternalInvalidationFullFlatten(t *testing.T) { ...@@ -132,7 +132,7 @@ func TestDiffLayerExternalInvalidationFullFlatten(t *testing.T) {
root: common.HexToHash("0x01"), root: common.HexToHash("0x01"),
cache: cache, cache: cache,
} }
snaps := &SnapshotTree{ snaps := &Tree{
layers: map[common.Hash]snapshot{ layers: map[common.Hash]snapshot{
base.root: base, base.root: base,
}, },
...@@ -181,7 +181,7 @@ func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { ...@@ -181,7 +181,7 @@ func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) {
root: common.HexToHash("0x01"), root: common.HexToHash("0x01"),
cache: cache, cache: cache,
} }
snaps := &SnapshotTree{ snaps := &Tree{
layers: map[common.Hash]snapshot{ layers: map[common.Hash]snapshot{
base.root: base, base.root: base,
}, },
...@@ -213,7 +213,6 @@ func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { ...@@ -213,7 +213,6 @@ func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) {
if got := len(snaps.layers); got != exp { if got := len(snaps.layers); got != exp {
t.Errorf("layers modified, got %d exp %d", got, exp) t.Errorf("layers modified, got %d exp %d", got, exp)
} }
// Flatten the diff layer into the bottom accumulator // Flatten the diff layer into the bottom accumulator
if err := snaps.Cap(common.HexToHash("0x04"), 2, 1024*1024); err != nil { if err := snaps.Cap(common.HexToHash("0x04"), 2, 1024*1024); err != nil {
t.Fatalf("failed to flatten diff layer into accumulator: %v", err) t.Fatalf("failed to flatten diff layer into accumulator: %v", err)
...@@ -247,7 +246,7 @@ func TestPostCapBasicDataAccess(t *testing.T) { ...@@ -247,7 +246,7 @@ func TestPostCapBasicDataAccess(t *testing.T) {
root: common.HexToHash("0x01"), root: common.HexToHash("0x01"),
cache: cache, cache: cache,
} }
snaps := &SnapshotTree{ snaps := &Tree{
layers: map[common.Hash]snapshot{ layers: map[common.Hash]snapshot{
base.root: base, base.root: base,
}, },
......
...@@ -68,7 +68,7 @@ type StateDB struct { ...@@ -68,7 +68,7 @@ type StateDB struct {
db Database db Database
trie Trie trie Trie
snaps *snapshot.SnapshotTree snaps *snapshot.Tree
snap snapshot.Snapshot snap snapshot.Snapshot
snapAccounts map[common.Hash][]byte snapAccounts map[common.Hash][]byte
snapStorage map[common.Hash]map[common.Hash][]byte snapStorage map[common.Hash]map[common.Hash][]byte
...@@ -117,7 +117,7 @@ type StateDB struct { ...@@ -117,7 +117,7 @@ type StateDB struct {
} }
// Create a new state from a given trie. // Create a new state from a given trie.
func New(root common.Hash, db Database, snaps *snapshot.SnapshotTree) (*StateDB, error) { func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) {
tr, err := db.OpenTrie(root) tr, err := db.OpenTrie(root)
if err != nil { if err != nil {
return nil, err return nil, err
...@@ -840,12 +840,14 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { ...@@ -840,12 +840,14 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
if metrics.EnabledExpensive { if metrics.EnabledExpensive {
defer func(start time.Time) { s.SnapshotCommits += time.Since(start) }(time.Now()) defer func(start time.Time) { s.SnapshotCommits += time.Since(start) }(time.Now())
} }
_, parentRoot := s.snap.Info() // Only update if there's a state transition (skip empty Clique blocks)
if err := s.snaps.Update(root, parentRoot, s.snapAccounts, s.snapStorage); err != nil { if parent := s.snap.Root(); parent != root {
log.Warn("Failed to update snapshot tree", "from", parentRoot, "to", root, "err", err) if err := s.snaps.Update(root, parent, s.snapAccounts, s.snapStorage); err != nil {
} log.Warn("Failed to update snapshot tree", "from", parent, "to", root, "err", err)
if err := s.snaps.Cap(root, 16, 4*1024*1024); err != nil { }
log.Warn("Failed to cap snapshot tree", "root", root, "layers", 16, "memory", 4*1024*1024, "err", err) if err := s.snaps.Cap(root, 16, 4*1024*1024); err != nil {
log.Warn("Failed to cap snapshot tree", "root", root, "layers", 16, "memory", 4*1024*1024, "err", err)
}
} }
s.snap, s.snapAccounts, s.snapStorage = nil, nil, nil s.snap, s.snapAccounts, s.snapStorage = nil, nil, nil
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment