Skip to content

core/filtermaps: fix log index initialization #31750

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 3, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
core/filtermaps: safely revert reorged log index during init
  • Loading branch information
zsfelfoldi committed May 1, 2025
commit 18b3664bd1cdd347110fb9dfbbf5d0d12c1efddf
8 changes: 0 additions & 8 deletions core/filtermaps/chain_view.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,14 +135,6 @@ func (cv *ChainView) SharedRange(cv2 *ChainView) common.Range[uint64] {
return common.NewRange(0, sharedLen)
}

// limitedView returns a new chain view that is a truncated version of the parent view.
func (cv *ChainView) limitedView(newHead uint64) *ChainView {
if newHead >= cv.headNumber {
return cv
}
return NewChainView(cv.chain, newHead, cv.BlockHash(newHead))
}

// equalViews returns true if the two chain views are equivalent.
func equalViews(cv1, cv2 *ChainView) bool {
if cv1 == nil || cv2 == nil {
Expand Down
61 changes: 33 additions & 28 deletions core/filtermaps/filtermaps.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,8 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f
disabledCh: make(chan struct{}),
exportFileName: config.ExportFileName,
Params: params,
targetView: initView,
indexedView: initView,
indexedRange: filterMapsRange{
initialized: initialized,
headIndexed: rs.HeadIndexed,
Expand All @@ -265,16 +267,8 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f
baseRowsCache: lru.NewCache[uint64, [][]uint32](cachedBaseRows),
renderSnapshots: lru.NewCache[uint64, *renderedMap](cachedRenderSnapshots),
}
f.checkRevertRange() // revert maps that are inconsistent with the current chain view

// Set initial indexer target.
f.targetView = initView
if f.indexedRange.initialized {
f.indexedView = f.initChainView(f.targetView)
f.indexedRange.headIndexed = f.indexedRange.blocks.AfterLast() == f.indexedView.HeadNumber()+1
if !f.indexedRange.headIndexed {
f.indexedRange.headDelimiter = 0
}
}
if f.indexedRange.hasIndexedBlocks() {
log.Info("Initialized log indexer",
"first block", f.indexedRange.blocks.First(), "last block", f.indexedRange.blocks.Last(),
Expand Down Expand Up @@ -303,29 +297,40 @@ func (f *FilterMaps) Stop() {
f.closeWg.Wait()
}

// initChainView returns a chain view consistent with both the current target
// view and the current state of the log index as found in the database, based
// on the last block of stored maps.
// Note that the returned view might be shorter than the existing index if
// the latest maps are not consistent with targetView.
func (f *FilterMaps) initChainView(chainView *ChainView) *ChainView {
mapIndex := f.indexedRange.maps.AfterLast()
for {
var ok bool
mapIndex, ok = f.lastMapBoundaryBefore(mapIndex)
if !ok {
break
// checkRevertRange checks whether the existing index is consistent with the
// current indexed view and reverts inconsistent maps if necessary.
func (f *FilterMaps) checkRevertRange() {
if f.indexedRange.maps.Count() == 0 {
return
}
lastMap := f.indexedRange.maps.Last()
lastBlockNumber, lastBlockId, err := f.getLastBlockOfMap(lastMap)
if err != nil {
log.Error("Error initializing log index database; resetting log index", "error", err)
f.reset()
return
}
for lastBlockNumber > f.indexedView.HeadNumber() || f.indexedView.BlockId(lastBlockNumber) != lastBlockId {
// revert last map
if f.indexedRange.maps.Count() == 1 {
f.reset() // reset database if no rendered maps remained
return
}
lastBlockNumber, lastBlockId, err := f.getLastBlockOfMap(mapIndex)
lastMap--
newRange := f.indexedRange
newRange.maps.SetLast(lastMap)
lastBlockNumber, lastBlockId, err = f.getLastBlockOfMap(lastMap)
if err != nil {
log.Error("Could not initialize indexed chain view", "error", err)
break
}
if lastBlockNumber <= chainView.HeadNumber() && chainView.BlockId(lastBlockNumber) == lastBlockId {
return chainView.limitedView(lastBlockNumber)
log.Error("Error initializing log index database; resetting log index", "error", err)
f.reset()
return
}
newRange.blocks.SetAfterLast(lastBlockNumber) // lastBlockNumber is probably partially indexed
Copy link
Member

@rjl493456442 rjl493456442 May 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This status is very weird.

  • Let's say the current indexed map is [0, 100] and headIndexed = true; the last block of map 100 is 10000;
  • However, the block 10000 beyonds the new indexedView (reorg);
  • The map 100 is reverted;
  • The indexedRange is converted to [0, 99];
  • The last block of map 99 is 9900;
  • The block 9900 is partially indexed in map 99 (it crosses the map 99 and 100);
  • The indexedRange is converted to map: [0, 99], blocks [0, 9899], headIndexed = false;

This status is very weird; the if the map is full, it's usually written with headIndexed = false; If the map is not full but all blocks have been indexed, then the headIndexed = true;

The initialized status of indexedRange is actually an invalid one (the last block is not the real one corresponds to the map)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

EDIT, never mind.

It's correct.

newRange.headIndexed = false
newRange.headDelimiter = 0
// only shorten range and leave map data; next head render will overwrite it
f.setRange(f.db, f.indexedView, newRange, false)
}
return chainView.limitedView(0)
}

// reset un-initializes the FilterMaps structure and removes all related data from
Expand Down
2 changes: 1 addition & 1 deletion core/filtermaps/map_renderer.go
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,7 @@ func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error {
// in order to always ensure continuous block pointers, initialize
// blockNumber based on the last block of the previous map, then verify
// against the first block associated with each rendered map
lastBlock, _, err := r.f.getLastBlockOfMap(r.finished.First()-1)
lastBlock, _, err := r.f.getLastBlockOfMap(r.finished.First() - 1)
if err != nil {
return fmt.Errorf("failed to get last block of previous map %d: %v", r.finished.First()-1, err)
}
Expand Down