diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 7d9c2770f0a..c5a12089071 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1215,57 +1215,32 @@ func (d *Domain) buildAccessor(ctx context.Context, fromStep, toStep uint64, dat } func (d *Domain) missedBtreeAccessors() (l []*filesItem) { - d.dirtyFiles.Walk(func(items []*filesItem) bool { // don't run slow logic while iterating on btree - for _, item := range items { - fromStep, toStep := item.startTxNum/d.aggregationStep, item.endTxNum/d.aggregationStep - fPath := d.kvBtFilePath(fromStep, toStep) - exists, err := dir.FileExist(fPath) - if err != nil { - panic(err) - } - if !exists { - l = append(l, item) - continue - } - fPath = d.kvExistenceIdxFilePath(fromStep, toStep) - exists, err = dir.FileExist(fPath) - if err != nil { - panic(err) - } - if !exists { - l = append(l, item) - continue - } + return fileItemsWithMissingAccessors(d.dirtyFiles, d.aggregationStep, func(fromStep uint64, toStep uint64) []string { + var accessors []string + if d.IndexList.Has(AccessorBTree) { + accessors = append(accessors, d.kvBtFilePath(fromStep, toStep)) } - return true + if d.IndexList.Has(AccessorExistence) { + accessors = append(accessors, d.kvExistenceIdxFilePath(fromStep, toStep)) + } + + return accessors }) - return l } + func (d *Domain) missedAccessors() (l []*filesItem) { - d.dirtyFiles.Walk(func(items []*filesItem) bool { // don't run slow logic while iterating on btree - for _, item := range items { - fromStep, toStep := item.startTxNum/d.aggregationStep, item.endTxNum/d.aggregationStep - fPath := d.kvAccessorFilePath(fromStep, toStep) - exists, err := dir.FileExist(fPath) - if err != nil { - panic(err) - } - if !exists { - l = append(l, item) - } - } - return true + if !d.IndexList.Has(AccessorHashMap) { + return nil + } + return fileItemsWithMissingAccessors(d.dirtyFiles, d.aggregationStep, func(fromStep uint64, toStep uint64) []string { + return []string{d.kvAccessorFilePath(fromStep, toStep)} }) - return l } // BuildMissedAccessors - produce .efi/.vi/.kvi from .ef/.v/.kv func (d *Domain) BuildMissedAccessors(ctx context.Context, g *errgroup.Group, ps *background.ProgressSet) { d.History.BuildMissedAccessors(ctx, g, ps) for _, item := range d.missedBtreeAccessors() { - if d.IndexList&AccessorBTree == 0 { - continue - } if item.decompressor == nil { log.Warn(fmt.Sprintf("[dbg] BuildMissedAccessors: item with nil decompressor %s %d-%d", d.filenameBase, item.startTxNum/d.aggregationStep, item.endTxNum/d.aggregationStep)) } @@ -1281,9 +1256,6 @@ func (d *Domain) BuildMissedAccessors(ctx context.Context, g *errgroup.Group, ps }) } for _, item := range d.missedAccessors() { - if d.IndexList&AccessorHashMap == 0 { - continue - } if item.decompressor == nil { log.Warn(fmt.Sprintf("[dbg] BuildMissedAccessors: item with nil decompressor %s %d-%d", d.filenameBase, item.startTxNum/d.aggregationStep, item.endTxNum/d.aggregationStep)) } diff --git a/erigon-lib/state/files_item.go b/erigon-lib/state/files_item.go index c782fb295be..43282f459e0 100644 --- a/erigon-lib/state/files_item.go +++ b/erigon-lib/state/files_item.go @@ -26,6 +26,7 @@ import ( btree2 "github.com/tidwall/btree" + "github.com/erigontech/erigon-lib/common/dir" "github.com/erigontech/erigon-lib/config3" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/recsplit" @@ -354,3 +355,23 @@ func (files visibleFiles) MergedRanges() []MergeRange { } return res } + +func fileItemsWithMissingAccessors(dirtyFiles *btree2.BTreeG[*filesItem], aggregationStep uint64, accessorsFor func(fromStep, toStep uint64) []string) (l []*filesItem) { + dirtyFiles.Walk(func(items []*filesItem) bool { + for _, item := range items { + fromStep, toStep := item.startTxNum/aggregationStep, item.endTxNum/aggregationStep + for _, fName := range accessorsFor(fromStep, toStep) { + exists, err := dir.FileExist(fName) + if err != nil { + panic(err) + } + if !exists { + l = append(l, item) + break + } + } + } + return true + }) + return +} diff --git a/erigon-lib/state/files_test.go b/erigon-lib/state/files_test.go new file mode 100644 index 00000000000..a63962576d2 --- /dev/null +++ b/erigon-lib/state/files_test.go @@ -0,0 +1,57 @@ +package state + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + btree2 "github.com/tidwall/btree" +) + +func TestFileItemWithMissingAccessor(t *testing.T) { + tmp := t.TempDir() + + // filesItem + f1 := &filesItem{ + startTxNum: 1, + endTxNum: 10, + } + f2 := &filesItem{ + startTxNum: 11, + endTxNum: 20, + } + f3 := &filesItem{ + startTxNum: 31, + endTxNum: 40, + } + aggStep := uint64(10) + + btree := btree2.NewBTreeGOptions(filesItemLess, btree2.Options{Degree: 128, NoLocks: false}) + btree.Set(f1) + btree.Set(f2) + btree.Set(f3) + + accessorFor := func(fromStep, toStep uint64) []string { + return []string{ + filepath.Join(tmp, fmt.Sprintf("testacc_%d_%d.bin", fromStep, toStep)), + filepath.Join(tmp, fmt.Sprintf("testacc2_%d_%d.bin", fromStep, toStep)), + } + } + + // create accesssor files for f1, f2 + for _, fname := range accessorFor(f1.startTxNum/aggStep, f1.endTxNum/aggStep) { + os.WriteFile(fname, []byte("test"), 0644) + defer os.Remove(fname) + } + + for _, fname := range accessorFor(f2.startTxNum/aggStep, f2.endTxNum/aggStep) { + os.WriteFile(fname, []byte("test"), 0644) + defer os.Remove(fname) + } + + fileItems := fileItemsWithMissingAccessors(btree, aggStep, accessorFor) + require.Equal(t, 1, len(fileItems)) + require.Equal(t, f3, fileItems[0]) +} diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index c6832962ce5..e99538870a1 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -315,21 +315,14 @@ func (ht *HistoryRoTx) Files() (res []string) { } func (h *History) missedAccessors() (l []*filesItem) { - h.dirtyFiles.Walk(func(items []*filesItem) bool { // don't run slow logic while iterating on btree - for _, item := range items { - fromStep, toStep := item.startTxNum/h.aggregationStep, item.endTxNum/h.aggregationStep - exists, err := dir.FileExist(h.vAccessorFilePath(fromStep, toStep)) - if err != nil { - _, fName := filepath.Split(h.vAccessorFilePath(fromStep, toStep)) - h.logger.Warn("[agg] History.missedAccessors", "err", err, "f", fName) - } - if !exists { - l = append(l, item) - } + if !h.indexList.Has(AccessorHashMap) { + return nil + } + return fileItemsWithMissingAccessors(h.dirtyFiles, h.aggregationStep, func(fromStep, toStep uint64) []string { + return []string{ + h.vAccessorFilePath(fromStep, toStep), } - return true }) - return l } func (h *History) buildVi(ctx context.Context, item *filesItem, ps *background.ProgressSet) (err error) { @@ -438,8 +431,7 @@ func (h *History) buildVI(ctx context.Context, historyIdxPath string, hist, efHi func (h *History) BuildMissedAccessors(ctx context.Context, g *errgroup.Group, ps *background.ProgressSet) { h.InvertedIndex.BuildMissedAccessors(ctx, g, ps) - missedFiles := h.missedAccessors() - for _, item := range missedFiles { + for _, item := range h.missedAccessors() { item := item g.Go(func() error { return h.buildVi(ctx, item, ps) diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index 3f1c1345662..1a951bb3202 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -215,21 +215,14 @@ func (ii *InvertedIndex) reCalcVisibleFiles(toTxNum uint64) { } func (ii *InvertedIndex) missedAccessors() (l []*filesItem) { - ii.dirtyFiles.Walk(func(items []*filesItem) bool { - for _, item := range items { - fromStep, toStep := item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep - exists, err := dir.FileExist(ii.efAccessorFilePath(fromStep, toStep)) - if err != nil { - _, fName := filepath.Split(ii.efAccessorFilePath(fromStep, toStep)) - ii.logger.Warn("[agg] InvertedIndex missedAccessors", "err", err, "f", fName) - } - if !exists { - l = append(l, item) - } + if !ii.indexList.Has(AccessorHashMap) { + return nil + } + return fileItemsWithMissingAccessors(ii.dirtyFiles, ii.aggregationStep, func(fromStep, toStep uint64) []string { + return []string{ + ii.efAccessorFilePath(fromStep, toStep), } - return true }) - return l } func (ii *InvertedIndex) buildEfAccessor(ctx context.Context, item *filesItem, ps *background.ProgressSet) (err error) {