Implement beacon db pruner (#14687)

* implement weak subjectivity pruner

* fix goimports

* add delete before slot method to database

* add method to interface

* update changelog

* add flags

* wire pruner

* align pruner with backfill service

* rename db method

* fix imports

* delete block slot indices

* check for backfill and initial sync

* add tests

* fix imports

* Update beacon-chain/db/kv/blocks.go

Co-authored-by: Radosław Kapka <[email protected]>

* Update cmd/beacon-chain/flags/base.go

Co-authored-by: Preston Van Loon <[email protected]>

* Update beacon-chain/db/pruner/pruner.go

Co-authored-by: Radosław Kapka <[email protected]>

* cleanup

* fix buildkite

* initialise atomic bool

* delete data from remaining buckets

* fix build

* fix build

* address review comments

* add test for blockParentRootIndicesBucket

* fix changelog

* fix build

* address kasey's comments

* fix build

* add trace span to blockRootsBySlotRange

---------

Co-authored-by: Radosław Kapka <[email protected]>
Co-authored-by: Preston Van Loon <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -60,7 +60,7 @@ Notable features:
 - Updated the default `scrape-interval` in `Client-stats` to 2 minutes to accommodate Beaconcha.in API rate limits.
 - Switch to compounding when consolidating with source==target.
 - Revert block db save when saving state fails.
-- Return false from HasBlock if the block is being synced. 
+- Return false from HasBlock if the block is being synced.
 - Cleanup forkchoice on failed insertions.
 - Use read only validator for core processing to avoid unnecessary copying.
 - Use ROBlock across block processing pipeline.
@@ -73,7 +73,7 @@ Notable features:
 - Simplified `EjectedValidatorIndices`.
 - `engine_newPayloadV4`,`engine_getPayloadV4` are changes due to new execution request serialization decisions, [PR](https://github.com/prysmaticlabs/prysm/pull/14580)
 - Fixed various small things in state-native code.
-- Use ROBlock earlier in block syncing pipeline. 
+- Use ROBlock earlier in block syncing pipeline.
 - Changed the signature of `ProcessPayload`.
 - Only Build the Protobuf state once during serialization.
 - Capella blocks are execution.
@@ -139,9 +139,9 @@ Notable features:
 
 ### Security
 
-## [v5.1.2](https://github.com/prysmaticlabs/prysm/compare/v5.1.1...v5.1.2) - 2024-10-16 
+## [v5.1.2](https://github.com/prysmaticlabs/prysm/compare/v5.1.1...v5.1.2) - 2024-10-16
 
-This is a hotfix release with one change. 
+This is a hotfix release with one change.
 
 Prysm v5.1.1 contains an updated implementation of the beacon api streaming events endpoint. This
 new implementation contains a bug that can cause a panic in certain conditions. The issue is
@@ -153,20 +153,20 @@ prysm REST mode validator (a feature which requires the validator to be configur
 api instead of prysm's stock grpc endpoints) or accessory software that connects to the events api,
 like https://github.com/ethpandaops/ethereum-metrics-exporter
 
-### Fixed 
+### Fixed
 
 - Recover from panics when writing the event stream [#14545](https://github.com/prysmaticlabs/prysm/pull/14545)
 
 ## [v5.1.1](https://github.com/prysmaticlabs/prysm/compare/v5.1.0...v5.1.1) - 2024-10-15
 
-This release has a number of features and improvements. Most notably, the feature flag 
-`--enable-experimental-state` has been flipped to "opt out" via `--disable-experimental-state`. 
+This release has a number of features and improvements. Most notably, the feature flag
+`--enable-experimental-state` has been flipped to "opt out" via `--disable-experimental-state`.
 The experimental state management design has shown significant improvements in memory usage at
 runtime. Updates to libp2p's gossipsub have some bandwidith stability improvements with support for
-IDONTWANT control messages. 
+IDONTWANT control messages.
 
 The gRPC gateway has been deprecated from Prysm in this release. If you need JSON data, consider the
-standardized beacon-APIs. 
+standardized beacon-APIs.
 
 Updating to this release is recommended at your convenience.
 
@@ -208,7 +208,7 @@ Updating to this release is recommended at your convenience.
 - `grpc-gateway-corsdomain` is renamed to http-cors-domain. The old name can still be used as an alias.
 - `api-timeout` is changed from int flag to duration flag, default value updated.
 - Light client support: abstracted out the light client headers with different versions.
-- `ApplyToEveryValidator` has been changed to prevent misuse bugs, it takes a closure that takes a `ReadOnlyValidator` and returns a raw pointer to a `Validator`. 
+- `ApplyToEveryValidator` has been changed to prevent misuse bugs, it takes a closure that takes a `ReadOnlyValidator` and returns a raw pointer to a `Validator`.
 - Removed gorilla mux library and replaced it with net/http updates in go 1.22.
 - Clean up `ProposeBlock` for validator client to reduce cognitive scoring and enable further changes.
 - Updated k8s-io/client-go to v0.30.4 and k8s-io/apimachinery to v0.30.4
@@ -219,7 +219,7 @@ Updating to this release is recommended at your convenience.
 - Updated Sepolia bootnodes.
 - Make committee aware packing the default by deprecating `--enable-committee-aware-packing`.
 - Moved `ConvertKzgCommitmentToVersionedHash` to the `primitives` package.
-- Updated correlation penalty for EIP-7251. 
+- Updated correlation penalty for EIP-7251.
 
 ### Deprecated
 - `--disable-grpc-gateway` flag is deprecated due to grpc gateway removal.
@@ -693,34 +693,34 @@ AVX support (eg Celeron) after the Deneb fork. This is not an issue for mainnet.
 
 - Linter: Wastedassign linter enabled to improve code quality.
 - API Enhancements:
-    - Added payload return in Wei for /eth/v3/validator/blocks.
-    - Added Holesky Deneb Epoch for better epoch management.
+  - Added payload return in Wei for /eth/v3/validator/blocks.
+  - Added Holesky Deneb Epoch for better epoch management.
 - Testing Enhancements:
-    - Clear cache in tests of core helpers to ensure test reliability.
-    - Added Debug State Transition Method for improved debugging.
-    - Backfilling test: Enabled backfill in E2E tests for more comprehensive coverage.
+  - Clear cache in tests of core helpers to ensure test reliability.
+  - Added Debug State Transition Method for improved debugging.
+  - Backfilling test: Enabled backfill in E2E tests for more comprehensive coverage.
 - API Updates: Re-enabled jwt on keymanager API for enhanced security.
 - Logging Improvements: Enhanced block by root log for better traceability.
 - Validator Client Improvements:
-    - Added Spans to Core Validator Methods for enhanced monitoring.
-    - Improved readability in validator client code for better maintenance (various commits).
+  - Added Spans to Core Validator Methods for enhanced monitoring.
+  - Improved readability in validator client code for better maintenance (various commits).
 
 ### Changed
 
 - Optimizations and Refinements:
-    - Lowered resource usage in certain processes for efficiency.
-    - Moved blob rpc validation closer to peer read for optimized processing.
-    - Cleaned up validate beacon block code for clarity and efficiency.
-    - Updated Sepolia Deneb fork epoch for alignment with network changes.
-    - Changed blob latency metrics to milliseconds for more precise measurement.
-    - Altered getLegacyDatabaseLocation message for better clarity.
-    - Improved wait for activation method for enhanced performance.
-    - Capitalized Aggregated Unaggregated Attestations Log for consistency.
-    - Modified HistoricalRoots usage for accuracy.
-    - Adjusted checking of attribute emptiness for efficiency.
+  - Lowered resource usage in certain processes for efficiency.
+  - Moved blob rpc validation closer to peer read for optimized processing.
+  - Cleaned up validate beacon block code for clarity and efficiency.
+  - Updated Sepolia Deneb fork epoch for alignment with network changes.
+  - Changed blob latency metrics to milliseconds for more precise measurement.
+  - Altered getLegacyDatabaseLocation message for better clarity.
+  - Improved wait for activation method for enhanced performance.
+  - Capitalized Aggregated Unaggregated Attestations Log for consistency.
+  - Modified HistoricalRoots usage for accuracy.
+  - Adjusted checking of attribute emptiness for efficiency.
 - Database Management:
-    - Moved --db-backup-output-dir as a deprecated flag for database management simplification.
-    - Added the Ability to Defragment the Beacon State for improved database performance.
+  - Moved --db-backup-output-dir as a deprecated flag for database management simplification.
+  - Added the Ability to Defragment the Beacon State for improved database performance.
 - Dependency Update: Bumped quic-go version from 0.39.3 to 0.39.4 for up-to-date dependencies.
 
 ### Removed
@@ -731,12 +731,12 @@ AVX support (eg Celeron) after the Deneb fork. This is not an issue for mainnet.
 ### Fixed
 
 - Bug Fixes:
-    - Fixed off by one error for improved accuracy.
-    - Resolved small typo in error messages for clarity.
-    - Addressed minor issue in blsToExecChange validator for better validation.
-    - Corrected blobsidecar json tag for commitment inclusion proof.
-    - Fixed ssz post-requests content type check.
-    - Resolved issue with port logging in bootnode.
+  - Fixed off by one error for improved accuracy.
+  - Resolved small typo in error messages for clarity.
+  - Addressed minor issue in blsToExecChange validator for better validation.
+  - Corrected blobsidecar json tag for commitment inclusion proof.
+  - Fixed ssz post-requests content type check.
+  - Resolved issue with port logging in bootnode.
 - Test Fixes: Re-enabled Slasher E2E Test for more comprehensive testing.
 
 ### Security
@@ -1163,9 +1163,9 @@ No security issues in this release.
   now features runtime detection, automatically enabling optimized code paths if your CPU supports it.
 - **Multiarch Containers Preview Available**: multiarch (:wave: arm64 support :wave:) containers will be offered for
   preview at the following locations:
-    - Beacon Chain: [gcr.io/prylabs-dev/prysm/beacon-chain:v4.1.0](gcr.io/prylabs-dev/prysm/beacon-chain:v4.1.0)
-    - Validator: [gcr.io/prylabs-dev/prysm/validator:v4.1.0](gcr.io/prylabs-dev/prysm/validator:v4.1.0)
-    - Please note that in the next cycle, we will exclusively use these containers at the canonical URLs.
+  - Beacon Chain: [gcr.io/prylabs-dev/prysm/beacon-chain:v4.1.0](gcr.io/prylabs-dev/prysm/beacon-chain:v4.1.0)
+  - Validator: [gcr.io/prylabs-dev/prysm/validator:v4.1.0](gcr.io/prylabs-dev/prysm/validator:v4.1.0)
+  - Please note that in the next cycle, we will exclusively use these containers at the canonical URLs.
 
 ### Added
 
@@ -2987,4 +2987,4 @@ There are no security updates in this release.
 
 # Older than v2.0.0
 
-For changelog history for releases older than v2.0.0, please refer to https://github.com/prysmaticlabs/prysm/releases
+For changelog history for releases older than v2.0.0, please refer to https://github.com/prysmaticlabs/prysm/releases

beacon-chain/db/iface/interface.go

@@ -101,6 +101,7 @@ type NoHeadAccessDatabase interface {
 	SaveLightClientBootstrap(ctx context.Context, blockRoot []byte, bootstrap interfaces.LightClientBootstrap) error
 
 	CleanUpDirtyStates(ctx context.Context, slotsPerArchivedPoint primitives.Slot) error
+	DeleteHistoricalDataBeforeSlot(ctx context.Context, slot primitives.Slot) error
 }
 
 // HeadAccessDatabase defines a struct with access to reading chain head data.

beacon-chain/db/kv/blocks.go

@@ -227,17 +227,97 @@ func (s *Store) DeleteBlock(ctx context.Context, root [32]byte) error {
 			return ErrDeleteJustifiedAndFinalized
 		}
 
-		if err := tx.Bucket(blocksBucket).Delete(root[:]); err != nil {
-			return err
-		}
-		if err := tx.Bucket(blockParentRootIndicesBucket).Delete(root[:]); err != nil {
+		if err := s.deleteBlock(tx, root[:]); err != nil {
 			return err
 		}
 		s.blockCache.Del(string(root[:]))
 		return nil
 	})
 }
 
+// DeleteHistoricalDataBeforeSlot deletes all blocks and states before the given slot.
+// This function deletes data from the following buckets:
+// - blocksBucket
+// - blockParentRootIndicesBucket
+// - finalizedBlockRootsIndexBucket
+// - stateBucket
+// - stateSummaryBucket
+// - blockRootValidatorHashesBucket
+// - blockSlotIndicesBucket
+// - stateSlotIndicesBucket
+func (s *Store) DeleteHistoricalDataBeforeSlot(ctx context.Context, cutoffSlot primitives.Slot) error {
+	ctx, span := trace.StartSpan(ctx, "BeaconDB.DeleteHistoricalDataBeforeSlot")
+	defer span.End()
+
+	// Collect slot/root pairs to perform deletions in a separate read only transaction.
+	var (
+		roots [][]byte
+		slts  []primitives.Slot
+	)
+	err := s.db.View(func(tx *bolt.Tx) error {
+		var err error
+		roots, slts, err = blockRootsBySlotRange(ctx, tx.Bucket(blockSlotIndicesBucket), primitives.Slot(0), cutoffSlot, nil, nil, nil)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve block roots")
+		}
+		return nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "could not retrieve block roots and slots")
+	}
+
+	// Perform all deletions in a single transaction for atomicity
+	return s.db.Update(func(tx *bolt.Tx) error {
+		for _, root := range roots {
+			// Delete block
+			if err = s.deleteBlock(tx, root); err != nil {
+				return err
+			}
+
+			// Delete finalized block roots index
+			if err = tx.Bucket(finalizedBlockRootsIndexBucket).Delete(root); err != nil {
+				return errors.Wrap(err, "could not delete finalized block root index")
+			}
+
+			// Delete state
+			if err = tx.Bucket(stateBucket).Delete(root); err != nil {
+				return errors.Wrap(err, "could not delete state")
+			}
+
+			// Delete state summary
+			if err = tx.Bucket(stateSummaryBucket).Delete(root); err != nil {
+				return errors.Wrap(err, "could not delete state summary")
+			}
+
+			// Delete validator entries
+			if err = s.deleteValidatorHashes(tx, root); err != nil {
+				return errors.Wrap(err, "could not delete validators")
+			}
+		}
+
+		for _, slot := range slts {
+			// Delete slot indices
+			if err = tx.Bucket(blockSlotIndicesBucket).Delete(bytesutil.SlotToBytesBigEndian(slot)); err != nil {
+				return errors.Wrap(err, "could not delete block slot index")
+			}
+			if err = tx.Bucket(stateSlotIndicesBucket).Delete(bytesutil.SlotToBytesBigEndian(slot)); err != nil {
+				return errors.Wrap(err, "could not delete state slot index")
+			}
+		}
+
+		// Delete all caches after we have deleted everything from buckets.
+		// This is done after the buckets are deleted to avoid any issues in case of transaction rollback.
+		for _, root := range roots {
+			// Delete block from cache
+			s.blockCache.Del(string(root))
+			// Delete state summary from cache
+			s.stateSummaryCache.delete([32]byte(root))
+		}
+
+		return nil
+	})
+}
+
 // SaveBlock to the db.
 func (s *Store) SaveBlock(ctx context.Context, signed interfaces.ReadOnlySignedBeaconBlock) error {
 	ctx, span := trace.StartSpan(ctx, "BeaconDB.SaveBlock")
@@ -609,7 +689,7 @@ func blockRootsByFilter(ctx context.Context, tx *bolt.Tx, f *filters.QueryFilter
 
 	// We retrieve block roots that match a filter criteria of slot ranges, if specified.
 	filtersMap := f.Filters()
-	rootsBySlotRange, err := blockRootsBySlotRange(
+	rootsBySlotRange, _, err := blockRootsBySlotRange(
 		ctx,
 		tx.Bucket(blockSlotIndicesBucket),
 		filtersMap[filters.StartSlot],
@@ -627,6 +707,7 @@ func blockRootsByFilter(ctx context.Context, tx *bolt.Tx, f *filters.QueryFilter
 	// that list of roots to lookup the block. These block will
 	// meet the filter criteria.
 	indices := lookupValuesForIndices(ctx, indicesByBucket, tx)
+
 	keys := rootsBySlotRange
 	if len(indices) > 0 {
 		// If we have found indices that meet the filter criteria, and there are also
@@ -653,13 +734,13 @@ func blockRootsBySlotRange(
 	ctx context.Context,
 	bkt *bolt.Bucket,
 	startSlotEncoded, endSlotEncoded, startEpochEncoded, endEpochEncoded, slotStepEncoded interface{},
-) ([][]byte, error) {
+) ([][]byte, []primitives.Slot, error) {
 	_, span := trace.StartSpan(ctx, "BeaconDB.blockRootsBySlotRange")
 	defer span.End()
 
 	// Return nothing when all slot parameters are missing
 	if startSlotEncoded == nil && endSlotEncoded == nil && startEpochEncoded == nil && endEpochEncoded == nil {
-		return [][]byte{}, nil
+		return [][]byte{}, nil, nil
 	}
 
 	var startSlot, endSlot primitives.Slot
@@ -680,11 +761,11 @@ func blockRootsBySlotRange(
 	if startEpochOk && endEpochOk {
 		startSlot, err = slots.EpochStart(startEpoch)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		endSlot, err = slots.EpochStart(endEpoch)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		endSlot = endSlot + params.BeaconConfig().SlotsPerEpoch - 1
 	}
@@ -695,14 +776,15 @@ func blockRootsBySlotRange(
 		return key != nil && bytes.Compare(key, max) <= 0
 	}
 	if endSlot < startSlot {
-		return nil, errInvalidSlotRange
+		return nil, nil, errInvalidSlotRange
 	}
 	rootsRange := endSlot.SubSlot(startSlot).Div(step)
 	roots := make([][]byte, 0, rootsRange)
+	var slts []primitives.Slot
 	c := bkt.Cursor()
 	for k, v := c.Seek(min); conditional(k, max); k, v = c.Next() {
+		slot := bytesutil.BytesToSlotBigEndian(k)
 		if step > 1 {
-			slot := bytesutil.BytesToSlotBigEndian(k)
 			if slot.SubSlot(startSlot).Mod(step) != 0 {
 				continue
 			}
@@ -713,8 +795,9 @@ func blockRootsBySlotRange(
 			splitRoots = append(splitRoots, v[i:i+32])
 		}
 		roots = append(roots, splitRoots...)
+		slts = append(slts, slot)
 	}
-	return roots, nil
+	return roots, slts, nil
 }
 
 // blockRootsBySlot retrieves the block roots by slot
@@ -908,3 +991,32 @@ func keyForBlock(blk interfaces.ReadOnlySignedBeaconBlock) ([]byte, error) {
 
 	return nil, fmt.Errorf("unsupported block version: %v", blk.Version())
 }
+
+func (s *Store) deleteBlock(tx *bolt.Tx, root []byte) error {
+	if err := tx.Bucket(blocksBucket).Delete(root); err != nil {
+		return errors.Wrap(err, "could not delete block")
+	}
+
+	if err := tx.Bucket(blockParentRootIndicesBucket).Delete(root); err != nil {
+		return errors.Wrap(err, "could not delete block parent indices")
+	}
+
+	return nil
+}
+
+func (s *Store) deleteValidatorHashes(tx *bolt.Tx, root []byte) error {
+	ok, err := s.isStateValidatorMigrationOver()
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return nil
+	}
+
+	// Delete the validator hash index
+	if err = tx.Bucket(blockRootValidatorHashesBucket).Delete(root); err != nil {
+		return errors.Wrap(err, "could not delete validator index")
+	}
+
+	return nil
+}