diff --git a/app/seidb.go b/app/seidb.go index 78eab0a06e..be6f550365 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -9,6 +9,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-cosmos/baseapp" servertypes "github.com/sei-protocol/sei-chain/sei-cosmos/server/types" "github.com/sei-protocol/sei-chain/sei-cosmos/storev2/rootmulti" + "github.com/sei-protocol/sei-chain/sei-cosmos/version" "github.com/sei-protocol/sei-chain/sei-db/config" seidb "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" sctypes "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" @@ -39,6 +40,13 @@ const ( FlagSCKeysToMigratePerBlock = "state-commit.sc-keys-to-migrate-per-block" FlagSCFlatKVReadWriteMetrics = "state-commit.flatkv.enable-read-write-metrics" + // Hash logger configs (per-block hash logging; enabled by default) + FlagSCHashLoggerEnable = "state-commit.sc-hash-logger-enable" + FlagSCHashLoggerDirectory = "state-commit.sc-hash-logger-directory" + FlagSCHashLoggerBlocksToRetain = "state-commit.sc-hash-logger-blocks-to-retain" + FlagSCHashLoggerTargetFileSize = "state-commit.sc-hash-logger-target-file-size" + FlagSCHashLoggerMaxDiskSize = "state-commit.sc-hash-logger-max-disk-size" + // SS Store configs FlagSSEnable = "state-store.ss-enable" FlagSSDirectory = "state-store.ss-db-directory" @@ -142,6 +150,31 @@ func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig { } } + // Hash logger. Guard each read with v != nil so an absent app.toml entry preserves the default + // (notably Enable, which defaults to true) instead of clobbering it to the zero value. + if v := appOpts.Get(FlagSCHashLoggerEnable); v != nil { + scConfig.HashLogger.Enable = cast.ToBool(v) + } + if v := appOpts.Get(FlagSCHashLoggerDirectory); v != nil { + scConfig.HashLogger.Directory = cast.ToString(v) + } + // BlocksToRetain and MaxDiskSize take a configured value verbatim, including 0 (which disables that + // retention dimension). TargetFileSize must stay > 0, so a 0/absent value preserves the default. + if v := appOpts.Get(FlagSCHashLoggerBlocksToRetain); v != nil { + scConfig.HashLogger.BlocksToRetain = cast.ToUint(v) + } + if v := appOpts.Get(FlagSCHashLoggerTargetFileSize); v != nil { + if n := cast.ToUint(v); n > 0 { + scConfig.HashLogger.TargetFileSize = n + } + } + if v := appOpts.Get(FlagSCHashLoggerMaxDiskSize); v != nil { + scConfig.HashLogger.MaxDiskSize = cast.ToUint(v) + } + // The software version is embedded in hash log file names so archives from different builds are + // distinguishable. Sourced from the node build version, not from app.toml. + scConfig.HashLogger.Version = version.Version + return scConfig } diff --git a/app/test_helpers.go b/app/test_helpers.go index 9a509d2149..8d25f981b0 100644 --- a/app/test_helpers.go +++ b/app/test_helpers.go @@ -92,6 +92,13 @@ func (t TestAppOpts) Get(s string) interface{} { if s == FlagSCSnapshotInterval { return uint32(0) // 0 = disabled } + // Disable hash logging in tests. It runs background writer/control goroutines that are only + // joined by Store.Close(); tests that don't close the app would otherwise leave those goroutines + // rotating files in the temp data dir while t.TempDir() cleanup removes it, failing with + // "directory not empty". + if s == FlagSCHashLoggerEnable { + return false + } if s == FlagSSEnable { return true } diff --git a/sei-cosmos/baseapp/abci.go b/sei-cosmos/baseapp/abci.go index 87972bb052..e6478fffbf 100644 --- a/sei-cosmos/baseapp/abci.go +++ b/sei-cosmos/baseapp/abci.go @@ -22,6 +22,7 @@ import ( sdkerrors "github.com/sei-protocol/sei-chain/sei-cosmos/types/errors" "github.com/sei-protocol/sei-chain/sei-cosmos/types/legacytm" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/crypto/merkle" tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" "go.opentelemetry.io/otel/attribute" otelmetric "go.opentelemetry.io/otel/metric" @@ -293,6 +294,17 @@ func (app *BaseApp) Commit(ctx context.Context) (res *abci.ResponseCommit, err e app.WriteState() app.GetWorkingHash() + // Hand the block hash to the commit store so it can record it in the per-block hash log under the + // same block number as the state hashes (the commit store cannot see the block hash on its own). + if reporter, ok := app.cms.(interface{ SetNextBlockHash([]byte) }); ok { + reporter.SetNextBlockHash(app.stateToCommit.ctx.HeaderHash()) + } + // Hand the result hash (computed in FinalizeBlock from the block's tx results) to the commit + // store so it lands in the same per-block hash log row as the block and state hashes. + if reporter, ok := app.cms.(interface{ SetNextResultHash([]byte) }); ok { + reporter.SetNextResultHash(app.nextResultHash) + } + app.nextResultHash = nil app.cms.Commit(true) // Reset the Check state to the latest committed. @@ -1073,6 +1085,21 @@ func (app *BaseApp) FinalizeBlock(ctx context.Context, req *abci.RequestFinalize if err != nil { return nil, err } + // Compute the result hash (merkle root over the block's deterministic tx results: Code, Data, + // GasWanted, GasUsed) so Commit can record it in the hash log. This is the same value Tendermint + // stores as the next block's header.LastResultsHash; logging it per block surfaces gas/result + // divergence (e.g. between executors) independently of the state AppHash. Only computed when the + // commit store is actively recording hashes (HashLoggingEnabled, not just method presence — the + // store always defines SetNextResultHash), and never fails the block on a marshal error. + if r, ok := app.cms.(interface{ HashLoggingEnabled() bool }); ok && r.HashLoggingEnabled() { + marshaled, mErr := abci.MarshalTxResults(res.TxResults) + if mErr != nil { + logger.Error("failed to marshal tx results for result hash", "err", mErr) + app.nextResultHash = nil + } else { + app.nextResultHash = merkle.HashFromByteSlices(marshaled) + } + } res.Events = sdk.MarkEventsToIndex(res.Events, app.IndexEvents) return res, nil } else { diff --git a/sei-cosmos/baseapp/baseapp.go b/sei-cosmos/baseapp/baseapp.go index 25cec9bbae..69401993b9 100644 --- a/sei-cosmos/baseapp/baseapp.go +++ b/sei-cosmos/baseapp/baseapp.go @@ -122,6 +122,10 @@ type BaseApp struct { processProposalCleanCtx sdk.Context // snapshot before optimistic processing stateToCommit *state + // nextResultHash is the result hash (merkle root over the block's deterministic tx results) + // computed in FinalizeBlock and handed to the commit store in Commit, mirroring nextBlockHash. + nextResultHash []byte + // paramStore is used to query for ABCI consensus parameters from an // application parameter store. paramStore ParamStore diff --git a/sei-cosmos/storev2/rootmulti/hashlog.go b/sei-cosmos/storev2/rootmulti/hashlog.go new file mode 100644 index 0000000000..4d3a91a25e --- /dev/null +++ b/sei-cosmos/storev2/rootmulti/hashlog.go @@ -0,0 +1,202 @@ +package rootmulti + +import ( + "fmt" + "path/filepath" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" +) + +// App-level hash logger categories owned by rootmulti. No backend can compute these: +// - appHashType: the root application hash returned to consensus. +// - blockHashType: the Tendermint block hash, supplied by baseapp via SetNextBlockHash. +// - resultHashType: the result hash (merkle root over the block's deterministic tx results), +// supplied by baseapp via SetNextResultHash. Equals the next block's header.LastResultsHash. +// - memIAVLRootHashType: the simple-merkle root over memIAVL's per-module hashes (requires the cosmos +// hashing utilities, which sei-db cannot reach), computed here via convertCommitInfo(...).Hash(). +const ( + appHashType = "appHash" + blockHashType = "blockHash" + resultHashType = "resultHash" + memIAVLRootHashType = "memIAVL/root" +) + +// hashReportingStore is the subset of the SC store that owns and reports its own hash categories. The +// composite commit store implements it. Type-asserting keeps these methods out of the broad +// sctypes.Committer interface. The caller registers the categories returned by HashCategories. +type hashReportingStore interface { + HashCategories() []string + RecordHashes(hashlog.HashLogger, uint64) error + MemIAVLCommitInfo() *proto.CommitInfo +} + +// SetNextBlockHash stashes the Tendermint block hash for the block currently being committed. baseapp +// calls it just before Commit; rootmulti records it (under the committed version) inside Commit so every +// hash for a block shares one block number. No-op when hash logging is disabled. +func (rs *Store) SetNextBlockHash(blockHash []byte) { + if rs.hashLoggerDisabled { + return + } + rs.nextBlockHash = append([]byte(nil), blockHash...) +} + +// SetNextResultHash stashes the result hash (merkle root over the block's deterministic tx results) +// for the block currently being committed. baseapp calls it just before Commit; rootmulti records it +// (under the committed version) inside Commit so every hash for a block shares one block number. +// No-op when hash logging is disabled. +func (rs *Store) SetNextResultHash(resultHash []byte) { + if rs.hashLoggerDisabled { + return + } + rs.nextResultHash = append([]byte(nil), resultHash...) +} + +// hashLogDir returns the directory hash log files are written to, defaulting to a "hash.log" directory +// under the state-commit store's data directory (sibling of committer.db / receipt.db). The ".log" +// suffix mirrors the data/ naming convention (.db, .wal); the files inside keep the .hlog format. +func (rs *Store) hashLogDir() string { + if rs.hashLoggerConfig.Directory != "" { + return rs.hashLoggerConfig.Directory + } + return filepath.Join(rs.scDir, "data", "hash.log") +} + +// desiredHashCategories computes the full caller-reported category set for the current backend state: +// the app-level categories plus whatever the live backends report (and memIAVL/root when memIAVL is +// present). The backend set is dynamic (memIAVL departs and flatKV arrives during migration), so this +// is recomputed each block and used to detect when the logger's column set must change. +func (rs *Store) desiredHashCategories() map[string]struct{} { + categories := map[string]struct{}{ + appHashType: {}, + blockHashType: {}, + resultHashType: {}, + } + if h, ok := rs.scStore.(hashReportingStore); ok { + for _, category := range h.HashCategories() { + categories[category] = struct{}{} + } + if h.MemIAVLCommitInfo() != nil { + categories[memIAVLRootHashType] = struct{}{} + } + } + return categories +} + +// openHashLogger constructs the logger once. It starts with no caller columns (just the changeset +// column); syncHashCategories then registers the live categories, which the logger handles as runtime +// column changes (each new column rotates to a fresh file, but the empty initial files are dropped and +// their indexes reused, so the first file with data starts at index 0). +func (rs *Store) openHashLogger() error { + loggerVersion := rs.hashLoggerConfig.Version + if loggerVersion == "" { + loggerVersion = "unknown" + } + cfg := hashlog.DefaultHashLoggerConfig(rs.hashLogDir(), loggerVersion) + // Propagate the operator-configured retention tunables verbatim. A configured 0 must reach the logger + // (where it disables that dimension); the old `if > 0` guards swallowed it. Defaults are applied at + // config construction (config.DefaultHashLoggerConfig), so these always carry a meaningful value. + cfg.BlocksToRetain = rs.hashLoggerConfig.BlocksToRetain + cfg.TargetFileSize = rs.hashLoggerConfig.TargetFileSize + cfg.MaxDiskSize = rs.hashLoggerConfig.MaxDiskSize + + hl, err := hashlog.NewHashLogger(cfg) + if err != nil { + return fmt.Errorf("failed to create hash logger: %w", err) + } + rs.hashLogger = hl + return nil +} + +// syncHashCategories brings the logger's column set in line with the desired set for the current backend +// state, registering newly-present categories and unregistering departed ones. The logger rotates files +// as needed on each change; a no-op when the set is unchanged (the common case after the first block). +func (rs *Store) syncHashCategories() { + desired := rs.desiredHashCategories() + for category := range rs.hashCategories { + if _, ok := desired[category]; !ok { + if err := rs.hashLogger.UnregisterHashType(category); err != nil { + logger.Error("failed to unregister hash category", "category", category, "err", err) + } + } + } + for category := range desired { + if _, ok := rs.hashCategories[category]; !ok { + if err := rs.hashLogger.RegisterHashType(category); err != nil { + logger.Error("failed to register hash category", "category", category, "err", err) + } + } + } + rs.hashCategories = desired +} + +// HashLoggingEnabled reports whether the store is actively recording hashes (config-enabled and not +// disabled by a prior fatal error). baseapp uses it to skip per-block hash computation when off. +func (rs *Store) HashLoggingEnabled() bool { + return !rs.hashLoggerDisabled +} + +// disableHashLogger turns hash logging off after a fatal error, closing the logger if it is open. +func (rs *Store) disableHashLogger() { + rs.hashLoggerDisabled = true + if rs.hashLogger != nil { + _ = rs.hashLogger.Close() + rs.hashLogger = nil + } +} + +// recordBlockHashes reports every hash for the just-committed block at the given version. It opens the +// logger on first use and keeps its column set in sync with the live backends. On open failure it +// disables hash logging rather than disrupting commit. Must be called with rs.mtx held (from Commit). +func (rs *Store) recordBlockHashes(version int64) { + if rs.hashLoggerDisabled { + return + } + + if rs.hashLogger == nil { + if err := rs.openHashLogger(); err != nil { + logger.Error("failed to open hash logger; disabling hash logging", "err", err) + rs.disableHashLogger() + return + } + } + rs.syncHashCategories() + + blockNumber := uint64(version) //nolint:gosec // commit versions are non-negative + + // Changeset: the aggregate of all modules' changes for this block, captured (sorted) in flush. + rs.hashLogger.ReportChangeset(blockNumber, rs.blockChangeSets) + rs.blockChangeSets = nil + + // appHash: the root application hash returned to consensus. + if rs.lastCommitInfo != nil { + appHash := append([]byte(nil), rs.lastCommitInfo.CommitID().Hash...) + if err := rs.hashLogger.ReportHash(blockNumber, appHashType, appHash); err != nil { + logger.Error("failed to report app hash", "err", err) + } + } + + // blockHash: supplied by baseapp before Commit. nil if it was not provided for this block. + if err := rs.hashLogger.ReportHash(blockNumber, blockHashType, rs.nextBlockHash); err != nil { + logger.Error("failed to report block hash", "err", err) + } + rs.nextBlockHash = nil + + // resultHash: supplied by baseapp before Commit. nil if it was not provided for this block. + if err := rs.hashLogger.ReportHash(blockNumber, resultHashType, rs.nextResultHash); err != nil { + logger.Error("failed to report result hash", "err", err) + } + rs.nextResultHash = nil + + if h, ok := rs.scStore.(hashReportingStore); ok { + if memInfo := h.MemIAVLCommitInfo(); memInfo != nil { + root := convertCommitInfo(memInfo).Hash() + if err := rs.hashLogger.ReportHash(blockNumber, memIAVLRootHashType, root); err != nil { + logger.Error("failed to report memIAVL root hash", "err", err) + } + } + if err := h.RecordHashes(rs.hashLogger, blockNumber); err != nil { + logger.Error("failed to record backend hashes", "err", err) + } + } +} diff --git a/sei-cosmos/storev2/rootmulti/hashlog_test.go b/sei-cosmos/storev2/rootmulti/hashlog_test.go new file mode 100644 index 0000000000..6649d9fb24 --- /dev/null +++ b/sei-cosmos/storev2/rootmulti/hashlog_test.go @@ -0,0 +1,97 @@ +package rootmulti + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" +) + +// End-to-end: committing blocks through rootmulti produces a per-block hash log on disk with every +// expected column (app/block/state hashes + changeset) populated, and the recorded values match the +// store's own commit hashes and the supplied block hashes. +func TestRootMultiHashLogging(t *testing.T) { + home := t.TempDir() + scCfg := config.DefaultStateCommitConfig() + scCfg.Enable = true + scCfg.HashLogger.Enable = true + scCfg.HashLogger.Version = "test-v1" + ssCfg := config.DefaultStateStoreConfig() + ssCfg.Enable = false + + store := NewStore(home, scCfg, ssCfg, []string{}) + store.MountStoreWithDB(types.NewKVStoreKey("bank"), types.StoreTypeIAVL, nil) + store.MountStoreWithDB(types.NewKVStoreKey("evm"), types.StoreTypeIAVL, nil) + require.NoError(t, store.LoadLatestVersion()) + + blockHashes := map[int64][]byte{} + resultHashes := map[int64][]byte{} + for h := int64(1); h <= 3; h++ { + store.GetStoreByName("bank").(types.KVStore).Set([]byte("k"), []byte{byte(h)}) + store.GetStoreByName("evm").(types.KVStore).Set([]byte("k"), []byte{byte(h + 10)}) + blockHash := []byte{0xBB, byte(h)} + blockHashes[h] = blockHash + store.SetNextBlockHash(blockHash) + resultHash := []byte{0xCC, byte(h)} + resultHashes[h] = resultHash + store.SetNextResultHash(resultHash) + commitID := store.Commit(true) + require.Equal(t, h, commitID.Version) + } + lastAppHash := store.LastCommitID().Hash + + // Close flushes the logger so all complete blocks are written. + require.NoError(t, store.Close()) + + dir := filepath.Join(home, "data", "hash.log") + expectedColumns := []string{ + "appHash", "blockHash", "resultHash", "memIAVL/root", + "memIAVL/mod/bank", "memIAVL/mod/evm", hashlog.ChangesetHashType, + } + for h := int64(1); h <= 3; h++ { + logs, err := hashlog.ReadHashForBlock(dir, uint64(h)) + require.NoError(t, err) + require.Len(t, logs, 1, "exactly one record per block") + hashes := logs[0].Hashes + for _, column := range expectedColumns { + require.Contains(t, hashes, column) + require.NotEmpty(t, hashes[column], "column %q for block %d should be populated", column, h) + } + require.Equal(t, blockHashes[h], hashes["blockHash"], "block hash for block %d", h) + require.Equal(t, resultHashes[h], hashes["resultHash"], "result hash for block %d", h) + } + + // The last block's appHash column equals the store's committed app hash. + logs, err := hashlog.ReadHashForBlock(dir, 3) + require.NoError(t, err) + require.Equal(t, lastAppHash, logs[0].Hashes["appHash"]) +} + +// When hash logging is disabled, no hash log directory/files are produced and commits still succeed. +func TestRootMultiHashLoggingDisabled(t *testing.T) { + home := t.TempDir() + scCfg := config.DefaultStateCommitConfig() + scCfg.Enable = true + scCfg.HashLogger.Enable = false + ssCfg := config.DefaultStateStoreConfig() + ssCfg.Enable = false + + store := NewStore(home, scCfg, ssCfg, []string{}) + store.MountStoreWithDB(types.NewKVStoreKey("bank"), types.StoreTypeIAVL, nil) + require.NoError(t, store.LoadLatestVersion()) + + store.GetStoreByName("bank").(types.KVStore).Set([]byte("k"), []byte("v")) + store.SetNextBlockHash([]byte{0x01}) // no-op when disabled + require.Equal(t, int64(1), store.Commit(true).Version) + require.NoError(t, store.Close()) + + logs, err := hashlog.ReadHashForBlock(filepath.Join(home, "data", "hash.log"), 1) + // Either the directory does not exist (error) or there are no records. + if err == nil { + require.Empty(t, logs) + } +} diff --git a/sei-cosmos/storev2/rootmulti/store.go b/sei-cosmos/storev2/rootmulti/store.go index 25861a13a0..889e88cbf5 100644 --- a/sei-cosmos/storev2/rootmulti/store.go +++ b/sei-cosmos/storev2/rootmulti/store.go @@ -34,6 +34,7 @@ import ( seidbtypes "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/composite" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" sctypes "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" @@ -61,6 +62,23 @@ type Store struct { histProofLimiter *rate.Limiter snapshotSCStoreWarnOnce sync.Once + + // Hash logger state (per-block hash logging; a debugging/forensics tool). See hashlog.go. + hashLoggerConfig config.HashLoggerConfig + hashLoggerDisabled bool + hashLogger hashlog.HashLogger + hashCategories map[string]struct{} // the category set the current logger was opened with + scDir string // state-commit directory, for the default hash log location + // blockChangeSets is the aggregate changeset captured in flush for the block being committed, then + // reported (and cleared) in Commit. changesetCapturedVersion guards against the double-flush so it is + // captured only once (with the real, non-empty changeset) per block. + blockChangeSets []*proto.NamedChangeSet + changesetCapturedVersion int64 + // nextBlockHash is the Tendermint block hash supplied by baseapp for the block being committed. + nextBlockHash []byte + // nextResultHash is the result hash (merkle root over the block's deterministic tx results) + // supplied by baseapp for the block being committed. + nextResultHash []byte } type VersionedChangesets struct { @@ -106,13 +124,16 @@ func NewStore( } } store := &Store{ - scStore: scStore, - storesParams: make(map[types.StoreKey]storeParams), - storeKeys: make(map[string]types.StoreKey), - ckvStores: make(map[types.StoreKey]types.CommitKVStore), - gigaKeys: gigaKeys, - histProofSem: make(chan struct{}, maxInFlight), - histProofLimiter: limiter, + scStore: scStore, + storesParams: make(map[types.StoreKey]storeParams), + storeKeys: make(map[string]types.StoreKey), + ckvStores: make(map[types.StoreKey]types.CommitKVStore), + gigaKeys: gigaKeys, + histProofSem: make(chan struct{}, maxInFlight), + histProofLimiter: limiter, + hashLoggerConfig: scConfig.HashLogger, + hashLoggerDisabled: !scConfig.HashLogger.Enable, + scDir: scDir, } if ssConfig.Enable { ssStore, err := ss.NewStateStore(homeDir, ssConfig) @@ -175,6 +196,7 @@ func (rs *Store) Commit(bumpVersion bool) types.CommitID { rs.lastCommitInfo = convertCommitInfo(rs.scStore.LastCommitInfo()) rs.lastCommitInfo = amendCommitInfo(rs.lastCommitInfo, rs.storesParams) + rs.recordBlockHashes(rs.lastCommitInfo.Version) return rs.lastCommitInfo.CommitID() } @@ -199,6 +221,21 @@ func (rs *Store) flush() error { sort.SliceStable(changeSets, func(i, j int) bool { return changeSets[i].Name < changeSets[j].Name }) + } + // Capture the (sorted) aggregate changeset for hash logging once per block. rootmulti flushes twice + // per block (GetWorkingHash then Commit) but only the first flush carries the real changeset — the + // second sees an empty set because PopChangeSet already drained it — so capture only the first time. + // nil is normalized to an empty (non-nil) set so an empty block records the stable empty-changeset + // hash rather than a nil one. + if !rs.hashLoggerDisabled && rs.changesetCapturedVersion != currentVersion { + if changeSets == nil { + rs.blockChangeSets = []*proto.NamedChangeSet{} + } else { + rs.blockChangeSets = changeSets + } + rs.changesetCapturedVersion = currentVersion + } + if len(changeSets) > 0 { if rs.ssStore != nil { if err := rs.ssStore.ApplyChangesetAsync(currentVersion, changeSets); err != nil { return err @@ -226,6 +263,9 @@ func (rs *Store) Close() error { if rs.ssStore != nil { err = commonerrors.Join(err, rs.ssStore.Close()) } + if rs.hashLogger != nil { + err = commonerrors.Join(err, rs.hashLogger.Close()) + } return err } diff --git a/sei-db/config/hashlog_config.go b/sei-db/config/hashlog_config.go new file mode 100644 index 0000000000..f4ddf5d2af --- /dev/null +++ b/sei-db/config/hashlog_config.go @@ -0,0 +1,44 @@ +package config + +import "github.com/sei-protocol/sei-chain/sei-db/common/unit" + +// HashLoggerConfig configures the per-block hash logger: a debugging/forensics tool that records a CSV +// of named block hashes (memIAVL module/root hashes, flatKV DB/root hashes, the app hash, the block +// hash, and the changeset hash) so that block-hash computation can be studied and compared across nodes. +type HashLoggerConfig struct { + // These fields are loaded by explicit flag reads in app.parseSCConfigs (keys: sc-hash-logger-*), + // not via mapstructure, so they carry no mapstructure tags. + + // Enable turns on per-block hash logging. Defaults to true. + Enable bool + + // Directory is where hash log files are written. If empty, defaults to a "hash.log" directory under + // the state-commit store's data directory. + Directory string + + // BlocksToRetain is the number of most-recent blocks to keep on disk. 0 disables block-count + // retention (the disk-size cap is then the only bound). + BlocksToRetain uint + + // TargetFileSize is the size in bytes a log file may reach before it is sealed and rotated. Must be > 0. + TargetFileSize uint + + // MaxDiskSize is a backstop cap (bytes) on the total size of sealed log files. 0 disables the disk-size + // cap (block-count retention is then the only bound). + MaxDiskSize uint + + // Version is the software version embedded in hash log file names. It is populated by the app layer + // (from the node's build version), not parsed from config. + Version string +} + +// DefaultHashLoggerConfig returns the default HashLoggerConfig. Retention is disk-driven: keep up to +// 16 GiB of sealed files (~7 weeks at tip), with block-count retention disabled. +func DefaultHashLoggerConfig() HashLoggerConfig { + return HashLoggerConfig{ + Enable: true, + BlocksToRetain: 0, // disabled — retention is purely disk-driven + TargetFileSize: 16 * unit.MB, + MaxDiskSize: 16 * unit.GB, + } +} diff --git a/sei-db/config/sc_config.go b/sei-db/config/sc_config.go index c1c78aa283..f128bb7c51 100644 --- a/sei-db/config/sc_config.go +++ b/sei-db/config/sc_config.go @@ -53,6 +53,10 @@ type StateCommitConfig struct { // The number of keys to migrate from memiavl to flatkv per block. Ignored if not in a migration mode. KeysToMigratePerBlock int `mapstructure:"keys-to-migrate-per-block"` + + // HashLogger configures the per-block hash logger (a debugging/forensics tool). Enabled by default. + // Loaded via explicit sc-hash-logger-* flag reads in app.parseSCConfigs, not mapstructure. + HashLogger HashLoggerConfig } // DefaultStateCommitConfig returns the default StateCommitConfig @@ -66,6 +70,7 @@ func DefaultStateCommitConfig() StateCommitConfig { HistoricalProofRateLimit: DefaultSCHistoricalProofRateLimit, HistoricalProofBurst: DefaultSCHistoricalProofBurst, KeysToMigratePerBlock: 1024, + HashLogger: DefaultHashLoggerConfig(), } } diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 2626475023..dcaa9a2474 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -71,6 +71,25 @@ sc-write-mode = "{{ .StateCommit.WriteMode }}" # Must be > 0; ignored entirely when not in a migration mode. sc-keys-to-migrate-per-block = {{ .StateCommit.KeysToMigratePerBlock }} +# HashLogger records a per-block CSV of named hashes (memIAVL module/root hashes, flatKV DB/root +# hashes, the app hash, the block hash, and the changeset hash) so block-hash computation can be +# studied and compared across nodes. It is a debugging/forensics tool; enabled by default. +sc-hash-logger-enable = {{ .StateCommit.HashLogger.Enable }} + +# Directory for hash log files. If empty, defaults to a "hash.log" directory under the SC store's data +# directory (i.e. /data/hash.log). +sc-hash-logger-directory = "{{ .StateCommit.HashLogger.Directory }}" + +# Number of most-recent blocks to retain on disk. 0 disables block-count retention (disk-size cap only). +sc-hash-logger-blocks-to-retain = {{ .StateCommit.HashLogger.BlocksToRetain }} + +# Size in bytes a hash log file may reach before it is sealed and rotated. Must be > 0. +sc-hash-logger-target-file-size = {{ .StateCommit.HashLogger.TargetFileSize }} + +# Backstop cap in bytes on the total size of sealed hash log files. 0 disables the disk-size cap +# (block-count retention only). +sc-hash-logger-max-disk-size = {{ .StateCommit.HashLogger.MaxDiskSize }} + ############################################################################### ### FlatKV (EVM) Configuration ### ############################################################################### diff --git a/sei-db/state_db/sc/composite/hashlog.go b/sei-db/state_db/sc/composite/hashlog.go new file mode 100644 index 0000000000..06df0f4963 --- /dev/null +++ b/sei-db/state_db/sc/composite/hashlog.go @@ -0,0 +1,46 @@ +package composite + +import ( + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" +) + +// HashCategories returns the union of the live backends' hash logger categories. An absent backend +// contributes nothing, so the set tracks which backends are active (used upstream to detect when the +// logger's category set must change). Note: the memIAVL root ("memIAVL/root") is not included here — it +// is a simple-merkle aggregation owned by the cosmos layer (see MemIAVLCommitInfo). +func (cs *CompositeCommitStore) HashCategories() []string { + var categories []string + if cs.memIAVL != nil { + categories = append(categories, cs.memIAVL.HashCategories()...) + } + if cs.flatKV != nil { + categories = append(categories, cs.flatKV.HashCategories()...) + } + return categories +} + +// RecordHashes reports every live backend's hashes for blockNumber. Call right after Commit. +func (cs *CompositeCommitStore) RecordHashes(hl hashlog.HashLogger, blockNumber uint64) error { + if cs.memIAVL != nil { + if err := cs.memIAVL.RecordHashes(hl, blockNumber); err != nil { + return err + } + } + if cs.flatKV != nil { + if err := cs.flatKV.RecordHashes(hl, blockNumber); err != nil { + return err + } + } + return nil +} + +// MemIAVLCommitInfo returns the raw memIAVL commit info (its per-store hashes), or nil when memIAVL is +// not present. The cosmos layer uses it to compute the memIAVL root hash (a simple-merkle aggregation +// that requires the cosmos hashing utilities), which sei-db cannot compute on its own. +func (cs *CompositeCommitStore) MemIAVLCommitInfo() *proto.CommitInfo { + if cs.memIAVL == nil { + return nil + } + return cs.memIAVL.LastCommitInfo() +} diff --git a/sei-db/state_db/sc/composite/store_test.go b/sei-db/state_db/sc/composite/store_test.go index 20f1193ac2..35226125be 100644 --- a/sei-db/state_db/sc/composite/store_test.go +++ b/sei-db/state_db/sc/composite/store_test.go @@ -17,6 +17,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/migration" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" ) @@ -41,18 +42,20 @@ func (f *failingEVMStore) RawGlobalIterator() (dbm.Iterator, error) { return nil func (f *failingEVMStore) Iterator(string, []byte, []byte, bool) (dbm.Iterator, error) { return nil, nil } -func (f *failingEVMStore) RootHash() []byte { return nil } -func (f *failingEVMStore) Version() int64 { return 0 } -func (f *failingEVMStore) EarliestVersion() int64 { return 0 } -func (f *failingEVMStore) GetLatestVersion() (int64, error) { return 0, nil } -func (f *failingEVMStore) WriteSnapshot(string) error { return nil } -func (f *failingEVMStore) Rollback(int64) error { return nil } -func (f *failingEVMStore) Exporter(int64) (types.Exporter, error) { return nil, nil } -func (f *failingEVMStore) Importer(int64) (types.Importer, error) { return nil, nil } -func (f *failingEVMStore) GetPhaseTimer() *metrics.PhaseTimer { return nil } -func (f *failingEVMStore) CommittedRootHash() []byte { return nil } -func (f *failingEVMStore) CleanupOrphanedReadOnlyDirs() error { return nil } -func (f *failingEVMStore) Close() error { return nil } +func (f *failingEVMStore) RootHash() []byte { return nil } +func (f *failingEVMStore) Version() int64 { return 0 } +func (f *failingEVMStore) EarliestVersion() int64 { return 0 } +func (f *failingEVMStore) GetLatestVersion() (int64, error) { return 0, nil } +func (f *failingEVMStore) WriteSnapshot(string) error { return nil } +func (f *failingEVMStore) Rollback(int64) error { return nil } +func (f *failingEVMStore) Exporter(int64) (types.Exporter, error) { return nil, nil } +func (f *failingEVMStore) Importer(int64) (types.Importer, error) { return nil, nil } +func (f *failingEVMStore) GetPhaseTimer() *metrics.PhaseTimer { return nil } +func (f *failingEVMStore) CommittedRootHash() []byte { return nil } +func (f *failingEVMStore) HashCategories() []string { return nil } +func (f *failingEVMStore) RecordHashes(hashlog.HashLogger, uint64) error { return nil } +func (f *failingEVMStore) CleanupOrphanedReadOnlyDirs() error { return nil } +func (f *failingEVMStore) Close() error { return nil } // eraFailingEVMStore is a failingEVMStore with a configurable // EarliestVersion, used to exercise Exporter's pre-era vs in-history diff --git a/sei-db/state_db/sc/flatkv/api.go b/sei-db/state_db/sc/flatkv/api.go index 70993d6625..070ba27dce 100644 --- a/sei-db/state_db/sc/flatkv/api.go +++ b/sei-db/state_db/sc/flatkv/api.go @@ -7,6 +7,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/common/metrics" "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" ) @@ -107,6 +108,13 @@ type Store interface { // CommittedRootHash returns the 32-byte checksum of the last committed LtHash. CommittedRootHash() []byte + // HashCategories returns the hash logger category names this store reports (the global root plus one + // per data DB). The set is fixed. The caller registers these on the logger. + HashCategories() []string + + // RecordHashes reports this store's hashes (root + per-DB) for blockNumber. Call right after Commit. + RecordHashes(hl hashlog.HashLogger, blockNumber uint64) error + // Version returns the latest committed version. Version() int64 diff --git a/sei-db/state_db/sc/flatkv/hashlog.go b/sei-db/state_db/sc/flatkv/hashlog.go new file mode 100644 index 0000000000..63a7ed084a --- /dev/null +++ b/sei-db/state_db/sc/flatkv/hashlog.go @@ -0,0 +1,48 @@ +package flatkv + +import ( + "fmt" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" +) + +// Hash logger category names owned by the flatKV backend. flatKVDBHashPrefix is joined with a data DB +// directory name (e.g. "flatKV/db/account"). The metadata DB is intentionally excluded — it holds only +// watermarks, not state. +const ( + FlatKVRootHashType = "flatKV/root" + flatKVDBHashPrefix = "flatKV/db/" +) + +// HashCategories returns the hash logger categories this store reports: the global flatKV root plus one +// per data DB. The set is fixed (the data DBs never change), so callers can use it to detect when the +// overall logged category set has changed. +func (s *CommitStore) HashCategories() []string { + categories := make([]string, 0, len(dataDBDirs)+1) + categories = append(categories, FlatKVRootHashType) + for _, dir := range dataDBDirs { + categories = append(categories, flatKVDBHashPrefix+dir) + } + return categories +} + +// RecordHashes reports this store's hashes for blockNumber: the committed global root and each data DB's +// committed per-DB LtHash checksum. Intended to be called right after Commit, when localMeta holds the +// just-committed per-DB hashes and CommittedRootHash reflects the same version. +func (s *CommitStore) RecordHashes(hl hashlog.HashLogger, blockNumber uint64) error { + if err := hl.ReportHash(blockNumber, FlatKVRootHashType, s.CommittedRootHash()); err != nil { + return fmt.Errorf("failed to report flatkv root hash: %w", err) + } + for _, dir := range dataDBDirs { + var hash []byte + if meta := s.localMeta[dir]; meta != nil && meta.LtHash != nil { + checksum := meta.LtHash.Checksum() + hash = checksum[:] + } + category := flatKVDBHashPrefix + dir + if err := hl.ReportHash(blockNumber, category, hash); err != nil { + return fmt.Errorf("failed to report flatkv db hash %q: %w", category, err) + } + } + return nil +} diff --git a/sei-db/state_db/sc/flatkv/hashlog_test.go b/sei-db/state_db/sc/flatkv/hashlog_test.go new file mode 100644 index 0000000000..bc58103136 --- /dev/null +++ b/sei-db/state_db/sc/flatkv/hashlog_test.go @@ -0,0 +1,89 @@ +package flatkv + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" +) + +// captureLogger is a HashLogger test double that records registered categories and reported hashes. +type captureLogger struct { + registered map[string]struct{} + hashes map[string][]byte + changesets int +} + +func newCaptureLogger() *captureLogger { + return &captureLogger{registered: map[string]struct{}{}, hashes: map[string][]byte{}} +} + +func (c *captureLogger) RegisterHashType(hashType string) error { + c.registered[hashType] = struct{}{} + return nil +} + +func (c *captureLogger) UnregisterHashType(hashType string) error { + delete(c.registered, hashType) + return nil +} + +func (c *captureLogger) ReportHash(_ uint64, hashType string, hash []byte) error { + c.hashes[hashType] = hash + return nil +} + +func (c *captureLogger) ReportChangeset(uint64, []*proto.NamedChangeSet) { c.changesets++ } + +func (c *captureLogger) Close() error { return nil } + +func TestFlatKVHashReporting(t *testing.T) { + s := setupTestStore(t) + defer func() { require.NoError(t, s.Close()) }() + + // Write some EVM storage so the account/storage DBs have non-empty LtHashes. + key := evmStorageKey(ktype.Address{0x11}, ktype.Slot{0x22}) + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, padLeft32(0x33), false)})) + _, err := s.Commit() + require.NoError(t, err) + + // Categories: the global root plus one per data DB (metadata DB excluded). + require.Equal(t, []string{ + "flatKV/root", + "flatKV/db/account", + "flatKV/db/code", + "flatKV/db/storage", + "flatKV/db/legacy", + }, s.HashCategories()) + + logger := newCaptureLogger() + for _, category := range s.HashCategories() { + require.NoError(t, logger.RegisterHashType(category)) + } + require.Len(t, logger.registered, 5) + + require.NoError(t, s.RecordHashes(logger, 1)) + + // Every category is reported, and the root matches CommittedRootHash. + for _, category := range s.HashCategories() { + _, ok := logger.hashes[category] + require.True(t, ok, "expected a hash for %q", category) + } + require.Equal(t, s.CommittedRootHash(), logger.hashes["flatKV/root"]) + + // Each reported per-DB hash is the checksum of that DB's committed LtHash. + for _, dir := range dataDBDirs { + checksum := s.localMeta[dir].LtHash.Checksum() + require.Equal(t, checksum[:], logger.hashes["flatKV/db/"+dir]) + } + + // Homomorphic invariant: the per-DB LtHashes sum to the committed global LtHash. + sum := lthash.New() + for _, dir := range dataDBDirs { + sum.MixIn(s.localMeta[dir].LtHash) + } + require.True(t, sum.Equal(s.committedLtHash)) +} diff --git a/sei-db/state_db/sc/hashlog/hash_logger.go b/sei-db/state_db/sc/hashlog/hash_logger.go index 4e07437d77..66e85d425c 100644 --- a/sei-db/state_db/sc/hashlog/hash_logger.go +++ b/sei-db/state_db/sc/hashlog/hash_logger.go @@ -32,8 +32,28 @@ type HashLogger interface { // treated differently. ReportChangeset(blockNumber uint64, cs []*proto.NamedChangeSet) + // Register an additional caller-reported hash type (CSV column). This supplements the types supplied at + // construction via HashLoggerConfig.HashTypes, letting a caller (e.g. a database that owns its own hash + // categories) populate the column set without knowing every type up front. + // + // It may be called at any time, including after blocks have been logged. Because a hash log file's + // header is fixed, changing the column set rotates the file: the logger flushes all complete blocks to + // the current file, seals it, and opens a fresh file whose header includes the new column. Registering a + // type that is already present is a no-op (no rotation). The reserved changeset type is rejected, as are + // names containing characters outside the legal allow-list. Returns nil once the change has been applied + // (the call blocks until then), so a subsequent ReportHash for the new column is accepted. + // + // Callers must not invoke the Register/Unregister/Report methods concurrently from multiple goroutines. + RegisterHashType(hashType string) error + + // Unregister a previously registered caller-reported hash type, removing its column. Like + // RegisterHashType this rotates to a fresh file whose header omits the column. Removing a type that is + // not present is a no-op; the reserved changeset column cannot be removed. + UnregisterHashType(hashType string) error + // Report a hash for a block under the given type. The type must be one of the types this logger was - // configured to record, otherwise an error is returned. The changeset hash type is reserved for the + // configured to record (via HashLoggerConfig.HashTypes or RegisterHashType), otherwise an error is + // returned. The changeset hash type is reserved for the // logger-computed changeset column (use ReportChangeset) and is also rejected when changeset hashing is enabled. A // subsystem that is disabled should report a nil hash for its type rather than skipping the call, so that // the block can still be completed. diff --git a/sei-db/state_db/sc/hashlog/hash_logger_config.go b/sei-db/state_db/sc/hashlog/hash_logger_config.go index 83d00ecb5e..cc887cafb9 100644 --- a/sei-db/state_db/sc/hashlog/hash_logger_config.go +++ b/sei-db/state_db/sc/hashlog/hash_logger_config.go @@ -15,8 +15,10 @@ import ( const ChangesetHashType = "changeset" // Hash type names are written verbatim into CSV headers and must not collide with the "," field -// separator or any other structural character. We restrict them to a small, safe allowlist. -var legalHashTypeRegex = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) +// separator or any other structural character. We restrict them to a small, safe allowlist. "/" is +// permitted so callers can namespace columns hierarchically (e.g. "memIAVL/mod/bank", "flatKV/root"); +// it is CSV-safe and hash type names never appear in file names (only the sanitized version does). +var legalHashTypeRegex = regexp.MustCompile(`^[A-Za-z0-9_./-]+$`) // Configuration for a HashLogger. type HashLoggerConfig struct { @@ -53,15 +55,17 @@ type HashLoggerConfig struct { // This bounds memory if a registered hash type is never reported for some block. MaxBufferedBlocks uint - // The number of HashLog entries to retain on disk. + // The number of HashLog entries to retain on disk. Zero disables block-count retention (the disk-size + // cap is then the only bound). BlocksToRetain uint - // The size log files are allowed to get before we close one and open another. + // The size log files are allowed to get before we close one and open another. Must be greater than 0. TargetFileSize uint // A backstop against runaway disk growth. When the total size of sealed log files exceeds this // value, the oldest sealed files are deleted until it no longer does, even if that means retaining - // fewer than BlocksToRetain blocks. + // fewer than BlocksToRetain blocks. Zero disables the disk-size cap (block-count retention is then the + // only bound). MaxDiskSize uint } @@ -89,9 +93,8 @@ func (c *HashLoggerConfig) Validate() error { if c.Version == "" { return fmt.Errorf("version is required") } - if c.MaxDiskSize == 0 { - return fmt.Errorf("max disk size must be greater than 0") - } + // MaxDiskSize == 0 is allowed: it disables the disk-size cap (block-count retention is then the only + // bound; if both are disabled the logger grows without bound, which is a deliberate operator choice). if c.MaxBufferedBlocks == 0 { return fmt.Errorf("max buffered blocks must be greater than 0") } diff --git a/sei-db/state_db/sc/hashlog/hash_logger_config_test.go b/sei-db/state_db/sc/hashlog/hash_logger_config_test.go index 1c39606cf7..494a03fedf 100644 --- a/sei-db/state_db/sc/hashlog/hash_logger_config_test.go +++ b/sei-db/state_db/sc/hashlog/hash_logger_config_test.go @@ -20,10 +20,11 @@ func TestConfigValidateEmptyVersion(t *testing.T) { require.ErrorContains(t, c.Validate(), "version is required") } -func TestConfigValidateZeroMaxDiskSize(t *testing.T) { +func TestConfigValidateZeroMaxDiskSizeIsAllowed(t *testing.T) { + // MaxDiskSize == 0 disables the disk-size cap rather than being an error. c := DefaultHashLoggerConfig("/tmp/hashlog", "v1.0.0") c.MaxDiskSize = 0 - require.ErrorContains(t, c.Validate(), "max disk size") + require.NoError(t, c.Validate()) } func TestConfigValidateZeroMaxBufferedBlocks(t *testing.T) { diff --git a/sei-db/state_db/sc/hashlog/hash_logger_impl.go b/sei-db/state_db/sc/hashlog/hash_logger_impl.go index 5b38a3b70c..e46e91b5a6 100644 --- a/sei-db/state_db/sc/hashlog/hash_logger_impl.go +++ b/sei-db/state_db/sc/hashlog/hash_logger_impl.go @@ -25,6 +25,7 @@ type controlMsgKind int const ( ctrlHashReport controlMsgKind = iota // a caller-reported hash for a block ctrlChangesetRequest // a changeset to be hashed on the hasher thread + ctrlColumnChange // add or remove a hash column (sent by Register/UnregisterHashType) ctrlClose // a graceful-shutdown signal (sent by Close) ) @@ -46,10 +47,11 @@ const ( type controlMessage struct { kind controlMsgKind blockNumber uint64 - hashType string // ctrlHashReport: the type being reported + hashType string // ctrlHashReport / ctrlColumnChange: the type being reported/changed hash []byte // ctrlHashReport: the reported hash (may be nil) cs []*proto.NamedChangeSet // ctrlChangesetRequest: the change set to hash - done chan struct{} // ctrlClose: closed once the drain completes + add bool // ctrlColumnChange: true to add the column, false to remove it + done chan struct{} // ctrlColumnChange / ctrlClose: closed once the loop has applied the message } // A change set dispatched from the control loop to the hasher. @@ -64,9 +66,15 @@ type hashResult struct { hash []byte } -// A message destined for the writer: a block to append to the current file. +// A message destined for the writer: either a block to append to the current file, or (when rotate is +// true) a directive to seal the current file and open a fresh one with the given columns. The control +// loop sends complete blocks ahead of a rotate so they land in the file whose header matches their +// columns; messages are processed in FIFO order so blocks before a column change go to the old file and +// blocks after go to the new one. type writerMessage struct { - log *HashLog + log *HashLog + rotate bool + hashTypes []string // rotate: the column set (header) for the new file } // Bookkeeping for a sealed hash log file (owned by the writer goroutine). @@ -86,10 +94,14 @@ type hashLoggerImpl struct { version string // The ordered set of hash columns recorded per block; the changeset column is prepended when changeset hashing is - // enabled. + // enabled. Mutated only by the control loop (handling a ctrlColumnChange), so the loop reads len(hashTypes) for + // block completion without synchronization. Register/UnregisterHashType change it through that message. hashTypes []string - // The membership set over hashTypes, for O(1) validation of caller-supplied hash types in ReportHash. + // The membership set over hashTypes, for O(1) validation of caller-supplied hash types in ReportHash. Written + // only by the control loop (handling ctrlColumnChange) and read by the caller in Register/Unregister/ReportHash. + // These callers are serialized (Register/Unregister block on the loop's ack via the done channel, establishing + // happens-before), so the read is race-free as long as callers do not invoke the API concurrently. hashTypeSet map[string]struct{} // When true, changeset hashing is disabled: no hasher thread, ReportChangeset is a no-op, and no changeset column is @@ -338,6 +350,60 @@ func (h *hashLoggerImpl) scanDirectory() error { return nil } +// RegisterHashType adds a caller-reported hash column. May be called at any time, including after blocks +// have been logged: the logger flushes complete blocks to the current file, seals it, and opens a fresh +// file whose header includes the new column, so every file's header matches its rows. Registering a type +// that is already present is a no-op (no rotation). See the HashLogger interface for the full contract. +func (h *hashLoggerImpl) RegisterHashType(hashType string) error { + if !h.changesetHashingDisabled && hashType == ChangesetHashType { + return fmt.Errorf("hash type %q is reserved for the logger-computed changeset column", hashType) + } + if !legalHashTypeRegex.MatchString(hashType) { + return fmt.Errorf("hash type %q contains illegal characters (must match %s)", + hashType, legalHashTypeRegex.String()) + } + if _, ok := h.hashTypeSet[hashType]; ok { + return nil // already registered; idempotent no-op (no rotation) + } + return h.sendColumnChange(hashType, true) +} + +// UnregisterHashType removes a caller-reported hash column, rotating to a fresh file whose header omits +// it (same flush/seal/open sequence as RegisterHashType). Removing a type that is not present is a no-op. +// The reserved changeset column cannot be removed. +func (h *hashLoggerImpl) UnregisterHashType(hashType string) error { + if !h.changesetHashingDisabled && hashType == ChangesetHashType { + return fmt.Errorf("hash type %q is the logger-computed changeset column and cannot be removed", hashType) + } + if _, ok := h.hashTypeSet[hashType]; !ok { + return nil // not registered; idempotent no-op (no rotation) + } + return h.sendColumnChange(hashType, false) +} + +// sendColumnChange forwards a column add/remove to the control loop and waits for it to be applied (the +// loop flushes/seals/rotates and updates hashTypes/hashTypeSet before acking). The synchronous handshake +// guarantees that a subsequent ReportHash for the new column is accepted, and establishes happens-before +// for the caller's later reads of hashTypeSet. If the logger is shutting down before the change is +// applied, it returns the relevant context error so the caller knows the registration did not land. +func (h *hashLoggerImpl) sendColumnChange(hashType string, add bool) error { + if h.closed.Load() { + return fmt.Errorf("hash logger is closed") + } + done := make(chan struct{}) + select { + case h.controlChan <- controlMessage{kind: ctrlColumnChange, hashType: hashType, add: add, done: done}: + select { + case <-done: + return nil + case <-h.ctx.Done(): + return h.ctx.Err() + } + case <-h.senderCtx.Done(): + return h.senderCtx.Err() + } +} + func (h *hashLoggerImpl) ReportChangeset(blockNumber uint64, cs []*proto.NamedChangeSet) { // Calling Report* after Close() violates the contract; fail fast (no-op) rather than risk a send on a // closed channel. @@ -483,7 +549,48 @@ func (h *hashLoggerImpl) handleControlMessage(msg controlMessage) { h.handleHashReport(msg.blockNumber, msg.hashType, msg.hash) case ctrlChangesetRequest: h.handleChangesetRequest(msg.blockNumber, msg.cs) + case ctrlColumnChange: + h.handleColumnChange(msg.hashType, msg.add) + close(msg.done) + } +} + +// handleColumnChange adds or removes a hash column and rotates to a fresh file whose header reflects the +// new column set. Complete blocks are flushed to the current file first so they keep the header that +// matches their columns; the rotation directive is then enqueued behind them, and subsequent blocks land +// in the new file. Any still-incomplete buffered block carries over and is written to the new file once +// complete (callers are expected to change columns at block boundaries, where nothing is buffered). +func (h *hashLoggerImpl) handleColumnChange(hashType string, add bool) { + _, present := h.hashTypeSet[hashType] + if add == present { + return // already in the desired state; nothing to do (defensive — callers pre-check) + } + + // Flush everything that is complete under the current columns to the current file. + h.drainComplete() + + // Always build a fresh slice rather than appending in place: the writer may still hold the current + // hashTypes backing array (it became the initial file's header), so mutating it would race. + if add { + updated := make([]string, len(h.hashTypes), len(h.hashTypes)+1) + copy(updated, h.hashTypes) + h.hashTypes = append(updated, hashType) + h.hashTypeSet[hashType] = struct{}{} + } else { + delete(h.hashTypeSet, hashType) + remaining := make([]string, 0, len(h.hashTypes)-1) + for _, t := range h.hashTypes { + if t != hashType { + remaining = append(remaining, t) + } + } + h.hashTypes = remaining } + + // Direct the writer to seal the current file and open a fresh one with the new header. Pass a copy so a + // later column change cannot mutate the slice the writer holds. + newColumns := append([]string(nil), h.hashTypes...) + h.blockingSendToWriter(writerMessage{rotate: true, hashTypes: newColumns}) } // handleHashReport records a caller-reported hash, discarding it if the block has already been flushed. @@ -638,6 +745,14 @@ func (h *hashLoggerImpl) writer() { } return } + if msg.rotate { + // Column change: seal the current file and open a fresh one with the new header. + if err := h.rotateToColumns(msg.hashTypes); err != nil { + h.fail(err) + return + } + continue + } if err := h.handleWrite(msg.log); err != nil { h.fail(err) return @@ -654,20 +769,33 @@ func (h *hashLoggerImpl) handleWrite(log *HashLog) error { h.latestBlock = log.BlockNumber } if h.mutableFile.size >= h.targetFileSize { - if err := h.rotate(); err != nil { + // Size-based rotation keeps the current column set (the new file has the same header). + if err := h.rotateToColumns(h.mutableFile.hashTypes); err != nil { return err } } return nil } -// rotate seals the current mutable file, records its bookkeeping, opens a fresh mutable file, and runs GC. -func (h *hashLoggerImpl) rotate() error { - if err := h.recordSealedFile(); err != nil { +// rotateToColumns seals the current mutable file, records its bookkeeping, opens a fresh mutable file +// with the given columns as its header, and runs GC. An empty current file (no blocks written) is +// removed by recordSealedFile rather than sealed, and its file index is reused for the new file rather +// than burned — so a burst of column changes between blocks (e.g. the startup registration of every +// category) leaves neither orphan files nor index gaps behind. The column set is owned by the writer +// (mutableFile.hashTypes); the control loop hands new columns in via the rotate message, so the two +// goroutines never share the slice. +func (h *hashLoggerImpl) rotateToColumns(columns []string) error { + hadBlocks, err := h.recordSealedFile() + if err != nil { return err } - h.mutableLogFileIndex++ - newFile, err := newHashLogFile(h.directory, h.mutableLogFileIndex, h.version, h.hashTypes) + // Only consume the index if the sealed file actually held blocks. An unwritten file was just removed + // and recorded nothing, so reuse its index. The reused index is always > every sealed index (it is + // the current mutable index), so this never collides with an existing sealed file. + if hadBlocks { + h.mutableLogFileIndex++ + } + newFile, err := newHashLogFile(h.directory, h.mutableLogFileIndex, h.version, columns) if err != nil { return fmt.Errorf("failed to open new mutable hash log file: %w", err) } @@ -677,7 +805,7 @@ func (h *hashLoggerImpl) rotate() error { } func (h *hashLoggerImpl) sealMutableAndGC() error { - if err := h.recordSealedFile(); err != nil { + if _, err := h.recordSealedFile(); err != nil { return err } h.runGC() @@ -685,8 +813,9 @@ func (h *hashLoggerImpl) sealMutableAndGC() error { } // recordSealedFile seals the current mutable file and, if it held any blocks, adds it to the sealed-file -// bookkeeping. An empty file is removed by close() and leaves no bookkeeping behind. -func (h *hashLoggerImpl) recordSealedFile() error { +// bookkeeping. An empty file is removed by close() and leaves no bookkeeping behind. The returned bool +// reports whether the file held any blocks (and thus consumed its file index). +func (h *hashLoggerImpl) recordSealedFile() (bool, error) { hadBlocks := h.mutableFile.hasBlocks idx := h.mutableFile.index first := h.mutableFile.firstBlockIndex @@ -694,10 +823,10 @@ func (h *hashLoggerImpl) recordSealedFile() error { size := h.mutableFile.size if err := h.mutableFile.close(); err != nil { - return fmt.Errorf("failed to seal hash log file: %w", err) + return false, fmt.Errorf("failed to seal hash log file: %w", err) } if !hadBlocks { - return nil + return false, nil } h.sealedFiles[idx] = &sealedFileInfo{ name: sealedFileName(idx, first, last, h.version), @@ -706,7 +835,7 @@ func (h *hashLoggerImpl) recordSealedFile() error { size: size, } h.currentDiskSpaceUsed += size - return nil + return true, nil } // runGC deletes the oldest sealed files while either the block-count retention window or the disk-size cap is @@ -723,9 +852,10 @@ func (h *hashLoggerImpl) runGC() { // Retain exactly the most-recent blocksToRetain blocks: a file is over the window once its newest block // is more than blocksToRetain-1 behind the latest. Written as an addition to avoid unsigned underflow - // when latestBlock < blocksToRetain (in which case nothing is over the window). - overBlockRetention := info.lastBlock+h.blocksToRetain <= h.latestBlock - overSizeCap := h.currentDiskSpaceUsed > h.maxDiskSize + // when latestBlock < blocksToRetain (in which case nothing is over the window). A zero limit disables + // that dimension entirely (no block-count window / no disk cap). + overBlockRetention := h.blocksToRetain > 0 && info.lastBlock+h.blocksToRetain <= h.latestBlock + overSizeCap := h.maxDiskSize > 0 && h.currentDiskSpaceUsed > h.maxDiskSize if !overBlockRetention && !overSizeCap { break } diff --git a/sei-db/state_db/sc/hashlog/hash_logger_impl_test.go b/sei-db/state_db/sc/hashlog/hash_logger_impl_test.go index 7374d4dc87..8c169cc97d 100644 --- a/sei-db/state_db/sc/hashlog/hash_logger_impl_test.go +++ b/sei-db/state_db/sc/hashlog/hash_logger_impl_test.go @@ -334,6 +334,56 @@ func TestImplGCHonorsMaxDiskSize(t *testing.T) { require.Len(t, kept, 1, "the most recent block should be retained") } +func TestImplGCDisabledWhenLimitsZero(t *testing.T) { + dir := t.TempDir() + config := testConfig(dir) + config.TargetFileSize = 1 // one block per file: 20 sealed files + config.BlocksToRetain = 0 // disabled + config.MaxDiskSize = 0 // disabled + l, err := NewHashLogger(config) + require.NoError(t, err) + + const blocks = 20 + for block := uint64(1); block <= blocks; block++ { + require.NoError(t, l.ReportHash(block, "a", []byte{byte(block)})) + require.NoError(t, l.ReportHash(block, "b", []byte{byte(block)})) + } + require.NoError(t, l.Close()) + + // With both retention dimensions disabled, nothing is garbage collected — even the oldest block survives. + oldest, err := ReadHashForBlock(dir, 1) + require.NoError(t, err) + require.Len(t, oldest, 1, "block 1 must be retained when both GC limits are disabled") + newest, err := ReadHashForBlock(dir, blocks) + require.NoError(t, err) + require.Len(t, newest, 1, "the most recent block must be retained") +} + +func TestImplRotationReusesIndexForUnwrittenFiles(t *testing.T) { + dir := t.TempDir() + l, err := NewHashLogger(testConfig(dir)) // starts with columns "a", "b" + require.NoError(t, err) + + // Register several new columns before any block is written. Each registration rotates the (still + // empty) mutable file; because nothing was written, the index must be reused, not burned. + for _, ht := range []string{"c", "d", "e"} { + require.NoError(t, l.RegisterHashType(ht)) + } + + for _, ht := range []string{"a", "b", "c", "d", "e"} { + require.NoError(t, l.ReportHash(1, ht, []byte{0x01})) + } + require.NoError(t, l.Close()) + + files, err := listArchiveFiles(dir) + require.NoError(t, err) + require.Len(t, files, 1, "the pre-block registration burst must not leave extra files") + parsed, ok := parseFileName(files[0].name) + require.True(t, ok) + require.Equal(t, uint64(0), parsed.index, + "an unwritten file's index must be reused, so the first file with data is index 0") +} + func TestImplResumesAfterReopen(t *testing.T) { dir := t.TempDir() diff --git a/sei-db/state_db/sc/hashlog/noop_hash_logger.go b/sei-db/state_db/sc/hashlog/noop_hash_logger.go index 43c92e4c81..75a99647df 100644 --- a/sei-db/state_db/sc/hashlog/noop_hash_logger.go +++ b/sei-db/state_db/sc/hashlog/noop_hash_logger.go @@ -17,6 +17,16 @@ func (n *noOpHashLogger) ReportChangeset(uint64, []*proto.NamedChangeSet) { // intentional no-op } +func (n *noOpHashLogger) RegisterHashType(string) error { + // intentional no-op + return nil +} + +func (n *noOpHashLogger) UnregisterHashType(string) error { + // intentional no-op + return nil +} + func (n *noOpHashLogger) ReportHash(uint64, string, []byte) error { // intentional no-op return nil diff --git a/sei-db/state_db/sc/hashlog/register_hash_type_test.go b/sei-db/state_db/sc/hashlog/register_hash_type_test.go new file mode 100644 index 0000000000..160640095b --- /dev/null +++ b/sei-db/state_db/sc/hashlog/register_hash_type_test.go @@ -0,0 +1,119 @@ +package hashlog + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// A logger opened with no caller types can have its columns populated via RegisterHashType, and the +// registered columns are recorded. +func TestRegisterHashTypePopulatesColumns(t *testing.T) { + dir := t.TempDir() + config := testConfig(dir) + config.HashTypes = nil // start empty; populate via RegisterHashType + config.DisableChangesetHashing = false // changeset column makes an empty caller set valid (real usage) + l, err := NewHashLogger(config) + require.NoError(t, err) + + require.NoError(t, l.RegisterHashType("a")) + require.NoError(t, l.RegisterHashType("b")) + require.NoError(t, l.RegisterHashType("a")) // idempotent + + require.NoError(t, l.ReportHash(1, "a", []byte{0x01})) + require.NoError(t, l.ReportHash(1, "b", []byte{0x02})) + l.ReportChangeset(1, nil) // complete the changeset column with a nil hash + require.NoError(t, l.Close()) + + logs := readAllLogs(t, dir) + require.Len(t, logs, 1) + require.Equal(t, []byte{0x01}, logs[0].Hashes["a"]) + require.Equal(t, []byte{0x02}, logs[0].Hashes["b"]) +} + +// RegisterHashType rejects the reserved changeset column and illegal names. +func TestRegisterHashTypeRejectsReservedAndIllegal(t *testing.T) { + dir := t.TempDir() + config := testConfig(dir) + config.HashTypes = nil + config.DisableChangesetHashing = false // changeset column is active, so the name is reserved + l, err := NewHashLogger(config) + require.NoError(t, err) + defer func() { require.NoError(t, l.Close()) }() + + require.Error(t, l.RegisterHashType(ChangesetHashType)) + require.Error(t, l.UnregisterHashType(ChangesetHashType)) + require.Error(t, l.RegisterHashType("bad,name")) + require.NoError(t, l.RegisterHashType("memIAVL/mod/bank")) +} + +// Changing the column set after blocks have been logged seals the current file and starts a new one +// whose header reflects the new columns. Blocks logged before and after the change read back correctly. +func TestColumnChangeRotatesFile(t *testing.T) { + dir := t.TempDir() + config := testConfig(dir) // changeset hashing disabled; caller types only + config.HashTypes = []string{"a"} + l, err := NewHashLogger(config) + require.NoError(t, err) + + // Block 1 with only column "a". + require.NoError(t, l.ReportHash(1, "a", []byte{0x01})) + + // Add column "b" mid-run, then log block 2 with both columns. + require.NoError(t, l.RegisterHashType("b")) + require.NoError(t, l.ReportHash(2, "a", []byte{0x02})) + require.NoError(t, l.ReportHash(2, "b", []byte{0x12})) + + // Remove column "a", then log block 3 with only "b". + require.NoError(t, l.UnregisterHashType("a")) + require.NoError(t, l.ReportHash(3, "b", []byte{0x13})) + + require.NoError(t, l.Close()) + + // More than one file should exist (the set changed twice). + files, err := listArchiveFiles(dir) + require.NoError(t, err) + require.Greater(t, len(files), 1, "a column change should have rotated to a new file") + + byBlock := map[uint64]HashLog{} + for _, log := range readAllLogs(t, dir) { + byBlock[log.BlockNumber] = log + } + require.Len(t, byBlock, 3) + require.Equal(t, []byte{0x01}, byBlock[1].Hashes["a"]) + require.Equal(t, []byte{0x02}, byBlock[2].Hashes["a"]) + require.Equal(t, []byte{0x12}, byBlock[2].Hashes["b"]) + require.Equal(t, []byte{0x13}, byBlock[3].Hashes["b"]) + // Block 3's file no longer carries column "a". + _, hasA := byBlock[3].Hashes["a"] + require.False(t, hasA, "column a should be gone from block 3") +} + +// A burst of column registrations before any block is written leaves no orphan files: the empty +// intermediate files are removed as each rotation seals them. +func TestColumnChangeBurstLeavesNoOrphans(t *testing.T) { + dir := t.TempDir() + config := testConfig(dir) + config.HashTypes = nil + config.DisableChangesetHashing = false + l, err := NewHashLogger(config) + require.NoError(t, err) + + for _, name := range []string{"a", "b", "c", "d"} { + require.NoError(t, l.RegisterHashType(name)) + } + require.NoError(t, l.ReportHash(1, "a", []byte{0x01})) + require.NoError(t, l.ReportHash(1, "b", []byte{0x02})) + require.NoError(t, l.ReportHash(1, "c", []byte{0x03})) + require.NoError(t, l.ReportHash(1, "d", []byte{0x04})) + l.ReportChangeset(1, nil) + require.NoError(t, l.Close()) + + files, err := listArchiveFiles(dir) + require.NoError(t, err) + require.Len(t, files, 1, "intermediate empty files should have been removed") + + logs := readAllLogs(t, dir) + require.Len(t, logs, 1) + require.Equal(t, []byte{0x04}, logs[0].Hashes["d"]) +} diff --git a/sei-db/state_db/sc/memiavl/hashlog.go b/sei-db/state_db/sc/memiavl/hashlog.go new file mode 100644 index 0000000000..c6c53f692d --- /dev/null +++ b/sei-db/state_db/sc/memiavl/hashlog.go @@ -0,0 +1,53 @@ +package memiavl + +import ( + "fmt" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" +) + +// memIAVLModHashPrefix is joined with a store/module name to form that module's hash logger category +// (e.g. "memIAVL/mod/bank"). The memIAVL root hash ("memIAVL/root") is a simple-merkle aggregation over +// these per-module hashes and is reported by the cosmos layer (rootmulti), which owns the merkle +// computation; this backend reports only the per-module hashes it computes natively. +const memIAVLModHashPrefix = "memIAVL/mod/" + +// HashCategories returns one category per module currently in the tree. The set is dynamic: it is empty +// on a fresh (genesis) store and grows/shrinks as modules are added/removed, so the overall logged set +// changes over time (handled upstream by reopening the logger when the set changes). +func (cs *CommitStore) HashCategories() []string { + if cs == nil || cs.db == nil { + return nil + } + commitInfo := cs.db.LastCommitInfo() + if commitInfo == nil { + return nil + } + categories := make([]string, 0, len(commitInfo.StoreInfos)) + for _, storeInfo := range commitInfo.StoreInfos { + categories = append(categories, memIAVLModHashPrefix+storeInfo.Name) + } + return categories +} + +// RecordHashes reports each module's committed root hash for blockNumber. Intended to be called right +// after Commit, when LastCommitInfo reflects the just-committed version. +func (cs *CommitStore) RecordHashes(hl hashlog.HashLogger, blockNumber uint64) error { + if cs == nil || cs.db == nil { + return nil + } + commitInfo := cs.db.LastCommitInfo() + if commitInfo == nil { + return nil + } + for _, storeInfo := range commitInfo.StoreInfos { + category := memIAVLModHashPrefix + storeInfo.Name + // Copy: the logger retains the slice and reads it asynchronously, while the next commit replaces + // the commit info's hashes. + hash := append([]byte(nil), storeInfo.CommitId.Hash...) + if err := hl.ReportHash(blockNumber, category, hash); err != nil { + return fmt.Errorf("failed to report memiavl mod hash %q: %w", category, err) + } + } + return nil +} diff --git a/sei-db/state_db/sc/memiavl/hashlog_test.go b/sei-db/state_db/sc/memiavl/hashlog_test.go new file mode 100644 index 0000000000..55627a932d --- /dev/null +++ b/sei-db/state_db/sc/memiavl/hashlog_test.go @@ -0,0 +1,69 @@ +package memiavl + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +// captureLogger is a HashLogger test double that records registered categories and reported hashes. +type captureLogger struct { + registered map[string]struct{} + hashes map[string][]byte +} + +func newCaptureLogger() *captureLogger { + return &captureLogger{registered: map[string]struct{}{}, hashes: map[string][]byte{}} +} + +func (c *captureLogger) RegisterHashType(hashType string) error { + c.registered[hashType] = struct{}{} + return nil +} + +func (c *captureLogger) UnregisterHashType(hashType string) error { + delete(c.registered, hashType) + return nil +} + +func (c *captureLogger) ReportHash(_ uint64, hashType string, hash []byte) error { + c.hashes[hashType] = hash + return nil +} + +func (c *captureLogger) ReportChangeset(uint64, []*proto.NamedChangeSet) {} + +func (c *captureLogger) Close() error { return nil } + +func TestMemIAVLHashReporting(t *testing.T) { + cs := setupCS(t) // stores "test" and "other"; "test" has committed data + + // One category per tree (no root — that is owned by the cosmos layer). + require.ElementsMatch(t, []string{"memIAVL/mod/test", "memIAVL/mod/other"}, cs.HashCategories()) + + logger := newCaptureLogger() + for _, category := range cs.HashCategories() { + require.NoError(t, logger.RegisterHashType(category)) + } + require.Len(t, logger.registered, 2) + + require.NoError(t, cs.RecordHashes(logger, 1)) + + // Each module's reported hash matches its commit info store hash. + for _, storeInfo := range cs.LastCommitInfo().StoreInfos { + reported, ok := logger.hashes["memIAVL/mod/"+storeInfo.Name] + require.True(t, ok, "expected a hash for module %q", storeInfo.Name) + require.Equal(t, storeInfo.CommitId.Hash, reported) + } +} + +// A store that is not loaded reports no categories and records nothing, without panicking. +func TestMemIAVLHashReportingNilSafe(t *testing.T) { + var cs *CommitStore + require.Nil(t, cs.HashCategories()) + logger := newCaptureLogger() + require.NoError(t, cs.RecordHashes(logger, 1)) + require.Empty(t, logger.hashes) +}