Skip to content

[pull] memtable_as_log_index from topling:memtable_as_log_index #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pull merged 1 commit into from
Aug 1, 2026
Merged

[pull] memtable_as_log_index from topling:memtable_as_log_index #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions .github/scripts/run_dcompact_bench.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# PREFIX Topling install prefix (bin/, lib/, toplingdb-conf/)
# YAML Path to dcompact CI yaml (default: PREFIX/toplingdb-conf/...)
# NUM db_bench -num (default 1000000 local; CI overrides)
# VALUE_SIZE db_bench -value_size (default 50)
# WRITE_BUFFER_SIZE bytes; if set, rewrite yaml temp copy write_buffer_size
# LOGDIR_BASE Parent of per-engine log dirs (default: logs)
# CPU_QUOTA write-side db_bench systemd CPUQuota (default 50%)
Expand All @@ -35,6 +36,7 @@ set -euo pipefail
PREFIX="${PREFIX:?PREFIX (Topling install root) required}"
YAML="${YAML:-$PREFIX/toplingdb-conf/db_bench_enterprise_dcompact_ci.yaml}"
NUM="${NUM:-1000000}"
VALUE_SIZE="${VALUE_SIZE:-50}"
LOGDIR_BASE="${LOGDIR_BASE:-logs}"
CPU_QUOTA="${CPU_QUOTA:-50%}"
L1_WRITER="${L1_WRITER:-simple}"
Expand Down Expand Up @@ -267,7 +269,7 @@ run_engine_suite() {
-json "$yaml"
-num="$NUM"
-key_size=8
-value_size=15
-value_size="$VALUE_SIZE"
-batch_size=1000
-benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom
-enable_zero_copy
Expand All @@ -290,7 +292,7 @@ run_engine_suite() {
-json "$yaml"
-num="$NUM"
-key_size=8
-value_size=15
-value_size="$VALUE_SIZE"
-batch_size=1000
-benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom
-enable_zero_copy
Expand Down Expand Up @@ -327,4 +329,4 @@ for eng in $ENGINES; do
run_engine_suite "$eng"
done

echo "run_dcompact_bench.sh OK (NUM=${NUM} CPU_QUOTA=${CPU_QUOTA} ENGINES=${ENGINES})"
echo "run_dcompact_bench.sh OK (NUM=${NUM} VALUE_SIZE=${VALUE_SIZE} CPU_QUOTA=${CPU_QUOTA} ENGINES=${ENGINES})"
10 changes: 6 additions & 4 deletions .github/scripts/run_local_simple_top_pages.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#
# Env:
# NUM=20000000 key count (CI uses 100000000; local shm often needs smaller)
# VALUE_SIZE=50 db_bench -value_size (align with db_bench-run.yml)
# RUN_MEMTABLE=0|1 run memtablerep_bench after topling suite (default 0)
# RUN_ID=local-simpletop
# Sampler: sample_statm_fdcache (same as CI db_bench-run).
Expand All @@ -30,6 +31,7 @@ DB_PATH=/dev/shm/db_bench_enterprise
PAGES_EMIT="$LOCAL/_pages"
PAGES_SITE="$LOCAL/site_pages"
NUM="${NUM:-20000000}"
VALUE_SIZE="${VALUE_SIZE:-50}"
RUN_MEMTABLE="${RUN_MEMTABLE:-0}"
RUN_ID="${RUN_ID:-local-simpletop}"

Expand Down Expand Up @@ -57,7 +59,7 @@ prepare_yaml() {
}
mkdir -p "$(dirname "$YAML_DZ10")"
cp -a "$YAML_BASE" "$YAML_DZ10"
# Align with db_bench-run.yml: enable DictZip for value_size=15.
# Align with db_bench-run.yml: enable DictZip for small CI values (default 50).
sed -i 's/^\([[:space:]]*minDictZipValueSize:[[:space:]]*\)3000/\110/' "$YAML_DZ10"
grep -E 'minDictZipValueSize:[[:space:]]*10$' "$YAML_DZ10"
grep -E 'class: SimpleTopTable' "$YAML_BASE"
Expand Down Expand Up @@ -149,7 +151,7 @@ run_topling_suite() {
-json "$yaml"
-num="$NUM"
-key_size=8
-value_size=15
-value_size="$VALUE_SIZE"
-batch_size=1000
-benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom
-enable_zero_copy
Expand All @@ -167,7 +169,7 @@ run_topling_suite() {
-json "$yaml"
-num="$NUM"
-key_size=8
-value_size=15
-value_size="$VALUE_SIZE"
-batch_size=1000
-benchmarks=nextwithkey,nextwithkey,nextwithkey,readseq,readseq,readseq
-scan_omit_key -scan_omit_value
Expand All @@ -189,7 +191,7 @@ run_topling_suite() {
-json "$yaml"
-num="$NUM"
-key_size=8
-value_size=15
-value_size="$VALUE_SIZE"
-batch_size=1000
-benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom
-enable_zero_copy
Expand Down
4 changes: 3 additions & 1 deletion .github/scripts/run_rocksdb_cs_bench.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
# PREFIX rocksdb engine prefix (bin/, lib/)
# LOGDIR log directory
# NUM db_bench -num
# VALUE_SIZE db_bench -value_size (default 50)
# CPU_QUOTA systemd CPUQuota (default 50%)
# CACHE_SIZE -cache_size bytes
# WRITE_BUFFER_SIZE bytes (default 67108864)
Expand All @@ -18,6 +19,7 @@ set -euo pipefail
PREFIX="${PREFIX:?PREFIX required}"
LOGDIR="${LOGDIR:?LOGDIR required}"
NUM="${NUM:?NUM required}"
VALUE_SIZE="${VALUE_SIZE:-50}"
CPU_QUOTA="${CPU_QUOTA:-50%}"
CACHE_SIZE="${CACHE_SIZE:?CACHE_SIZE required}"
WRITE_BUFFER_SIZE="${WRITE_BUFFER_SIZE:-67108864}"
Expand Down Expand Up @@ -137,7 +139,7 @@ run_suite() {
rm -rf "$DB_PATH"
mkdir -p "$DB_PATH"
local args=(
-db="$DB_PATH" -num="$NUM" -key_size=8 -value_size=15 -batch_size=1000
-db="$DB_PATH" -num="$NUM" -key_size=8 -value_size="$VALUE_SIZE" -batch_size=1000
-cache_size="$CACHE_SIZE"
-write_buffer_size="$WRITE_BUFFER_SIZE"
-compaction_spool_dir="$SPOOL_DIR"
Expand Down
23 changes: 15 additions & 8 deletions .github/workflows/db_bench-avx512-run.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ on:
description: Build workflow run id whose artifact to download
required: true
type: string
value_size:
description: db_bench -value_size bytes (default 50)
required: false
default: "50"
type: string

concurrency:
group: pages-deploy
Expand All @@ -30,6 +35,7 @@ jobs:
env:
VARIANT: avx512
ARTIFACT_NAME: db_bench-bin-avx512
VALUE_SIZE: ${{ inputs.value_size || '50' }}
steps:
- name: Checkout build commit
uses: actions/checkout@v4
Expand Down Expand Up @@ -88,7 +94,8 @@ jobs:
# CI-only: shrink write_buffer without touching rockside submodule.
sed -i 's/^\([[:space:]]*write_buffer_size:[[:space:]]*\)2G/\1512M/' "$YAML_BASE"
grep -E 'write_buffer_size:[[:space:]]*512M' "$YAML_BASE"
# Parallel config: enable DictZip for value_size=15 (default yaml is 3000).
# Parallel config: enable DictZip for small values (default yaml is 3000;
# CI -value_size defaults to 50, so minDictZipValueSize=10 applies).
YAML_DZ10="$TOPLING/toplingdb-conf/db_bench_enterprise_dictzip10.yaml"
cp -a "$YAML_BASE" "$YAML_DZ10"
sed -i 's/^\([[:space:]]*minDictZipValueSize:[[:space:]]*\)3000/\110/' "$YAML_DZ10"
Expand Down Expand Up @@ -135,7 +142,7 @@ jobs:
-json "$yaml"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom
-enable_zero_copy
Expand All @@ -148,7 +155,7 @@ jobs:
-json "$yaml"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-benchmarks=nextwithkey,nextwithkey,nextwithkey,readseq,readseq,readseq
-scan_omit_key -scan_omit_value
Expand All @@ -170,7 +177,7 @@ jobs:
-json "$yaml"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom
-enable_zero_copy
Expand Down Expand Up @@ -268,7 +275,7 @@ jobs:
-db="$DB_PATH"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-cache_size="$ROCKSDB_CACHE_SIZE"
-benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom
Expand All @@ -288,7 +295,7 @@ jobs:
-db="$DB_PATH"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-cache_size="$ROCKSDB_CACHE_SIZE"
-benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom
Expand Down Expand Up @@ -353,7 +360,7 @@ jobs:
-db="$DB_PATH"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-cache_size="$ROCKSDB_CACHE_SIZE"
-benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom
Expand All @@ -372,7 +379,7 @@ jobs:
-db="$DB_PATH"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-cache_size="$ROCKSDB_CACHE_SIZE"
-benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/db_bench-dcompact-run.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ on:
- light_zip
- zip
- bb
value_size:
description: db_bench -value_size bytes (default 50)
required: false
default: "50"
type: string

# Must match plain run — serialize gh-pages pushes.
concurrency:
Expand All @@ -55,6 +60,7 @@ jobs:
NUM: ${{ inputs.num }}
CPU_QUOTA: ${{ inputs.cpu_quota }}
L1_WRITER: ${{ inputs.l1_writer }}
VALUE_SIZE: ${{ inputs.value_size || '50' }}
steps:
- name: Checkout build commit
uses: actions/checkout@v4
Expand Down
23 changes: 15 additions & 8 deletions .github/workflows/db_bench-run.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ on:
- light_zip
- zip
- bb
value_size:
description: db_bench -value_size bytes (default 50)
required: false
default: "50"
type: string

concurrency:
group: pages-deploy
Expand All @@ -49,6 +54,7 @@ jobs:
# workflow_run has no inputs; defaults match dcompact-run.
CPU_QUOTA: ${{ inputs.cpu_quota || '50%' }}
L1_WRITER: ${{ inputs.l1_writer || 'simple' }}
VALUE_SIZE: ${{ inputs.value_size || '50' }}
steps:
- name: Checkout build commit
uses: actions/checkout@v4
Expand Down Expand Up @@ -126,7 +132,8 @@ jobs:
echo "FAIL: plain yaml still has compaction_executor_factory" >&2
exit 1
fi
# Parallel config: enable DictZip for value_size=15 (default yaml is 3000).
# Parallel config: enable DictZip for small values (default yaml is 3000;
# CI -value_size defaults to 50, so minDictZipValueSize=10 applies).
YAML_DZ10="$TOPLING/toplingdb-conf/db_bench_enterprise_dictzip10.yaml"
cp -a "$YAML_BASE" "$YAML_DZ10"
sed -i 's/^\([[:space:]]*minDictZipValueSize:[[:space:]]*\)3000/\110/' "$YAML_DZ10"
Expand Down Expand Up @@ -178,7 +185,7 @@ jobs:
-json "$yaml"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom
-enable_zero_copy
Expand All @@ -194,7 +201,7 @@ jobs:
-json "$yaml"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-benchmarks=nextwithkey,nextwithkey,nextwithkey,readseq,readseq,readseq
-scan_omit_key -scan_omit_value
Expand All @@ -217,7 +224,7 @@ jobs:
-json "$yaml"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom
-enable_zero_copy
Expand Down Expand Up @@ -326,7 +333,7 @@ jobs:
-db="$DB_PATH"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-cache_size="$ROCKSDB_CACHE_SIZE"
-benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom
Expand All @@ -350,7 +357,7 @@ jobs:
-db="$DB_PATH"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-cache_size="$ROCKSDB_CACHE_SIZE"
-benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom
Expand Down Expand Up @@ -424,7 +431,7 @@ jobs:
-db="$DB_PATH"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-cache_size="$ROCKSDB_CACHE_SIZE"
-benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom
Expand All @@ -446,7 +453,7 @@ jobs:
-db="$DB_PATH"
-num=100000000
-key_size=8
-value_size=15
-value_size="${VALUE_SIZE}"
-batch_size=1000
-cache_size="$ROCKSDB_CACHE_SIZE"
-benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom
Expand Down
Loading