From c73391793087527491020097d8c7f136353f0abe Mon Sep 17 00:00:00 2001 From: leipeng Date: Sun, 2 Aug 2026 00:06:25 +0800 Subject: [PATCH] ci(bench): default value_size to 50 and expose as workflow input Make db_bench -value_size configurable on plain/dcompact/avx512 run workflows (default 50) and thread VALUE_SIZE through local/CI runners. --- .github/scripts/run_dcompact_bench.sh | 8 ++++--- .github/scripts/run_local_simple_top_pages.sh | 10 ++++---- .github/scripts/run_rocksdb_cs_bench.sh | 4 +++- .github/workflows/db_bench-avx512-run.yml | 23 ++++++++++++------- .github/workflows/db_bench-dcompact-run.yml | 6 +++++ .github/workflows/db_bench-run.yml | 23 ++++++++++++------- 6 files changed, 50 insertions(+), 24 deletions(-) diff --git a/.github/scripts/run_dcompact_bench.sh b/.github/scripts/run_dcompact_bench.sh index 897389fc0..422c00edd 100644 --- a/.github/scripts/run_dcompact_bench.sh +++ b/.github/scripts/run_dcompact_bench.sh @@ -12,6 +12,7 @@ # PREFIX Topling install prefix (bin/, lib/, toplingdb-conf/) # YAML Path to dcompact CI yaml (default: PREFIX/toplingdb-conf/...) # NUM db_bench -num (default 1000000 local; CI overrides) +# VALUE_SIZE db_bench -value_size (default 50) # WRITE_BUFFER_SIZE bytes; if set, rewrite yaml temp copy write_buffer_size # LOGDIR_BASE Parent of per-engine log dirs (default: logs) # CPU_QUOTA write-side db_bench systemd CPUQuota (default 50%) @@ -35,6 +36,7 @@ set -euo pipefail PREFIX="${PREFIX:?PREFIX (Topling install root) required}" YAML="${YAML:-$PREFIX/toplingdb-conf/db_bench_enterprise_dcompact_ci.yaml}" NUM="${NUM:-1000000}" +VALUE_SIZE="${VALUE_SIZE:-50}" LOGDIR_BASE="${LOGDIR_BASE:-logs}" CPU_QUOTA="${CPU_QUOTA:-50%}" L1_WRITER="${L1_WRITER:-simple}" @@ -267,7 +269,7 @@ run_engine_suite() { -json "$yaml" -num="$NUM" -key_size=8 - -value_size=15 + -value_size="$VALUE_SIZE" -batch_size=1000 -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom -enable_zero_copy @@ -290,7 +292,7 @@ run_engine_suite() { -json "$yaml" -num="$NUM" -key_size=8 - -value_size=15 + -value_size="$VALUE_SIZE" -batch_size=1000 -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom -enable_zero_copy @@ -327,4 +329,4 @@ for eng in $ENGINES; do run_engine_suite "$eng" done -echo "run_dcompact_bench.sh OK (NUM=${NUM} CPU_QUOTA=${CPU_QUOTA} ENGINES=${ENGINES})" +echo "run_dcompact_bench.sh OK (NUM=${NUM} VALUE_SIZE=${VALUE_SIZE} CPU_QUOTA=${CPU_QUOTA} ENGINES=${ENGINES})" diff --git a/.github/scripts/run_local_simple_top_pages.sh b/.github/scripts/run_local_simple_top_pages.sh index a08881f1f..f222c679e 100755 --- a/.github/scripts/run_local_simple_top_pages.sh +++ b/.github/scripts/run_local_simple_top_pages.sh @@ -16,6 +16,7 @@ # # Env: # NUM=20000000 key count (CI uses 100000000; local shm often needs smaller) +# VALUE_SIZE=50 db_bench -value_size (align with db_bench-run.yml) # RUN_MEMTABLE=0|1 run memtablerep_bench after topling suite (default 0) # RUN_ID=local-simpletop # Sampler: sample_statm_fdcache (same as CI db_bench-run). @@ -30,6 +31,7 @@ DB_PATH=/dev/shm/db_bench_enterprise PAGES_EMIT="$LOCAL/_pages" PAGES_SITE="$LOCAL/site_pages" NUM="${NUM:-20000000}" +VALUE_SIZE="${VALUE_SIZE:-50}" RUN_MEMTABLE="${RUN_MEMTABLE:-0}" RUN_ID="${RUN_ID:-local-simpletop}" @@ -57,7 +59,7 @@ prepare_yaml() { } mkdir -p "$(dirname "$YAML_DZ10")" cp -a "$YAML_BASE" "$YAML_DZ10" - # Align with db_bench-run.yml: enable DictZip for value_size=15. + # Align with db_bench-run.yml: enable DictZip for small CI values (default 50). sed -i 's/^\([[:space:]]*minDictZipValueSize:[[:space:]]*\)3000/\110/' "$YAML_DZ10" grep -E 'minDictZipValueSize:[[:space:]]*10$' "$YAML_DZ10" grep -E 'class: SimpleTopTable' "$YAML_BASE" @@ -149,7 +151,7 @@ run_topling_suite() { -json "$yaml" -num="$NUM" -key_size=8 - -value_size=15 + -value_size="$VALUE_SIZE" -batch_size=1000 -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom -enable_zero_copy @@ -167,7 +169,7 @@ run_topling_suite() { -json "$yaml" -num="$NUM" -key_size=8 - -value_size=15 + -value_size="$VALUE_SIZE" -batch_size=1000 -benchmarks=nextwithkey,nextwithkey,nextwithkey,readseq,readseq,readseq -scan_omit_key -scan_omit_value @@ -189,7 +191,7 @@ run_topling_suite() { -json "$yaml" -num="$NUM" -key_size=8 - -value_size=15 + -value_size="$VALUE_SIZE" -batch_size=1000 -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom -enable_zero_copy diff --git a/.github/scripts/run_rocksdb_cs_bench.sh b/.github/scripts/run_rocksdb_cs_bench.sh index 204ea181d..323066626 100644 --- a/.github/scripts/run_rocksdb_cs_bench.sh +++ b/.github/scripts/run_rocksdb_cs_bench.sh @@ -6,6 +6,7 @@ # PREFIX rocksdb engine prefix (bin/, lib/) # LOGDIR log directory # NUM db_bench -num +# VALUE_SIZE db_bench -value_size (default 50) # CPU_QUOTA systemd CPUQuota (default 50%) # CACHE_SIZE -cache_size bytes # WRITE_BUFFER_SIZE bytes (default 67108864) @@ -18,6 +19,7 @@ set -euo pipefail PREFIX="${PREFIX:?PREFIX required}" LOGDIR="${LOGDIR:?LOGDIR required}" NUM="${NUM:?NUM required}" +VALUE_SIZE="${VALUE_SIZE:-50}" CPU_QUOTA="${CPU_QUOTA:-50%}" CACHE_SIZE="${CACHE_SIZE:?CACHE_SIZE required}" WRITE_BUFFER_SIZE="${WRITE_BUFFER_SIZE:-67108864}" @@ -137,7 +139,7 @@ run_suite() { rm -rf "$DB_PATH" mkdir -p "$DB_PATH" local args=( - -db="$DB_PATH" -num="$NUM" -key_size=8 -value_size=15 -batch_size=1000 + -db="$DB_PATH" -num="$NUM" -key_size=8 -value_size="$VALUE_SIZE" -batch_size=1000 -cache_size="$CACHE_SIZE" -write_buffer_size="$WRITE_BUFFER_SIZE" -compaction_spool_dir="$SPOOL_DIR" diff --git a/.github/workflows/db_bench-avx512-run.yml b/.github/workflows/db_bench-avx512-run.yml index 26ef2312a..f54ece640 100644 --- a/.github/workflows/db_bench-avx512-run.yml +++ b/.github/workflows/db_bench-avx512-run.yml @@ -12,6 +12,11 @@ on: description: Build workflow run id whose artifact to download required: true type: string + value_size: + description: db_bench -value_size bytes (default 50) + required: false + default: "50" + type: string concurrency: group: pages-deploy @@ -30,6 +35,7 @@ jobs: env: VARIANT: avx512 ARTIFACT_NAME: db_bench-bin-avx512 + VALUE_SIZE: ${{ inputs.value_size || '50' }} steps: - name: Checkout build commit uses: actions/checkout@v4 @@ -88,7 +94,8 @@ jobs: # CI-only: shrink write_buffer without touching rockside submodule. sed -i 's/^\([[:space:]]*write_buffer_size:[[:space:]]*\)2G/\1512M/' "$YAML_BASE" grep -E 'write_buffer_size:[[:space:]]*512M' "$YAML_BASE" - # Parallel config: enable DictZip for value_size=15 (default yaml is 3000). + # Parallel config: enable DictZip for small values (default yaml is 3000; + # CI -value_size defaults to 50, so minDictZipValueSize=10 applies). YAML_DZ10="$TOPLING/toplingdb-conf/db_bench_enterprise_dictzip10.yaml" cp -a "$YAML_BASE" "$YAML_DZ10" sed -i 's/^\([[:space:]]*minDictZipValueSize:[[:space:]]*\)3000/\110/' "$YAML_DZ10" @@ -135,7 +142,7 @@ jobs: -json "$yaml" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom -enable_zero_copy @@ -148,7 +155,7 @@ jobs: -json "$yaml" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -benchmarks=nextwithkey,nextwithkey,nextwithkey,readseq,readseq,readseq -scan_omit_key -scan_omit_value @@ -170,7 +177,7 @@ jobs: -json "$yaml" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom -enable_zero_copy @@ -268,7 +275,7 @@ jobs: -db="$DB_PATH" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -cache_size="$ROCKSDB_CACHE_SIZE" -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom @@ -288,7 +295,7 @@ jobs: -db="$DB_PATH" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -cache_size="$ROCKSDB_CACHE_SIZE" -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom @@ -353,7 +360,7 @@ jobs: -db="$DB_PATH" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -cache_size="$ROCKSDB_CACHE_SIZE" -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom @@ -372,7 +379,7 @@ jobs: -db="$DB_PATH" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -cache_size="$ROCKSDB_CACHE_SIZE" -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom diff --git a/.github/workflows/db_bench-dcompact-run.yml b/.github/workflows/db_bench-dcompact-run.yml index 1a6cc9190..74e488040 100644 --- a/.github/workflows/db_bench-dcompact-run.yml +++ b/.github/workflows/db_bench-dcompact-run.yml @@ -34,6 +34,11 @@ on: - light_zip - zip - bb + value_size: + description: db_bench -value_size bytes (default 50) + required: false + default: "50" + type: string # Must match plain run — serialize gh-pages pushes. concurrency: @@ -55,6 +60,7 @@ jobs: NUM: ${{ inputs.num }} CPU_QUOTA: ${{ inputs.cpu_quota }} L1_WRITER: ${{ inputs.l1_writer }} + VALUE_SIZE: ${{ inputs.value_size || '50' }} steps: - name: Checkout build commit uses: actions/checkout@v4 diff --git a/.github/workflows/db_bench-run.yml b/.github/workflows/db_bench-run.yml index a2ada96b0..0fe772978 100644 --- a/.github/workflows/db_bench-run.yml +++ b/.github/workflows/db_bench-run.yml @@ -28,6 +28,11 @@ on: - light_zip - zip - bb + value_size: + description: db_bench -value_size bytes (default 50) + required: false + default: "50" + type: string concurrency: group: pages-deploy @@ -49,6 +54,7 @@ jobs: # workflow_run has no inputs; defaults match dcompact-run. CPU_QUOTA: ${{ inputs.cpu_quota || '50%' }} L1_WRITER: ${{ inputs.l1_writer || 'simple' }} + VALUE_SIZE: ${{ inputs.value_size || '50' }} steps: - name: Checkout build commit uses: actions/checkout@v4 @@ -126,7 +132,8 @@ jobs: echo "FAIL: plain yaml still has compaction_executor_factory" >&2 exit 1 fi - # Parallel config: enable DictZip for value_size=15 (default yaml is 3000). + # Parallel config: enable DictZip for small values (default yaml is 3000; + # CI -value_size defaults to 50, so minDictZipValueSize=10 applies). YAML_DZ10="$TOPLING/toplingdb-conf/db_bench_enterprise_dictzip10.yaml" cp -a "$YAML_BASE" "$YAML_DZ10" sed -i 's/^\([[:space:]]*minDictZipValueSize:[[:space:]]*\)3000/\110/' "$YAML_DZ10" @@ -178,7 +185,7 @@ jobs: -json "$yaml" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom -enable_zero_copy @@ -194,7 +201,7 @@ jobs: -json "$yaml" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -benchmarks=nextwithkey,nextwithkey,nextwithkey,readseq,readseq,readseq -scan_omit_key -scan_omit_value @@ -217,7 +224,7 @@ jobs: -json "$yaml" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom -enable_zero_copy @@ -326,7 +333,7 @@ jobs: -db="$DB_PATH" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -cache_size="$ROCKSDB_CACHE_SIZE" -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom @@ -350,7 +357,7 @@ jobs: -db="$DB_PATH" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -cache_size="$ROCKSDB_CACHE_SIZE" -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom @@ -424,7 +431,7 @@ jobs: -db="$DB_PATH" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -cache_size="$ROCKSDB_CACHE_SIZE" -benchmarks=fillrandom,flush,compact,readseq,readseq,readseq,readrandom @@ -446,7 +453,7 @@ jobs: -db="$DB_PATH" -num=100000000 -key_size=8 - -value_size=15 + -value_size="${VALUE_SIZE}" -batch_size=1000 -cache_size="$ROCKSDB_CACHE_SIZE" -benchmarks=fillseq,flush,compact,readseq,readseq,readseq,readrandom