Skip to content

Commit a9b9573

Browse files
avolkov-intelavolkov-intel
authored
Update Xgboost Benchmarks (#220)
* Initial commit * Add bioresponse and nyc_taxi_green datasets, update xgboost configs, save preprocessing args to results * Add task_dispatch option to avoid data duplication for airline_depdelay dataset * Minor optimization
1 parent 62f89b6 commit a9b9573

9 files changed

Lines changed: 475 additions & 101 deletions

File tree

‎configs/common/xgboost.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
"device": "cpu",
77
"estimator_params": {
88
"tree_method": "hist",
9-
"n_jobs": "[SPECIAL_VALUE]physical_cpus"
9+
"n_jobs": "[SPECIAL_VALUE]physical_cpus",
10+
"enable_categorical": true
1011
},
1112
"enable_modelbuilders": false
1213
}
@@ -16,7 +17,8 @@
1617
"device": "gpu",
1718
"estimator_params": {
1819
"tree_method": "hist",
19-
"n_jobs": "[SPECIAL_VALUE]physical_cpus"
20+
"n_jobs": "[SPECIAL_VALUE]physical_cpus",
21+
"enable_categorical": true
2022
}
2123
},
2224
"data": { "format": "cudf" }

‎configs/regular/xgboost_binary.json‎

Lines changed: 81 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@
66
"data": {
77
"dataset": "airline_depdelay",
88
"preprocessing_kwargs": {
9-
"category_encoding": ["onehot", "ordinal"],
10-
"subsample": 600000
9+
"category_encoding": ["ordinal"],
10+
"subsample": 700000
1111
},
1212
"split_kwargs": {
13-
"train_size": 100000,
13+
"train_size": 200000,
1414
"test_size": 500000
1515
}
1616
},
@@ -26,37 +26,102 @@
2626
},
2727
{
2828
"data": {
29-
"dataset": "hepmass",
29+
"dataset": "airline_satisfaction",
30+
"preprocessing_kwargs": {
31+
"category_encoding": ["ignore", "ordinal"]
32+
},
3033
"split_kwargs": {
31-
"train_size": 200000,
32-
"test_size": 1000000
34+
"test_size": 0.2,
35+
"random_state": 42
3336
}
3437
},
3538
"algorithm": {
3639
"estimator_params": {
3740
"learning_rate": 0.1,
38-
"reg_alpha": 1.0,
41+
"max_depth": 6,
3942
"reg_lambda": 1.0,
40-
"max_leaves": 256,
41-
"n_estimators": 500
43+
"n_estimators": 1000,
44+
"enable_categorical": true
45+
}
46+
}
47+
},
48+
{
49+
"data": {
50+
"dataset": "amazon_employee_access",
51+
"preprocessing_kwargs": {
52+
"category_encoding": ["ignore", "ordinal"]
53+
},
54+
"split_kwargs": {
55+
"test_size": 0.2,
56+
"random_state": 42
57+
}
58+
},
59+
"algorithm": {
60+
"estimator_params": {
61+
"learning_rate": 0.05,
62+
"max_depth": 6,
63+
"reg_lambda": 1.0,
64+
"n_estimators": 500,
65+
"enable_categorical": true
4266
}
4367
}
4468
},
4569
{
4670
"data": {
47-
"dataset": "gisette",
71+
"dataset": "apsfailure",
72+
"preprocessing_kwargs": {
73+
"replace_nan": ["ignore", "mean"]
74+
},
4875
"split_kwargs": {
49-
"train_size": 2000,
50-
"test_size": 5000
76+
"test_size": 0.2,
77+
"random_state": 42
5178
}
5279
},
5380
"algorithm": {
5481
"estimator_params": {
55-
"learning_rate": 0.15,
82+
"learning_rate": 0.05,
83+
"max_depth": 6,
84+
"subsample": 0.8,
85+
"colsample_bytree": 0.8,
86+
"reg_lambda": 1.0,
87+
"n_estimators": 200
88+
}
89+
}
90+
},
91+
{
92+
"data": {
93+
"dataset": "bioresponse",
94+
"split_kwargs": {
95+
"test_size": 0.2,
96+
"random_state": 42
97+
}
98+
},
99+
"algorithm": {
100+
"estimator_params": {
101+
"learning_rate": 0.05,
102+
"max_depth": 6,
103+
"subsample": 0.8,
104+
"colsample_bytree": 0.8,
105+
"reg_lambda": 1.0,
106+
"n_estimators": 200
107+
}
108+
}
109+
},
110+
{
111+
"data": {
112+
"dataset": "hepmass",
113+
"split_kwargs": {
114+
"train_size": 400000,
115+
"test_size": 1000000
116+
}
117+
},
118+
"algorithm": {
119+
"estimator_params": {
120+
"learning_rate": 0.1,
121+
"reg_alpha": 1.0,
122+
"reg_lambda": 1.0,
56123
"max_leaves": 256,
57-
"colsample_bytree": 0.1,
58-
"colsample_bynode": 0.1,
59-
"n_estimators": 100
124+
"n_estimators": 200
60125
}
61126
}
62127
}

‎configs/regular/xgboost_multi.json‎

Lines changed: 5 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -4,35 +4,11 @@
44
"gbt multi classification data": [
55
{
66
"data": {
7-
"dataset": "letters",
8-
"split_kwargs": {
9-
"train_size": 0.5,
10-
"test_size": 0.5
11-
}
12-
},
13-
"algorithm": {
14-
"estimator_params": {
15-
"learning_rate": 0.05,
16-
"reg_lambda": 1,
17-
"max_depth": 6,
18-
"n_estimators": 200
19-
}
20-
}
21-
},
22-
{
23-
"data": {
24-
"dataset": "mnist",
25-
"split_kwargs": {
26-
"train_size": 20000,
27-
"test_size": 50000
28-
}
7+
"dataset": "connect"
298
},
309
"algorithm": {
3110
"estimator_params": {
32-
"max_leaves": 256,
33-
"colsample_bytree": 0.2,
34-
"colsample_bynode": 0.5,
35-
"n_estimators": 100
11+
"n_estimators": 500
3612
}
3713
}
3814
},
@@ -42,20 +18,10 @@
4218
},
4319
"algorithm": {
4420
"estimator_params": {
45-
"learning_rate": 0.1,
21+
"learning_rate": 0.3,
4622
"reg_lambda": 1,
47-
"max_depth": 8,
48-
"n_estimators": 200
49-
}
50-
}
51-
},
52-
{
53-
"data": {
54-
"dataset": "connect"
55-
},
56-
"algorithm": {
57-
"estimator_params": {
58-
"n_estimators": 500
23+
"max_depth": 14,
24+
"n_estimators": 100
5925
}
6026
}
6127
}

‎configs/regular/xgboost_regression.json‎

Lines changed: 58 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,25 @@
44
"gbt regression data": [
55
{
66
"data": {
7-
"dataset": "twodplanes",
7+
"dataset": "airline_depdelay",
8+
"dataset_kwargs": {
9+
"task": "regression"
10+
},
11+
"preprocessing_kwargs": {
12+
"category_encoding": "ordinal",
13+
"subsample": 700000
14+
},
815
"split_kwargs": {
9-
"train_size": 0.33,
10-
"test_size": null
16+
"train_size": 200000,
17+
"test_size": 500000
1118
}
1219
},
1320
"algorithm": {
1421
"estimator_params": {
15-
"learning_rate": 0.01,
22+
"learning_rate": 0.1,
23+
"max_depth": 8,
1624
"reg_alpha": 1.0,
17-
"reg_lambda": 0.1,
25+
"reg_lambda": 1.0,
1826
"n_estimators": 500
1927
}
2028
}
@@ -23,8 +31,8 @@
2331
"data": {
2432
"dataset": "medical_charges_nominal",
2533
"split_kwargs": {
26-
"train_size": 0.1,
27-
"test_size": 0.9
34+
"train_size": 0.5,
35+
"test_size": 0.5
2836
}
2937
},
3038
"algorithm": {
@@ -39,54 +47,76 @@
3947
},
4048
{
4149
"data": {
42-
"dataset": "year_prediction_msd",
50+
"dataset": "nyc_taxi_green",
51+
"preprocessing_kwargs": {
52+
"category_encoding": ["ordinal", "ignore"]
53+
},
4354
"split_kwargs": {
44-
"train_size": 0.25,
45-
"test_size": 0.75
55+
"test_size": 0.2,
56+
"random_state": 42
4657
}
4758
},
4859
"algorithm": {
4960
"estimator_params": {
5061
"learning_rate": 0.1,
51-
"reg_alpha": 0.5,
52-
"reg_lambda": 0.5,
53-
"n_estimators": 200
62+
"max_depth": 8,
63+
"reg_lambda": 1.0,
64+
"n_estimators": 500
5465
}
5566
}
5667
},
5768
{
5869
"data": {
59-
"dataset": "hepmass",
70+
"dataset": "qsar_tid_11",
6071
"split_kwargs": {
61-
"train_size": 200000,
62-
"test_size": 1000000
72+
"test_size": 0.2,
73+
"random_state": 42
6374
}
6475
},
6576
"algorithm": {
6677
"estimator_params": {
67-
"learning_rate": 0.1,
68-
"reg_alpha": 1.0,
78+
"learning_rate": 0.05,
79+
"max_depth": 6,
80+
"subsample": 0.8,
81+
"colsample_bytree": 0.8,
6982
"reg_lambda": 1.0,
70-
"max_leaves": 256,
71-
"n_estimators": 500
83+
"n_estimators": 1000
84+
}
85+
}
86+
},
87+
{
88+
"data": {
89+
"dataset": "superconductivity",
90+
"split_kwargs": {
91+
"test_size": 0.2,
92+
"random_state": 42
93+
}
94+
},
95+
"algorithm": {
96+
"estimator_params": {
97+
"learning_rate": 0.02,
98+
"max_depth": 7,
99+
"subsample": 0.8,
100+
"colsample_bytree": 0.8,
101+
"reg_lambda": 1.0,
102+
"n_estimators": 1500
72103
}
73104
}
74105
},
75106
{
76107
"data": {
77-
"dataset": "gisette",
108+
"dataset": "year_prediction_msd",
78109
"split_kwargs": {
79-
"train_size": 2000,
80-
"test_size": 5000
110+
"train_size": 0.5,
111+
"test_size": 0.5
81112
}
82113
},
83114
"algorithm": {
84115
"estimator_params": {
85-
"learning_rate": 0.15,
86-
"max_leaves": 256,
87-
"colsample_bytree": 0.1,
88-
"colsample_bynode": 0.1,
89-
"n_estimators": 100
116+
"learning_rate": 0.1,
117+
"reg_alpha": 0.5,
118+
"reg_lambda": 0.5,
119+
"n_estimators": 200
90120
}
91121
}
92122
}

‎sklbench/benchmarks/common.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@
1515
# ===============================================================================
1616

1717
import argparse
18+
import inspect
1819
import json
1920
from typing import Dict
2021

22+
from ..datasets.common import preprocess_x
2123
from ..utils.bench_case import get_bench_case_value, get_data_name
2224
from ..utils.custom_types import BenchCase
2325
from ..utils.logger import logger
@@ -50,6 +52,22 @@ def enrich_result(result: Dict, bench_case: BenchCase) -> Dict:
5052
f"`{result['library']}` to `sklearnex` in benchmark output."
5153
)
5254
result["library"] = "sklearnex"
55+
preproc_kwargs = get_bench_case_value(bench_case, "data:preprocessing_kwargs", dict())
56+
preproc_defaults = {
57+
name: param.default
58+
for name, param in inspect.signature(preprocess_x).parameters.items()
59+
}
60+
result.update(
61+
{
62+
"category_encoding": preproc_kwargs.get(
63+
"category_encoding", preproc_defaults["category_encoding"]
64+
),
65+
"replace_nan": preproc_kwargs.get(
66+
"replace_nan", preproc_defaults["replace_nan"]
67+
),
68+
"normalize": preproc_kwargs.get("normalize", preproc_defaults["normalize"]),
69+
}
70+
)
5371
taskset = get_bench_case_value(bench_case, "bench:taskset", None)
5472
if taskset is not None:
5573
result.update({"taskset": taskset})

0 commit comments

Comments
 (0)