From 5fb9475fc7d562d943cbc1e87665c8c2be93f805 Mon Sep 17 00:00:00 2001 From: "jinmanx (generated by with_the_same_user script)" Date: Fri, 26 Jun 2026 08:15:08 +0000 Subject: [PATCH] Add cuTile backends for 8 kernels ported from TileGym + benchmarks cuTile implementations for group_norm, kl_div, llama4_rope, qwen2vl_mrope, rope, sparsemax, tiled_mlp, and multi_token_attention, dispatched via LIGER_KERNEL_IMPL=cutile. Includes three-way (liger_triton / liger_cutile / torch|huggingface) speed+memory benchmark data on B200 generated with the open-source pip nvidia-cuda-tileiras 13.3.36. Notable kernel choices: - rope/qwen2vl_mrope: stay in input dtype, drop redundant .contiguous() copies - kl_div: drop scale constexpr to avoid per-iter JIT recompile - group_norm: fp32 stats for numerical parity - multi_token_attention: conv-backward runs under the same cuDNN heuristic as the Triton path (no forced cudnn.benchmark) for apples-to-apples comparison Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmark/data/all_benchmark_data_cutile.csv | 576 ++++++++++++++++++ src/liger_kernel/ops/cutile/ops/__init__.py | 26 + src/liger_kernel/ops/cutile/ops/group_norm.py | 282 +++++++++ src/liger_kernel/ops/cutile/ops/kl_div.py | 266 ++++++++ .../ops/cutile/ops/llama4_rope.py | 205 +++++++ .../ops/cutile/ops/multi_token_attention.py | 310 ++++++++++ .../ops/cutile/ops/qwen2vl_mrope.py | 235 +++++++ src/liger_kernel/ops/cutile/ops/rope.py | 312 ++++++++++ src/liger_kernel/ops/cutile/ops/sparsemax.py | 179 ++++++ src/liger_kernel/ops/cutile/ops/tiled_mlp.py | 88 +++ 10 files changed, 2479 insertions(+) create mode 100644 src/liger_kernel/ops/cutile/ops/group_norm.py create mode 100644 src/liger_kernel/ops/cutile/ops/kl_div.py create mode 100644 src/liger_kernel/ops/cutile/ops/llama4_rope.py create mode 100644 src/liger_kernel/ops/cutile/ops/multi_token_attention.py create mode 100644 src/liger_kernel/ops/cutile/ops/qwen2vl_mrope.py create mode 100644 src/liger_kernel/ops/cutile/ops/rope.py create mode 100644 src/liger_kernel/ops/cutile/ops/sparsemax.py create mode 100644 src/liger_kernel/ops/cutile/ops/tiled_mlp.py diff --git a/benchmark/data/all_benchmark_data_cutile.csv b/benchmark/data/all_benchmark_data_cutile.csv index a1563b414..3fb1f368e 100644 --- a/benchmark/data/all_benchmark_data_cutile.csv +++ b/benchmark/data/all_benchmark_data_cutile.csv @@ -251,3 +251,579 @@ cross_entropy,liger_cutile,full,memory,MB,BT,B * T,1024,1504.0087890625,1504.008 cross_entropy,liger_cutile,full,memory,MB,BT,B * T,2048,3006.0166015625,3006.0166015625,3006.0166015625,"{""vocab_size"": 128256, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-16 06:43:23,0.8.0 cross_entropy,liger_cutile,full,memory,MB,BT,B * T,4096,6012.0322265625,6012.0322265625,6012.0322265625,"{""vocab_size"": 128256, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-16 06:43:23,0.8.0 cross_entropy,liger_cutile,full,memory,MB,BT,B * T,8192,12024.0634765625,12024.0634765625,12024.0634765625,"{""vocab_size"": 128256, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-16 06:43:23,0.8.0 +rope,huggingface,forward,speed,ms,T,sequence length,1024,0.09523200243711472,0.09516800194978714,0.09529600292444229,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:24,0.8.0 +rope,huggingface,forward,speed,ms,T,sequence length,2048,0.158720001578331,0.15865600109100342,0.15881599485874176,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:24,0.8.0 +rope,huggingface,forward,speed,ms,T,sequence length,4096,0.2959679961204529,0.2958976089954376,0.2962623953819275,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:24,0.8.0 +rope,huggingface,forward,speed,ms,T,sequence length,8192,0.5867199897766113,0.5848320126533508,0.5867519974708557,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:24,0.8.0 +rope,liger_triton,forward,speed,ms,T,sequence length,1024,0.010208000428974628,0.010143999941647053,0.010304000228643417,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,forward,speed,ms,T,sequence length,2048,0.014271999709308147,0.012345599941909313,0.014399999752640724,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,forward,speed,ms,T,sequence length,4096,0.020479999482631683,0.020416000857949257,0.02054399996995926,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,forward,speed,ms,T,sequence length,8192,0.03276799991726875,0.03270399942994118,0.03283200040459633,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,backward,speed,ms,T,sequence length,1024,0.5349119901657104,0.414899206161499,0.5765952229499817,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,backward,speed,ms,T,sequence length,2048,0.5061439871788025,0.3785152077674866,0.5573887944221496,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,backward,speed,ms,T,sequence length,4096,0.5373760163784027,0.3992895960807801,0.5855231881141663,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,backward,speed,ms,T,sequence length,8192,0.5427520275115967,0.5426432013511657,0.5428799986839294,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,backward,speed,ms,T,sequence length,1024,0.11868800222873688,0.07877119779586791,0.5847488045692444,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,backward,speed,ms,T,sequence length,2048,0.5905920267105103,0.3321728050708771,0.6310655832290649,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,backward,speed,ms,T,sequence length,4096,0.45369601249694824,0.3548160016536713,0.6121216177940368,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,backward,speed,ms,T,sequence length,8192,0.10756799951195717,0.08524799942970276,0.43775359988212587,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,full,speed,ms,T,sequence length,1024,0.6416000127792358,0.5105728149414063,0.7040896058082581,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,full,speed,ms,T,sequence length,2048,0.3123199939727783,0.31187199950218203,0.3236928105354309,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,full,speed,ms,T,sequence length,4096,0.5930879712104797,0.5918847799301148,0.683577573299408,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,full,speed,ms,T,sequence length,8192,1.186352014541626,1.185638427734375,1.1868352174758912,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,full,speed,ms,T,sequence length,1024,0.7196159958839417,0.5948544025421143,0.739897620677948,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,full,speed,ms,T,sequence length,2048,0.7146880030632019,0.6211968064308167,0.7483648180961608,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,full,speed,ms,T,sequence length,4096,0.6727039813995361,0.5496576189994813,0.7523391842842102,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,full,speed,ms,T,sequence length,8192,0.6881279945373535,0.5457343935966492,0.724460780620575,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,full,memory,MB,T,sequence length,1024,90.5,90.5,90.5,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,full,memory,MB,T,sequence length,2048,149.0,149.0,149.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,full,memory,MB,T,sequence length,4096,266.0,266.0,266.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,huggingface,full,memory,MB,T,sequence length,8192,500.0,500.0,500.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,full,memory,MB,T,sequence length,1024,64.5078125,64.5078125,64.5078125,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,full,memory,MB,T,sequence length,2048,97.015625,97.015625,97.015625,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,full,memory,MB,T,sequence length,4096,162.03125,162.03125,162.03125,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_triton,full,memory,MB,T,sequence length,8192,292.0625,292.0625,292.0625,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:25,0.8.0 +rope,liger_cutile,forward,speed,ms,T,sequence length,1024,0.010208000428974628,0.010143999941647053,0.01027199998497963,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:36,0.8.0 +rope,liger_cutile,forward,speed,ms,T,sequence length,2048,0.01235199999064207,0.012256000190973282,0.014336000196635723,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:36,0.8.0 +rope,liger_cutile,forward,speed,ms,T,sequence length,4096,0.020416000857949257,0.018432000651955605,0.02051199972629547,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:36,0.8.0 +rope,liger_cutile,forward,speed,ms,T,sequence length,8192,0.03270399942994118,0.030751999467611313,0.03280000016093254,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:36,0.8.0 +rope,liger_cutile,backward,speed,ms,T,sequence length,1024,0.5556319952011108,0.41656320691108706,0.5874624133110047,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,backward,speed,ms,T,sequence length,2048,0.06267200037837029,0.06172800064086914,0.06371200084686279,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,backward,speed,ms,T,sequence length,4096,0.06268800050020218,0.06186880096793175,0.06423039883375167,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,backward,speed,ms,T,sequence length,8192,0.08499199897050858,0.08490239828824997,0.08502399921417236,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,full,speed,ms,T,sequence length,1024,0.5871680080890656,0.4573439955711365,0.6426943898200989,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,full,speed,ms,T,sequence length,2048,0.6166400015354156,0.5661823987960816,0.6690752148628235,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,full,speed,ms,T,sequence length,4096,0.6217759847640991,0.500844806432724,0.6595520019531249,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,full,speed,ms,T,sequence length,8192,0.6234079897403717,0.5634239912033081,0.6671680212020874,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,full,memory,MB,T,sequence length,1024,62.5078125,62.5078125,62.5078125,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,full,memory,MB,T,sequence length,2048,93.015625,93.015625,93.015625,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,full,memory,MB,T,sequence length,4096,154.03125,154.03125,154.03125,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +rope,liger_cutile,full,memory,MB,T,sequence length,8192,276.0625,276.0625,276.0625,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:37,0.8.0 +group_norm,liger_triton,full,speed,ms,BT,B*T,1024,0.2499839961528778,0.22896000146865844,0.321619200706482,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,liger_triton,full,speed,ms,BT,B*T,2048,0.7440959811210632,0.6282047748565673,0.820249593257904,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,liger_triton,full,speed,ms,BT,B*T,4096,0.7464640140533447,0.6358847856521607,0.8035712122917176,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,liger_triton,full,speed,ms,BT,B*T,8192,0.7666560113430023,0.745855987071991,0.7858560085296631,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,torch,full,speed,ms,BT,B*T,1024,0.5640000104904175,0.4509888052940369,0.615340793132782,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,torch,full,speed,ms,BT,B*T,2048,0.5589119791984558,0.4487231969833374,0.6116544127464295,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,torch,full,speed,ms,BT,B*T,4096,0.5837759971618652,0.5054720044136047,0.6145920157432556,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,torch,full,speed,ms,BT,B*T,8192,0.6369120180606842,0.636025607585907,0.6374207854270935,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,liger_triton,forward,speed,ms,BT,B*T,1024,0.027135999873280525,0.026316799223423004,0.02879999950528145,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,liger_triton,forward,speed,ms,BT,B*T,2048,0.027871999889612198,0.026976000517606735,0.028896000236272812,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,liger_triton,forward,speed,ms,BT,B*T,4096,0.03379200026392937,0.033695999532938004,0.033824000507593155,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,liger_triton,forward,speed,ms,BT,B*T,8192,0.05833600088953972,0.05827200040221214,0.058400001376867294,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:55,0.8.0 +group_norm,torch,forward,speed,ms,BT,B*T,1024,0.04198399931192398,0.04188799858093262,0.042047999799251556,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,forward,speed,ms,BT,B*T,2048,0.06860800087451935,0.06851840019226074,0.06867200136184692,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,forward,speed,ms,BT,B*T,4096,0.12191999703645706,0.12177280336618423,0.12387199699878693,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,forward,speed,ms,BT,B*T,8192,0.24063999950885773,0.24057599902153015,0.2407039999961853,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,liger_triton,backward,speed,ms,BT,B*T,1024,0.5983839929103851,0.44815999269485474,0.6384959816932678,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,liger_triton,backward,speed,ms,BT,B*T,2048,0.0737600028514862,0.07171200215816498,0.08581119924783706,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,liger_triton,backward,speed,ms,BT,B*T,4096,0.6188480257987976,0.4707648098468781,0.6479168176651001,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,liger_triton,backward,speed,ms,BT,B*T,8192,0.6175040006637573,0.3928000032901764,0.6422591924667359,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,backward,speed,ms,BT,B*T,1024,0.4949119985103607,0.39532798528671265,0.521120011806488,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,backward,speed,ms,BT,B*T,2048,0.47204799950122833,0.34908159375190734,0.5194239974021911,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,backward,speed,ms,BT,B*T,4096,0.4106239974498749,0.24727680683135989,0.4713920056819916,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,backward,speed,ms,BT,B*T,8192,0.506879985332489,0.335097599029541,0.5487039923667908,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,liger_triton,full,memory,MB,BT,B*T,1024,40.0546875,40.0546875,40.0546875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,liger_triton,full,memory,MB,BT,B*T,2048,80.0625,80.0625,80.0625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,liger_triton,full,memory,MB,BT,B*T,4096,160.078125,160.078125,160.078125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,liger_triton,full,memory,MB,BT,B*T,8192,320.109375,320.109375,320.109375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,full,memory,MB,BT,B*T,1024,40.1640625,40.1640625,40.1640625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,full,memory,MB,BT,B*T,2048,80.28125,80.28125,80.28125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,full,memory,MB,BT,B*T,4096,160.515625,160.515625,160.515625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,torch,full,memory,MB,BT,B*T,8192,320.984375,320.984375,320.984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:44:56,0.8.0 +group_norm,liger_cutile,full,speed,ms,BT,B*T,1024,0.15996800363063812,0.15906560122966767,0.1764416038990021,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,full,speed,ms,BT,B*T,2048,0.5320640206336975,0.44520959854125974,0.7338111996650696,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,full,speed,ms,BT,B*T,4096,0.7078719735145569,0.5920000076293945,0.7390080094337463,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,full,speed,ms,BT,B*T,8192,0.492000013589859,0.23293440341949462,0.661958408355713,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,forward,speed,ms,BT,B*T,1024,0.011504000052809715,0.010208000428974628,0.012319999746978283,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,forward,speed,ms,BT,B*T,2048,0.016224000602960587,0.014336000196635723,0.016383999958634377,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,forward,speed,ms,BT,B*T,4096,0.02252800017595291,0.022463999688625336,0.02271359972655773,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,forward,speed,ms,BT,B*T,8192,0.03891199827194214,0.038816001266241074,0.038975998759269714,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,backward,speed,ms,BT,B*T,1024,0.5781280100345612,0.3702720105648041,0.643455982208252,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,backward,speed,ms,BT,B*T,2048,0.5400000214576721,0.4125887930393219,0.6117759943008423,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,backward,speed,ms,BT,B*T,4096,0.41251198947429657,0.360345596075058,0.6095679998397827,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,backward,speed,ms,BT,B*T,8192,0.5383679866790771,0.42373760938644406,0.6129599928855897,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:00,0.8.0 +group_norm,liger_cutile,full,memory,MB,BT,B*T,1024,40.09375,40.09375,40.09375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:01,0.8.0 +group_norm,liger_cutile,full,memory,MB,BT,B*T,2048,80.140625,80.140625,80.140625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:01,0.8.0 +group_norm,liger_cutile,full,memory,MB,BT,B*T,4096,160.234375,160.234375,160.234375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:01,0.8.0 +group_norm,liger_cutile,full,memory,MB,BT,B*T,8192,320.421875,320.421875,320.421875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""channels_per_group"": 4, ""eps"": 1e-06, ""H"": 512, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:01,0.8.0 +kl_div,torch,forward,speed,ms,BT,total tokens,1024,0.7342399954795837,0.7341567873954773,0.7342527985572815,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:04,0.8.0 +kl_div,torch,forward,speed,ms,BT,total tokens,2048,1.4357279539108276,1.434559941291809,1.438655972480774,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:04,0.8.0 +kl_div,torch,forward,speed,ms,BT,total tokens,4096,2.8252480030059814,2.824595260620117,2.825267171859741,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:04,0.8.0 +kl_div,torch,forward,speed,ms,BT,total tokens,8192,5.643295764923096,5.643295764923096,5.643295764923096,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:04,0.8.0 +kl_div,liger_triton,forward,speed,ms,BT,total tokens,1024,0.19750399887561798,0.1955839991569519,0.19760000705718994,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,liger_triton,forward,speed,ms,BT,total tokens,2048,0.3574720025062561,0.35734400153160095,0.3593344032764435,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,liger_triton,forward,speed,ms,BT,total tokens,4096,0.699455976486206,0.6994240283966064,0.7004544019699097,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,liger_triton,forward,speed,ms,BT,total tokens,8192,1.3752319812774658,1.375167965888977,1.3769792318344116,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,torch,backward,speed,ms,BT,total tokens,1024,1.1336640119552612,1.132262396812439,1.135321593284607,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,torch,backward,speed,ms,BT,total tokens,2048,2.133039951324463,2.1330111980438233,2.1330559253692627,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,torch,backward,speed,ms,BT,total tokens,4096,4.195375919342041,4.193542289733887,4.1972095489501955,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,torch,backward,speed,ms,BT,total tokens,8192,8.298463821411133,8.298463821411133,8.298463821411133,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,liger_triton,backward,speed,ms,BT,total tokens,1024,1.3199679851531982,1.3179199695587158,1.3220160007476807,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,liger_triton,backward,speed,ms,BT,total tokens,2048,2.4453439712524414,2.4453439712524414,2.4471871852874756,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,liger_triton,backward,speed,ms,BT,total tokens,4096,4.853711843490601,4.840147018432617,4.867276668548584,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,liger_triton,backward,speed,ms,BT,total tokens,8192,9.604127883911133,9.604127883911133,9.604127883911133,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:05,0.8.0 +kl_div,torch,full,speed,ms,BT,total tokens,1024,1.819648027420044,1.8193663597106933,1.8212032079696656,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,torch,full,speed,ms,BT,total tokens,2048,3.531808018684387,3.5311552047729493,3.532460832595825,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,torch,full,speed,ms,BT,total tokens,4096,6.985856056213379,6.985856056213379,6.985856056213379,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,torch,full,speed,ms,BT,total tokens,8192,13.896832466125488,13.896832466125488,13.896832466125488,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,liger_triton,full,speed,ms,BT,total tokens,1024,1.5084159970283508,1.483839988708496,1.508512020111084,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,liger_triton,full,speed,ms,BT,total tokens,2048,2.8303680419921875,2.8303680419921875,2.832806348800659,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,liger_triton,full,speed,ms,BT,total tokens,4096,5.557248115539551,5.557248115539551,5.557248115539551,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,liger_triton,full,speed,ms,BT,total tokens,8192,10.924063682556152,10.924063682556152,10.924063682556152,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,torch,full,memory,MB,BT,total tokens,1024,3514.0,3514.0,3514.0,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,torch,full,memory,MB,BT,total tokens,2048,7014.0,7014.0,7014.0,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,torch,full,memory,MB,BT,total tokens,4096,14028.0,14028.0,14028.0,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,torch,full,memory,MB,BT,total tokens,8192,28056.0,28056.0,28056.0,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,liger_triton,full,memory,MB,BT,total tokens,1024,3514.0009765625,3514.0009765625,3514.0009765625,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,liger_triton,full,memory,MB,BT,total tokens,2048,7014.0009765625,7014.0009765625,7014.0009765625,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,liger_triton,full,memory,MB,BT,total tokens,4096,14028.0009765625,14028.0009765625,14028.0009765625,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,liger_triton,full,memory,MB,BT,total tokens,8192,28056.0,28056.0,28056.0,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:06,0.8.0 +kl_div,liger_cutile,forward,speed,ms,BT,total tokens,1024,0.20235199481248856,0.2017280012369156,0.20377600193023682,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:10,0.8.0 +kl_div,liger_cutile,forward,speed,ms,BT,total tokens,2048,0.3635199964046478,0.3631871879100799,0.3636223912239075,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:10,0.8.0 +kl_div,liger_cutile,forward,speed,ms,BT,total tokens,4096,0.666703999042511,0.666643214225769,0.6686848044395447,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:10,0.8.0 +kl_div,liger_cutile,forward,speed,ms,BT,total tokens,8192,1.3159680366516113,1.3157824039459227,1.316883158683777,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:10,0.8.0 +kl_div,liger_cutile,backward,speed,ms,BT,total tokens,1024,0.8325440287590027,0.830560028553009,0.836575984954834,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:11,0.8.0 +kl_div,liger_cutile,backward,speed,ms,BT,total tokens,2048,1.5252320170402527,1.5216959714889526,1.525920033454895,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:11,0.8.0 +kl_div,liger_cutile,backward,speed,ms,BT,total tokens,4096,2.935839891433716,2.9112639904022215,2.9413695335388184,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:11,0.8.0 +kl_div,liger_cutile,backward,speed,ms,BT,total tokens,8192,5.671840190887451,5.671840190887451,5.671840190887451,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:11,0.8.0 +kl_div,liger_cutile,full,speed,ms,BT,total tokens,1024,0.9810240268707275,0.9745023846626282,0.9890367984771729,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:12,0.8.0 +kl_div,liger_cutile,full,speed,ms,BT,total tokens,2048,1.8770240545272827,1.8760000467300415,1.8801023721694945,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:12,0.8.0 +kl_div,liger_cutile,full,speed,ms,BT,total tokens,4096,3.5901600122451782,3.5766335964202884,3.6036864280700684,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:12,0.8.0 +kl_div,liger_cutile,full,speed,ms,BT,total tokens,8192,7.0278401374816895,7.0278401374816895,7.0278401374816895,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:12,0.8.0 +kl_div,liger_cutile,full,memory,MB,BT,total tokens,1024,3012.0009765625,3012.0009765625,3012.0009765625,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:12,0.8.0 +kl_div,liger_cutile,full,memory,MB,BT,total tokens,2048,6012.0009765625,6012.0009765625,6012.0009765625,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:12,0.8.0 +kl_div,liger_cutile,full,memory,MB,BT,total tokens,4096,12024.0009765625,12024.0009765625,12024.0009765625,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:12,0.8.0 +kl_div,liger_cutile,full,memory,MB,BT,total tokens,8192,24048.0,24048.0,24048.0,"{""vocab_size"": 128256, ""reduction"": ""batchmean"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:12,0.8.0 +llama4_rope,huggingface,forward,speed,ms,T,sequence length,1024,0.04915200173854828,0.0490880012512207,0.049215998500585556,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:16,0.8.0 +llama4_rope,huggingface,forward,speed,ms,T,sequence length,2048,0.07985599711537361,0.07977599650621414,0.07993599772453308,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:16,0.8.0 +llama4_rope,huggingface,forward,speed,ms,T,sequence length,4096,0.14536000043153763,0.14345599710941315,0.1454399973154068,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:16,0.8.0 +llama4_rope,huggingface,forward,speed,ms,T,sequence length,8192,0.27024000883102417,0.2682560086250305,0.2703744053840637,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:16,0.8.0 +llama4_rope,liger_triton,forward,speed,ms,T,sequence length,1024,0.0297279991209507,0.029664000496268272,0.029798399657011032,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,forward,speed,ms,T,sequence length,2048,0.05020799860358238,0.04934399947524071,0.0502655990421772,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,forward,speed,ms,T,sequence length,4096,0.09116800129413605,0.09107200056314468,0.09123200178146362,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,forward,speed,ms,T,sequence length,8192,0.17110399901866913,0.17098239958286285,0.17215999960899353,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,backward,speed,ms,T,sequence length,1024,0.35359999537467957,0.2188352018594742,0.4876543879508972,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,backward,speed,ms,T,sequence length,2048,0.33139199018478394,0.05936000123620033,0.4795967936515808,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,backward,speed,ms,T,sequence length,4096,0.5065280199050903,0.3521344006061554,0.553766393661499,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,backward,speed,ms,T,sequence length,8192,0.4472000002861023,0.3665919899940491,0.5784000158309937,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,backward,speed,ms,T,sequence length,1024,0.6083840131759644,0.3575039982795715,0.6624191999435425,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,backward,speed,ms,T,sequence length,2048,0.08294399827718735,0.08007679730653763,0.08648959994316101,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,backward,speed,ms,T,sequence length,4096,0.4219839870929718,0.10652799904346466,0.6199616193771362,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,backward,speed,ms,T,sequence length,8192,0.6026239991188049,0.3756160080432892,0.6403199911117554,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,full,speed,ms,T,sequence length,1024,0.4391839951276779,0.3625600039958954,0.6384320259094238,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,full,speed,ms,T,sequence length,2048,0.4601760059595108,0.3656063973903656,0.6447487831115724,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,full,speed,ms,T,sequence length,4096,0.6929280161857605,0.6059200167655945,0.7140799760818481,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,full,speed,ms,T,sequence length,8192,0.5355360209941864,0.5348672032356262,0.5370175838470459,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,full,speed,ms,T,sequence length,1024,0.6835039854049683,0.599449610710144,0.7411519885063171,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,full,speed,ms,T,sequence length,2048,0.47673600912094116,0.20494080185890198,0.6308415889739991,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,full,speed,ms,T,sequence length,4096,0.7275040149688721,0.5783936023712158,0.7822656035423279,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,full,speed,ms,T,sequence length,8192,0.7310400009155273,0.6908159971237182,0.7700864195823669,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,full,memory,MB,T,sequence length,1024,100.5,100.5,100.5,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,full,memory,MB,T,sequence length,2048,169.0,169.0,169.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,full,memory,MB,T,sequence length,4096,306.0,306.0,306.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,huggingface,full,memory,MB,T,sequence length,8192,580.0,580.0,580.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:17,0.8.0 +llama4_rope,liger_triton,full,memory,MB,T,sequence length,1024,63.0,63.0,63.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:18,0.8.0 +llama4_rope,liger_triton,full,memory,MB,T,sequence length,2048,94.0,94.0,94.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:18,0.8.0 +llama4_rope,liger_triton,full,memory,MB,T,sequence length,4096,156.0,156.0,156.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:18,0.8.0 +llama4_rope,liger_triton,full,memory,MB,T,sequence length,8192,280.0,280.0,280.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:18,0.8.0 +llama4_rope,liger_cutile,forward,speed,ms,T,sequence length,1024,0.027712000533938408,0.027615999802947044,0.028736000880599022,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:22,0.8.0 +llama4_rope,liger_cutile,forward,speed,ms,T,sequence length,2048,0.04806400090456009,0.0461760014295578,0.04819199815392494,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:22,0.8.0 +llama4_rope,liger_cutile,forward,speed,ms,T,sequence length,4096,0.08499199897050858,0.08409599959850311,0.08508799970149994,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:22,0.8.0 +llama4_rope,liger_cutile,forward,speed,ms,T,sequence length,8192,0.15891200304031372,0.158720001578331,0.16076800227165222,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:22,0.8.0 +llama4_rope,liger_cutile,backward,speed,ms,T,sequence length,1024,0.6017280220985413,0.539852786064148,0.6345343828201294,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:22,0.8.0 +llama4_rope,liger_cutile,backward,speed,ms,T,sequence length,2048,0.10023999959230423,0.07660800218582153,0.4804608106613161,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:22,0.8.0 +llama4_rope,liger_cutile,backward,speed,ms,T,sequence length,4096,0.48127999901771545,0.3771519958972931,0.6027200222015381,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:22,0.8.0 +llama4_rope,liger_cutile,backward,speed,ms,T,sequence length,8192,0.5959360003471375,0.5335552096366882,0.6111423969268799,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:22,0.8.0 +llama4_rope,liger_cutile,full,speed,ms,T,sequence length,1024,0.7049599885940552,0.4957760035991669,0.7362751960754395,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:23,0.8.0 +llama4_rope,liger_cutile,full,speed,ms,T,sequence length,2048,0.7149919867515564,0.6569280028343201,0.7863039970397949,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:23,0.8.0 +llama4_rope,liger_cutile,full,speed,ms,T,sequence length,4096,0.681056022644043,0.5516607880592346,0.7133503913879395,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:23,0.8.0 +llama4_rope,liger_cutile,full,speed,ms,T,sequence length,8192,0.656688004732132,0.4464640080928803,0.7335488080978394,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:23,0.8.0 +llama4_rope,liger_cutile,full,memory,MB,T,sequence length,1024,63.0,63.0,63.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:23,0.8.0 +llama4_rope,liger_cutile,full,memory,MB,T,sequence length,2048,94.0,94.0,94.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:23,0.8.0 +llama4_rope,liger_cutile,full,memory,MB,T,sequence length,4096,156.0,156.0,156.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:23,0.8.0 +llama4_rope,liger_cutile,full,memory,MB,T,sequence length,8192,280.0,280.0,280.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:23,0.8.0 +qwen2vl_mrope,huggingface,forward,speed,ms,T,sequence length,1024,0.1034879982471466,0.10342399775981903,0.10355199873447418,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:27,0.8.0 +qwen2vl_mrope,huggingface,forward,speed,ms,T,sequence length,2048,0.16899199783802032,0.16889600455760956,0.1690432012081146,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:27,0.8.0 +qwen2vl_mrope,huggingface,forward,speed,ms,T,sequence length,4096,0.30831998586654663,0.3082367956638336,0.3093824028968811,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:27,0.8.0 +qwen2vl_mrope,huggingface,forward,speed,ms,T,sequence length,8192,0.6031680107116699,0.6010879874229431,0.6032703757286072,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:27,0.8.0 +qwen2vl_mrope,liger_triton,forward,speed,ms,T,sequence length,1024,0.010208000428974628,0.010143999941647053,0.01027199998497963,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,liger_triton,forward,speed,ms,T,sequence length,2048,0.014336000196635723,0.014271999709308147,0.014399999752640724,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,liger_triton,forward,speed,ms,T,sequence length,4096,0.020479999482631683,0.020416000857949257,0.02054399996995926,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,liger_triton,forward,speed,ms,T,sequence length,8192,0.03481600061058998,0.03471999987959862,0.03488000109791756,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,huggingface,backward,speed,ms,T,sequence length,1024,0.5634880065917969,0.5428607940673829,0.6014976143836975,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,huggingface,backward,speed,ms,T,sequence length,2048,0.5027680099010468,0.39779200553894045,0.560422396659851,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,huggingface,backward,speed,ms,T,sequence length,4096,0.5095199942588806,0.41171200275421144,0.5752127766609192,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,huggingface,backward,speed,ms,T,sequence length,8192,0.5428479909896851,0.5427136182785034,0.5435967922210694,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,liger_triton,backward,speed,ms,T,sequence length,1024,0.5394880175590515,0.30158719420433044,0.5978816032409668,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,liger_triton,backward,speed,ms,T,sequence length,2048,0.5799679756164551,0.398086404800415,0.6035839796066285,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,liger_triton,backward,speed,ms,T,sequence length,4096,0.5827679932117462,0.5295040011405945,0.6292160153388977,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,liger_triton,backward,speed,ms,T,sequence length,8192,0.5707520246505737,0.44652798771858215,0.597599983215332,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,huggingface,full,speed,ms,T,sequence length,1024,0.734495997428894,0.6113855957984924,0.819865608215332,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,huggingface,full,speed,ms,T,sequence length,2048,0.6587840020656586,0.4720831930637359,0.7595648050308228,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,huggingface,full,speed,ms,T,sequence length,4096,0.704479992389679,0.6042815804481506,0.748748791217804,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,huggingface,full,speed,ms,T,sequence length,8192,1.2021439671516418,1.201248049736023,1.2028480529785157,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:28,0.8.0 +qwen2vl_mrope,liger_triton,full,speed,ms,T,sequence length,1024,0.7143359780311584,0.6648320078849792,0.766643190383911,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,liger_triton,full,speed,ms,T,sequence length,2048,0.6994239985942841,0.4365440011024475,0.7249280214309692,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,liger_triton,full,speed,ms,T,sequence length,4096,0.7043200135231018,0.4633343994617462,0.7334143996238709,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,liger_triton,full,speed,ms,T,sequence length,8192,0.6531839966773987,0.5222847938537598,0.714796781539917,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,huggingface,full,memory,MB,T,sequence length,1024,92.0,92.0,92.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,huggingface,full,memory,MB,T,sequence length,2048,152.0,152.0,152.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,huggingface,full,memory,MB,T,sequence length,4096,273.0,273.0,273.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,huggingface,full,memory,MB,T,sequence length,8192,512.0,512.0,512.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,liger_triton,full,memory,MB,T,sequence length,1024,65.5,65.5,65.5,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,liger_triton,full,memory,MB,T,sequence length,2048,100.0,100.0,100.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,liger_triton,full,memory,MB,T,sequence length,4096,167.0,167.0,167.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,liger_triton,full,memory,MB,T,sequence length,8192,300.0,300.0,300.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:29,0.8.0 +qwen2vl_mrope,liger_cutile,forward,speed,ms,T,sequence length,1024,0.01228800043463707,0.012223999947309494,0.01235199999064207,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:33,0.8.0 +qwen2vl_mrope,liger_cutile,forward,speed,ms,T,sequence length,2048,0.016383999958634377,0.0163199994713068,0.016448000445961952,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:33,0.8.0 +qwen2vl_mrope,liger_cutile,forward,speed,ms,T,sequence length,4096,0.02457600086927414,0.024512000381946564,0.024639999493956566,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:33,0.8.0 +qwen2vl_mrope,liger_cutile,forward,speed,ms,T,sequence length,8192,0.04095999896526337,0.04089599847793579,0.04102399945259094,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:33,0.8.0 +qwen2vl_mrope,liger_cutile,backward,speed,ms,T,sequence length,1024,0.5711199939250946,0.43836799860000614,0.6122944235801696,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,backward,speed,ms,T,sequence length,2048,0.5389119982719421,0.38092800974845886,0.5840319991111755,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,backward,speed,ms,T,sequence length,4096,0.5717759728431702,0.44033919572830205,0.6017728090286255,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,backward,speed,ms,T,sequence length,8192,0.5131840109825134,0.3661119997501373,0.6299647927284241,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,full,speed,ms,T,sequence length,1024,0.6312959790229797,0.4719935953617096,0.6950848221778869,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,full,speed,ms,T,sequence length,2048,0.6547840237617493,0.5117887973785401,0.6850239872932434,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,full,speed,ms,T,sequence length,4096,0.6442559957504272,0.511193597316742,0.664409601688385,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,full,speed,ms,T,sequence length,8192,0.6613920032978058,0.5410112142562866,0.6881983757019043,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,full,memory,MB,T,sequence length,1024,65.5,65.5,65.5,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,full,memory,MB,T,sequence length,2048,100.0,100.0,100.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,full,memory,MB,T,sequence length,4096,167.0,167.0,167.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +qwen2vl_mrope,liger_cutile,full,memory,MB,T,sequence length,8192,300.0,300.0,300.0,"{""hidden_size"": 4096, ""num_attention_heads"": 32, ""num_key_value_heads"": 8, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:34,0.8.0 +sparsemax,torch,forward,speed,ms,BT,B*T,1024,0.2467840015888214,0.24582399427890778,0.24687999486923218,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,torch,forward,speed,ms,BT,B*T,2048,0.38395199179649353,0.3829759955406189,0.3851200044155121,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,torch,forward,speed,ms,BT,B*T,4096,0.704479992389679,0.7028863906860352,0.7054911971092224,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,torch,forward,speed,ms,BT,B*T,8192,1.24617600440979,1.2451776027679442,1.2462015628814698,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,liger_triton,forward,speed,ms,BT,B*T,1024,0.1371839940547943,0.1363839954137802,0.13731199502944946,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,liger_triton,forward,speed,ms,BT,B*T,2048,0.25804799795150757,0.2579840123653412,0.25811201333999634,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,liger_triton,forward,speed,ms,BT,B*T,4096,0.50587198138237,0.5057536125183105,0.5059199929237366,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,liger_triton,forward,speed,ms,BT,B*T,8192,0.9830399751663208,0.9820863842964173,0.9831680059432983,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,torch,backward,speed,ms,BT,B*T,1024,0.10335999727249146,0.10326399654150009,0.1034879982471466,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,torch,backward,speed,ms,BT,B*T,2048,0.5374880135059357,0.4613055944442749,0.5898240089416504,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,torch,backward,speed,ms,BT,B*T,4096,0.568448007106781,0.40909439921379087,0.5878399968147278,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,torch,backward,speed,ms,BT,B*T,8192,0.5622240006923676,0.5612800002098084,0.5623296022415161,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:38,0.8.0 +sparsemax,liger_triton,backward,speed,ms,BT,B*T,1024,0.05007999949157238,0.047999999672174457,0.05270400047302246,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,backward,speed,ms,BT,B*T,2048,0.05020799860358238,0.04891519993543625,0.0513215996325016,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,backward,speed,ms,BT,B*T,4096,0.04923199862241745,0.04800000041723251,0.050572801381349564,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,backward,speed,ms,BT,B*T,8192,0.5578879714012146,0.40651519894599913,0.6081535816192627,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,torch,full,speed,ms,BT,B*T,1024,0.785152018070221,0.6892032146453857,0.8052223920822144,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,torch,full,speed,ms,BT,B*T,2048,0.653359979391098,0.5611647844314576,0.7851136088371278,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,torch,full,speed,ms,BT,B*T,4096,1.0260800123214722,1.0250112295150757,1.027020812034607,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,torch,full,speed,ms,BT,B*T,8192,1.8709440231323242,1.8700160026550294,1.871385622024536,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,full,speed,ms,BT,B*T,1024,0.7334719896316528,0.6339071989059448,0.7500927925109864,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,full,speed,ms,BT,B*T,2048,0.6941600143909454,0.584384024143219,0.7754560112953186,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,full,speed,ms,BT,B*T,4096,0.7055040001869202,0.602617597579956,0.751148796081543,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,full,speed,ms,BT,B*T,8192,1.0924639701843262,1.0915583848953248,1.092582368850708,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,torch,full,memory,MB,BT,B*T,1024,108.015625,108.015625,108.015625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,torch,full,memory,MB,BT,B*T,2048,216.0234375,216.0234375,216.0234375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,torch,full,memory,MB,BT,B*T,4096,432.0390625,432.0390625,432.0390625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,torch,full,memory,MB,BT,B*T,8192,864.0703125,864.0703125,864.0703125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,full,memory,MB,BT,B*T,1024,80.03125,80.03125,80.03125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,full,memory,MB,BT,B*T,2048,160.03125,160.03125,160.03125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,full,memory,MB,BT,B*T,4096,320.03125,320.03125,320.03125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_triton,full,memory,MB,BT,B*T,8192,640.03125,640.03125,640.03125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:39,0.8.0 +sparsemax,liger_cutile,forward,speed,ms,BT,B*T,1024,0.03280000016093254,0.03270399942994118,0.03286400064826012,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:43,0.8.0 +sparsemax,liger_cutile,forward,speed,ms,BT,B*T,2048,0.04918399825692177,0.04912000149488449,0.049247998744249344,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:43,0.8.0 +sparsemax,liger_cutile,forward,speed,ms,BT,B*T,4096,0.09011200070381165,0.08891520053148269,0.09015040099620819,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:43,0.8.0 +sparsemax,liger_cutile,forward,speed,ms,BT,B*T,8192,0.1679680049419403,0.167891201376915,0.16804480254650117,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:43,0.8.0 +sparsemax,liger_cutile,backward,speed,ms,BT,B*T,1024,0.05135999992489815,0.04981759935617447,0.05266560018062591,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:43,0.8.0 +sparsemax,liger_cutile,backward,speed,ms,BT,B*T,2048,0.050912000238895416,0.049484800547361374,0.05193600058555603,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:43,0.8.0 +sparsemax,liger_cutile,backward,speed,ms,BT,B*T,4096,0.05057599954307079,0.049350399523973465,0.05198720097541809,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:43,0.8.0 +sparsemax,liger_cutile,backward,speed,ms,BT,B*T,8192,0.04303999990224838,0.04297599941492081,0.04505600035190582,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:43,0.8.0 +sparsemax,liger_cutile,full,speed,ms,BT,B*T,1024,0.5376319885253906,0.42336639761924744,0.6180160045623779,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:44,0.8.0 +sparsemax,liger_cutile,full,speed,ms,BT,B*T,2048,0.6377120018005371,0.5495423913002014,0.675820803642273,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:44,0.8.0 +sparsemax,liger_cutile,full,speed,ms,BT,B*T,4096,0.4270400106906891,0.3429568052291871,0.6181247949600219,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:44,0.8.0 +sparsemax,liger_cutile,full,speed,ms,BT,B*T,8192,0.610368013381958,0.4133504033088684,0.6646400213241578,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:44,0.8.0 +sparsemax,liger_cutile,full,memory,MB,BT,B*T,1024,40.0,40.0,40.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:44,0.8.0 +sparsemax,liger_cutile,full,memory,MB,BT,B*T,2048,80.0,80.0,80.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:44,0.8.0 +sparsemax,liger_cutile,full,memory,MB,BT,B*T,4096,160.0,160.0,160.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:44,0.8.0 +sparsemax,liger_cutile,full,memory,MB,BT,B*T,8192,320.0,320.0,320.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""dim"": -1, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-06-26 06:45:44,0.8.0 +tiled_geglu,huggingface,forward,speed,ms,T,sequence length,1024,0.5044000148773193,0.5038080215454102,0.5056639909744263,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:52,0.8.0 +tiled_geglu,huggingface,forward,speed,ms,T,sequence length,2048,1.0055999755859375,1.0046208381652832,1.0070400476455688,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:52,0.8.0 +tiled_geglu,huggingface,forward,speed,ms,T,sequence length,4096,1.9549440145492554,1.9534656047821046,1.9618815898895263,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:52,0.8.0 +tiled_geglu,huggingface,forward,speed,ms,T,sequence length,8192,4.070544004440308,3.955718421936035,4.18536958694458,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:52,0.8.0 +tiled_geglu,liger_triton,forward,speed,ms,T,sequence length,1024,0.5048320293426514,0.5037440061569214,0.5049984097480774,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:56,0.8.0 +tiled_geglu,liger_triton,forward,speed,ms,T,sequence length,2048,1.0087679624557495,1.0071936130523682,1.0111552000045776,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:56,0.8.0 +tiled_geglu,liger_triton,forward,speed,ms,T,sequence length,4096,1.9712640047073364,1.9705087900161744,1.9721215963363647,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:56,0.8.0 +tiled_geglu,liger_triton,forward,speed,ms,T,sequence length,8192,3.929152011871338,3.929094362258911,3.929209661483765,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:56,0.8.0 +tiled_geglu,liger_tiled,forward,speed,ms,T,sequence length,1024,0.6850879788398743,0.6850048184394837,0.6867519974708557,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:59,0.8.0 +tiled_geglu,liger_tiled,forward,speed,ms,T,sequence length,2048,1.161679983139038,1.1595967769622804,1.1621951818466187,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:59,0.8.0 +tiled_geglu,liger_tiled,forward,speed,ms,T,sequence length,4096,2.306064009666443,2.2182528495788576,2.3586623668670654,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:59,0.8.0 +tiled_geglu,liger_tiled,forward,speed,ms,T,sequence length,8192,4.577343940734863,4.498719882965088,4.6559679985046385,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:45:59,0.8.0 +tiled_geglu,deepspeed_tiled,forward,speed,ms,T,sequence length,1024,0.6830080151557922,0.6829439997673035,0.6831231832504272,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:02,0.8.0 +tiled_geglu,deepspeed_tiled,forward,speed,ms,T,sequence length,2048,1.1566240191459656,1.1555328130722047,1.1580096006393432,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:02,0.8.0 +tiled_geglu,deepspeed_tiled,forward,speed,ms,T,sequence length,4096,2.1637279987335205,2.1629312515258787,2.1653696060180665,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:02,0.8.0 +tiled_geglu,deepspeed_tiled,forward,speed,ms,T,sequence length,8192,4.490368127822876,4.489772891998291,4.49096336364746,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:02,0.8.0 +tiled_geglu,huggingface,backward,speed,ms,T,sequence length,1024,1.1581439971923828,1.1573823690414429,1.247327971458435,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:05,0.8.0 +tiled_geglu,huggingface,backward,speed,ms,T,sequence length,2048,2.309104084968567,2.309011220932007,2.3091968536376952,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:05,0.8.0 +tiled_geglu,huggingface,backward,speed,ms,T,sequence length,4096,4.315152168273926,4.311513805389405,4.318790531158447,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:05,0.8.0 +tiled_geglu,huggingface,backward,speed,ms,T,sequence length,8192,8.095871925354004,8.095871925354004,8.095871925354004,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:05,0.8.0 +tiled_geglu,liger_triton,backward,speed,ms,T,sequence length,1024,1.1361599564552307,1.1331456184387207,1.1372799634933473,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:08,0.8.0 +tiled_geglu,liger_triton,backward,speed,ms,T,sequence length,2048,2.120736002922058,2.1187583923339846,2.122726392745972,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:08,0.8.0 +tiled_geglu,liger_triton,backward,speed,ms,T,sequence length,4096,4.415024042129517,4.358777618408204,4.47127046585083,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:08,0.8.0 +tiled_geglu,liger_triton,backward,speed,ms,T,sequence length,8192,8.21878433227539,8.21878433227539,8.21878433227539,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:08,0.8.0 +tiled_geglu,liger_tiled,backward,speed,ms,T,sequence length,1024,2.50163197517395,2.500383996963501,2.502208042144775,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:11,0.8.0 +tiled_geglu,liger_tiled,backward,speed,ms,T,sequence length,2048,3.91211199760437,3.9111807346344,3.913043260574341,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:11,0.8.0 +tiled_geglu,liger_tiled,backward,speed,ms,T,sequence length,4096,7.28166389465332,7.28166389465332,7.28166389465332,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:11,0.8.0 +tiled_geglu,liger_tiled,backward,speed,ms,T,sequence length,8192,13.307999610900879,13.307999610900879,13.307999610900879,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:11,0.8.0 +tiled_geglu,deepspeed_tiled,backward,speed,ms,T,sequence length,1024,2.6346240043640137,2.633414363861084,2.636582374572754,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:14,0.8.0 +tiled_geglu,deepspeed_tiled,backward,speed,ms,T,sequence length,2048,3.9173120260238647,3.916409587860108,3.918214464187622,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:14,0.8.0 +tiled_geglu,deepspeed_tiled,backward,speed,ms,T,sequence length,4096,7.55836820602417,7.55836820602417,7.55836820602417,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:14,0.8.0 +tiled_geglu,deepspeed_tiled,backward,speed,ms,T,sequence length,8192,14.314559936523438,14.314559936523438,14.314559936523438,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:14,0.8.0 +tiled_geglu,huggingface,full,speed,ms,T,sequence length,1024,1.6824320554733276,1.6824256420135497,1.6830144405364988,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:18,0.8.0 +tiled_geglu,huggingface,full,speed,ms,T,sequence length,2048,3.239039897918701,3.211315155029297,3.3969407558441165,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:18,0.8.0 +tiled_geglu,huggingface,full,speed,ms,T,sequence length,4096,6.583263874053955,6.583263874053955,6.583263874053955,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:18,0.8.0 +tiled_geglu,huggingface,full,speed,ms,T,sequence length,8192,12.791935920715332,12.791935920715332,12.791935920715332,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:18,0.8.0 +tiled_geglu,liger_triton,full,speed,ms,T,sequence length,1024,1.6723519563674927,1.6717824220657347,1.674227237701416,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:21,0.8.0 +tiled_geglu,liger_triton,full,speed,ms,T,sequence length,2048,3.4324800968170166,3.2831615924835207,3.447206497192383,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:21,0.8.0 +tiled_geglu,liger_triton,full,speed,ms,T,sequence length,4096,6.638591766357422,6.638591766357422,6.638591766357422,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:21,0.8.0 +tiled_geglu,liger_triton,full,speed,ms,T,sequence length,8192,13.89465618133545,13.89465618133545,13.89465618133545,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:21,0.8.0 +tiled_geglu,liger_tiled,full,speed,ms,T,sequence length,1024,3.4305601119995117,3.3119232177734377,3.4586881160736085,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:24,0.8.0 +tiled_geglu,liger_tiled,full,speed,ms,T,sequence length,2048,5.587935924530029,5.587935924530029,5.587935924530029,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:24,0.8.0 +tiled_geglu,liger_tiled,full,speed,ms,T,sequence length,4096,9.841695785522461,9.841695785522461,9.841695785522461,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:24,0.8.0 +tiled_geglu,liger_tiled,full,speed,ms,T,sequence length,8192,19.561504364013672,19.561504364013672,19.561504364013672,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:24,0.8.0 +tiled_geglu,deepspeed_tiled,full,speed,ms,T,sequence length,1024,3.3966400623321533,3.3959680080413817,3.4010560512542725,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:27,0.8.0 +tiled_geglu,deepspeed_tiled,full,speed,ms,T,sequence length,2048,5.430431842803955,5.430431842803955,5.430431842803955,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:27,0.8.0 +tiled_geglu,deepspeed_tiled,full,speed,ms,T,sequence length,4096,10.077183723449707,10.077183723449707,10.077183723449707,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:27,0.8.0 +tiled_geglu,deepspeed_tiled,full,speed,ms,T,sequence length,8192,20.271135330200195,20.271135330200195,20.271135330200195,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:46:27,0.8.0 +tiled_geglu,huggingface,full,memory,MB,T,sequence length,1024,1264.0,1264.0,1264.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:46,0.8.0 +tiled_geglu,huggingface,full,memory,MB,T,sequence length,2048,1680.0,1680.0,1680.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:46,0.8.0 +tiled_geglu,huggingface,full,memory,MB,T,sequence length,4096,2560.0,2560.0,2560.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:46,0.8.0 +tiled_geglu,huggingface,full,memory,MB,T,sequence length,8192,4384.0,4384.0,4384.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:46,0.8.0 +tiled_geglu,liger_triton,full,memory,MB,T,sequence length,1024,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:49,0.8.0 +tiled_geglu,liger_triton,full,memory,MB,T,sequence length,2048,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:49,0.8.0 +tiled_geglu,liger_triton,full,memory,MB,T,sequence length,4096,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:49,0.8.0 +tiled_geglu,liger_triton,full,memory,MB,T,sequence length,8192,3152.0,3152.0,3152.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:49,0.8.0 +tiled_geglu,liger_tiled,full,memory,MB,T,sequence length,1024,988.0,988.0,988.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:52,0.8.0 +tiled_geglu,liger_tiled,full,memory,MB,T,sequence length,2048,1128.0,1128.0,1128.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:52,0.8.0 +tiled_geglu,liger_tiled,full,memory,MB,T,sequence length,4096,1408.0,1408.0,1408.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:52,0.8.0 +tiled_geglu,liger_tiled,full,memory,MB,T,sequence length,8192,1968.0,1968.0,1968.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:52,0.8.0 +tiled_geglu,deepspeed_tiled,full,memory,MB,T,sequence length,1024,1002.0,1002.0,1002.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:55,0.8.0 +tiled_geglu,deepspeed_tiled,full,memory,MB,T,sequence length,2048,1156.0,1156.0,1156.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:55,0.8.0 +tiled_geglu,deepspeed_tiled,full,memory,MB,T,sequence length,4096,1464.0,1464.0,1464.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:55,0.8.0 +tiled_geglu,deepspeed_tiled,full,memory,MB,T,sequence length,8192,2080.0,2080.0,2080.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:55,0.8.0 +tiled_geglu,huggingface,forward,memory,MB,T,sequence length,1024,656.0,656.0,656.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:58,0.8.0 +tiled_geglu,huggingface,forward,memory,MB,T,sequence length,2048,912.0,912.0,912.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:58,0.8.0 +tiled_geglu,huggingface,forward,memory,MB,T,sequence length,4096,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:58,0.8.0 +tiled_geglu,huggingface,forward,memory,MB,T,sequence length,8192,2448.0,2448.0,2448.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:47:58,0.8.0 +tiled_geglu,liger_triton,forward,memory,MB,T,sequence length,1024,600.0,600.0,600.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:01,0.8.0 +tiled_geglu,liger_triton,forward,memory,MB,T,sequence length,2048,800.0,800.0,800.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:01,0.8.0 +tiled_geglu,liger_triton,forward,memory,MB,T,sequence length,4096,1200.0,1200.0,1200.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:01,0.8.0 +tiled_geglu,liger_triton,forward,memory,MB,T,sequence length,8192,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:01,0.8.0 +tiled_geglu,liger_tiled,forward,memory,MB,T,sequence length,1024,474.0,474.0,474.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:04,0.8.0 +tiled_geglu,liger_tiled,forward,memory,MB,T,sequence length,2048,548.0,548.0,548.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:04,0.8.0 +tiled_geglu,liger_tiled,forward,memory,MB,T,sequence length,4096,696.0,696.0,696.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:04,0.8.0 +tiled_geglu,liger_tiled,forward,memory,MB,T,sequence length,8192,992.0,992.0,992.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:04,0.8.0 +tiled_geglu,deepspeed_tiled,forward,memory,MB,T,sequence length,1024,470.0,470.0,470.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:07,0.8.0 +tiled_geglu,deepspeed_tiled,forward,memory,MB,T,sequence length,2048,540.0,540.0,540.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:07,0.8.0 +tiled_geglu,deepspeed_tiled,forward,memory,MB,T,sequence length,4096,680.0,680.0,680.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:07,0.8.0 +tiled_geglu,deepspeed_tiled,forward,memory,MB,T,sequence length,8192,960.0,960.0,960.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:07,0.8.0 +tiled_geglu,huggingface,backward,memory,MB,T,sequence length,1024,1264.0,1264.0,1264.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:10,0.8.0 +tiled_geglu,huggingface,backward,memory,MB,T,sequence length,2048,1680.0,1680.0,1680.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:10,0.8.0 +tiled_geglu,huggingface,backward,memory,MB,T,sequence length,4096,2560.0,2560.0,2560.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:10,0.8.0 +tiled_geglu,huggingface,backward,memory,MB,T,sequence length,8192,4384.0,4384.0,4384.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:10,0.8.0 +tiled_geglu,liger_triton,backward,memory,MB,T,sequence length,1024,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:13,0.8.0 +tiled_geglu,liger_triton,backward,memory,MB,T,sequence length,2048,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:13,0.8.0 +tiled_geglu,liger_triton,backward,memory,MB,T,sequence length,4096,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:13,0.8.0 +tiled_geglu,liger_triton,backward,memory,MB,T,sequence length,8192,3152.0,3152.0,3152.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:13,0.8.0 +tiled_geglu,liger_tiled,backward,memory,MB,T,sequence length,1024,988.0,988.0,988.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:15,0.8.0 +tiled_geglu,liger_tiled,backward,memory,MB,T,sequence length,2048,1128.0,1128.0,1128.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:15,0.8.0 +tiled_geglu,liger_tiled,backward,memory,MB,T,sequence length,4096,1408.0,1408.0,1408.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:15,0.8.0 +tiled_geglu,liger_tiled,backward,memory,MB,T,sequence length,8192,1968.0,1968.0,1968.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:15,0.8.0 +tiled_geglu,deepspeed_tiled,backward,memory,MB,T,sequence length,1024,1002.0,1002.0,1002.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:18,0.8.0 +tiled_geglu,deepspeed_tiled,backward,memory,MB,T,sequence length,2048,1156.0,1156.0,1156.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:18,0.8.0 +tiled_geglu,deepspeed_tiled,backward,memory,MB,T,sequence length,4096,1464.0,1464.0,1464.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:18,0.8.0 +tiled_geglu,deepspeed_tiled,backward,memory,MB,T,sequence length,8192,2080.0,2080.0,2080.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:18,0.8.0 +tiled_swiglu,huggingface,forward,speed,ms,T,sequence length,1024,0.5017920136451721,0.49987199902534485,0.5027648091316224,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:22,0.8.0 +tiled_swiglu,huggingface,forward,speed,ms,T,sequence length,2048,1.002560019493103,1.0011392116546631,1.0035200119018555,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:22,0.8.0 +tiled_swiglu,huggingface,forward,speed,ms,T,sequence length,4096,1.950719952583313,1.9495551586151123,1.953228783607483,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:22,0.8.0 +tiled_swiglu,huggingface,forward,speed,ms,T,sequence length,8192,3.9532320499420166,3.952896022796631,3.9535680770874024,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:22,0.8.0 +tiled_swiglu,liger_triton,forward,speed,ms,T,sequence length,1024,0.480320006608963,0.4798207998275757,0.4803839921951294,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:26,0.8.0 +tiled_swiglu,liger_triton,forward,speed,ms,T,sequence length,2048,0.9616000056266785,0.9601791977882386,0.9629952073097229,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:26,0.8.0 +tiled_swiglu,liger_triton,forward,speed,ms,T,sequence length,4096,1.9466240406036377,1.9465472221374513,1.9487423658370973,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:26,0.8.0 +tiled_swiglu,liger_triton,forward,speed,ms,T,sequence length,8192,3.650048017501831,3.6497600555419925,3.65033597946167,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:26,0.8.0 +tiled_swiglu,liger_tiled,forward,speed,ms,T,sequence length,1024,0.652319997549057,0.6522560119628906,0.6543488144874573,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:29,0.8.0 +tiled_swiglu,liger_tiled,forward,speed,ms,T,sequence length,2048,1.1151040196418762,1.1145535945892333,1.1161216020584106,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:29,0.8.0 +tiled_swiglu,liger_tiled,forward,speed,ms,T,sequence length,4096,2.0694079399108887,2.067801570892334,2.0694911003112795,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:29,0.8.0 +tiled_swiglu,liger_tiled,forward,speed,ms,T,sequence length,8192,4.604527950286865,4.585759925842286,4.6232959747314455,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:29,0.8.0 +tiled_swiglu,deepspeed_tiled,forward,speed,ms,T,sequence length,1024,0.6849920153617859,0.6830464005470276,0.68504319190979,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:32,0.8.0 +tiled_swiglu,deepspeed_tiled,forward,speed,ms,T,sequence length,2048,1.1570879817008972,1.155116820335388,1.1865535736083985,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:32,0.8.0 +tiled_swiglu,deepspeed_tiled,forward,speed,ms,T,sequence length,4096,2.160591959953308,2.1526400089263915,2.191609573364258,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:32,0.8.0 +tiled_swiglu,deepspeed_tiled,forward,speed,ms,T,sequence length,8192,4.768847942352295,4.76824312210083,4.76945276260376,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:32,0.8.0 +tiled_swiglu,huggingface,backward,speed,ms,T,sequence length,1024,1.1561919450759888,1.15534725189209,1.15699200630188,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:35,0.8.0 +tiled_swiglu,huggingface,backward,speed,ms,T,sequence length,2048,2.1504160165786743,2.143680047988892,2.151455974578857,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:35,0.8.0 +tiled_swiglu,huggingface,backward,speed,ms,T,sequence length,4096,4.248672008514404,4.246848106384277,4.250495910644531,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:35,0.8.0 +tiled_swiglu,huggingface,backward,speed,ms,T,sequence length,8192,8.73692798614502,8.73692798614502,8.73692798614502,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:35,0.8.0 +tiled_swiglu,liger_triton,backward,speed,ms,T,sequence length,1024,1.1402559876441956,1.1384511947631837,1.2165696382522584,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:38,0.8.0 +tiled_swiglu,liger_triton,backward,speed,ms,T,sequence length,2048,2.272256016731262,2.2308735847473145,2.275200033187866,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:38,0.8.0 +tiled_swiglu,liger_triton,backward,speed,ms,T,sequence length,4096,4.266111850738525,4.266092681884766,4.266131019592285,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:38,0.8.0 +tiled_swiglu,liger_triton,backward,speed,ms,T,sequence length,8192,8.31503963470459,8.31503963470459,8.31503963470459,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:38,0.8.0 +tiled_swiglu,liger_tiled,backward,speed,ms,T,sequence length,1024,2.596832036972046,2.596255970001221,2.598617601394653,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:41,0.8.0 +tiled_swiglu,liger_tiled,backward,speed,ms,T,sequence length,2048,3.934224009513855,3.933024024963379,3.935423994064331,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:41,0.8.0 +tiled_swiglu,liger_tiled,backward,speed,ms,T,sequence length,4096,7.436287879943848,7.436287879943848,7.436287879943848,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:41,0.8.0 +tiled_swiglu,liger_tiled,backward,speed,ms,T,sequence length,8192,13.935808181762695,13.935808181762695,13.935808181762695,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:41,0.8.0 +tiled_swiglu,deepspeed_tiled,backward,speed,ms,T,sequence length,1024,2.51094388961792,2.50896635055542,2.5668927669525146,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:44,0.8.0 +tiled_swiglu,deepspeed_tiled,backward,speed,ms,T,sequence length,2048,4.0017759799957275,4.000537586212158,4.003014373779297,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:44,0.8.0 +tiled_swiglu,deepspeed_tiled,backward,speed,ms,T,sequence length,4096,7.054304122924805,7.054304122924805,7.054304122924805,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:44,0.8.0 +tiled_swiglu,deepspeed_tiled,backward,speed,ms,T,sequence length,8192,14.50710391998291,14.50710391998291,14.50710391998291,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:44,0.8.0 +tiled_swiglu,huggingface,full,speed,ms,T,sequence length,1024,1.6812800168991089,1.6802879571914673,1.6823551893234252,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:47,0.8.0 +tiled_swiglu,huggingface,full,speed,ms,T,sequence length,2048,3.3905279636383057,3.2669183254241942,3.510336017608643,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:47,0.8.0 +tiled_swiglu,huggingface,full,speed,ms,T,sequence length,4096,6.338719844818115,6.338719844818115,6.338719844818115,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:47,0.8.0 +tiled_swiglu,huggingface,full,speed,ms,T,sequence length,8192,13.49942398071289,13.49942398071289,13.49942398071289,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:47,0.8.0 +tiled_swiglu,liger_triton,full,speed,ms,T,sequence length,1024,1.7335679531097412,1.7201151847839355,1.7356608390808104,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:51,0.8.0 +tiled_swiglu,liger_triton,full,speed,ms,T,sequence length,2048,3.3321280479431152,3.3290560245513916,3.3321472167968746,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:51,0.8.0 +tiled_swiglu,liger_triton,full,speed,ms,T,sequence length,4096,6.491199970245361,6.491199970245361,6.491199970245361,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:51,0.8.0 +tiled_swiglu,liger_triton,full,speed,ms,T,sequence length,8192,12.983327865600586,12.983327865600586,12.983327865600586,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:51,0.8.0 +tiled_swiglu,liger_tiled,full,speed,ms,T,sequence length,1024,3.151871919631958,3.151238346099854,3.1555583477020264,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:54,0.8.0 +tiled_swiglu,liger_tiled,full,speed,ms,T,sequence length,2048,5.250607967376709,5.245395183563232,5.2558207511901855,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:54,0.8.0 +tiled_swiglu,liger_tiled,full,speed,ms,T,sequence length,4096,9.709728240966797,9.709728240966797,9.709728240966797,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:54,0.8.0 +tiled_swiglu,liger_tiled,full,speed,ms,T,sequence length,8192,18.8057918548584,18.8057918548584,18.8057918548584,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:54,0.8.0 +tiled_swiglu,deepspeed_tiled,full,speed,ms,T,sequence length,1024,3.202143907546997,3.2013951778411864,3.2045055389404298,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:57,0.8.0 +tiled_swiglu,deepspeed_tiled,full,speed,ms,T,sequence length,2048,5.377056121826172,5.377056121826172,5.377056121826172,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:57,0.8.0 +tiled_swiglu,deepspeed_tiled,full,speed,ms,T,sequence length,4096,9.76476764678955,9.76476764678955,9.76476764678955,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:57,0.8.0 +tiled_swiglu,deepspeed_tiled,full,speed,ms,T,sequence length,8192,19.854368209838867,19.854368209838867,19.854368209838867,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:48:57,0.8.0 +tiled_swiglu,huggingface,full,memory,MB,T,sequence length,1024,1264.0,1264.0,1264.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:00,0.8.0 +tiled_swiglu,huggingface,full,memory,MB,T,sequence length,2048,1680.0,1680.0,1680.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:00,0.8.0 +tiled_swiglu,huggingface,full,memory,MB,T,sequence length,4096,2560.0,2560.0,2560.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:00,0.8.0 +tiled_swiglu,huggingface,full,memory,MB,T,sequence length,8192,4384.0,4384.0,4384.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:00,0.8.0 +tiled_swiglu,liger_triton,full,memory,MB,T,sequence length,1024,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:03,0.8.0 +tiled_swiglu,liger_triton,full,memory,MB,T,sequence length,2048,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:03,0.8.0 +tiled_swiglu,liger_triton,full,memory,MB,T,sequence length,4096,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:03,0.8.0 +tiled_swiglu,liger_triton,full,memory,MB,T,sequence length,8192,3152.0,3152.0,3152.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:03,0.8.0 +tiled_swiglu,liger_tiled,full,memory,MB,T,sequence length,1024,988.0,988.0,988.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:06,0.8.0 +tiled_swiglu,liger_tiled,full,memory,MB,T,sequence length,2048,1128.0,1128.0,1128.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:06,0.8.0 +tiled_swiglu,liger_tiled,full,memory,MB,T,sequence length,4096,1408.0,1408.0,1408.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:06,0.8.0 +tiled_swiglu,liger_tiled,full,memory,MB,T,sequence length,8192,1968.0,1968.0,1968.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:06,0.8.0 +tiled_swiglu,deepspeed_tiled,full,memory,MB,T,sequence length,1024,1002.0,1002.0,1002.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:09,0.8.0 +tiled_swiglu,deepspeed_tiled,full,memory,MB,T,sequence length,2048,1156.0,1156.0,1156.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:09,0.8.0 +tiled_swiglu,deepspeed_tiled,full,memory,MB,T,sequence length,4096,1464.0,1464.0,1464.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:09,0.8.0 +tiled_swiglu,deepspeed_tiled,full,memory,MB,T,sequence length,8192,2080.0,2080.0,2080.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:09,0.8.0 +tiled_swiglu,huggingface,forward,memory,MB,T,sequence length,1024,656.0,656.0,656.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:12,0.8.0 +tiled_swiglu,huggingface,forward,memory,MB,T,sequence length,2048,912.0,912.0,912.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:12,0.8.0 +tiled_swiglu,huggingface,forward,memory,MB,T,sequence length,4096,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:12,0.8.0 +tiled_swiglu,huggingface,forward,memory,MB,T,sequence length,8192,2448.0,2448.0,2448.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:12,0.8.0 +tiled_swiglu,liger_triton,forward,memory,MB,T,sequence length,1024,600.0,600.0,600.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:14,0.8.0 +tiled_swiglu,liger_triton,forward,memory,MB,T,sequence length,2048,800.0,800.0,800.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:14,0.8.0 +tiled_swiglu,liger_triton,forward,memory,MB,T,sequence length,4096,1200.0,1200.0,1200.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:14,0.8.0 +tiled_swiglu,liger_triton,forward,memory,MB,T,sequence length,8192,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:14,0.8.0 +tiled_swiglu,liger_tiled,forward,memory,MB,T,sequence length,1024,474.0,474.0,474.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:17,0.8.0 +tiled_swiglu,liger_tiled,forward,memory,MB,T,sequence length,2048,548.0,548.0,548.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:17,0.8.0 +tiled_swiglu,liger_tiled,forward,memory,MB,T,sequence length,4096,696.0,696.0,696.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:17,0.8.0 +tiled_swiglu,liger_tiled,forward,memory,MB,T,sequence length,8192,992.0,992.0,992.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:17,0.8.0 +tiled_swiglu,deepspeed_tiled,forward,memory,MB,T,sequence length,1024,470.0,470.0,470.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:20,0.8.0 +tiled_swiglu,deepspeed_tiled,forward,memory,MB,T,sequence length,2048,540.0,540.0,540.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:20,0.8.0 +tiled_swiglu,deepspeed_tiled,forward,memory,MB,T,sequence length,4096,680.0,680.0,680.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:20,0.8.0 +tiled_swiglu,deepspeed_tiled,forward,memory,MB,T,sequence length,8192,960.0,960.0,960.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:20,0.8.0 +tiled_swiglu,huggingface,backward,memory,MB,T,sequence length,1024,1264.0,1264.0,1264.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:23,0.8.0 +tiled_swiglu,huggingface,backward,memory,MB,T,sequence length,2048,1680.0,1680.0,1680.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:23,0.8.0 +tiled_swiglu,huggingface,backward,memory,MB,T,sequence length,4096,2560.0,2560.0,2560.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:23,0.8.0 +tiled_swiglu,huggingface,backward,memory,MB,T,sequence length,8192,4384.0,4384.0,4384.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:23,0.8.0 +tiled_swiglu,liger_triton,backward,memory,MB,T,sequence length,1024,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:26,0.8.0 +tiled_swiglu,liger_triton,backward,memory,MB,T,sequence length,2048,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:26,0.8.0 +tiled_swiglu,liger_triton,backward,memory,MB,T,sequence length,4096,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:26,0.8.0 +tiled_swiglu,liger_triton,backward,memory,MB,T,sequence length,8192,3152.0,3152.0,3152.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:26,0.8.0 +tiled_swiglu,liger_tiled,backward,memory,MB,T,sequence length,1024,988.0,988.0,988.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:29,0.8.0 +tiled_swiglu,liger_tiled,backward,memory,MB,T,sequence length,2048,1128.0,1128.0,1128.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:29,0.8.0 +tiled_swiglu,liger_tiled,backward,memory,MB,T,sequence length,4096,1408.0,1408.0,1408.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:29,0.8.0 +tiled_swiglu,liger_tiled,backward,memory,MB,T,sequence length,8192,1968.0,1968.0,1968.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:29,0.8.0 +tiled_swiglu,deepspeed_tiled,backward,memory,MB,T,sequence length,1024,1002.0,1002.0,1002.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:32,0.8.0 +tiled_swiglu,deepspeed_tiled,backward,memory,MB,T,sequence length,2048,1156.0,1156.0,1156.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:32,0.8.0 +tiled_swiglu,deepspeed_tiled,backward,memory,MB,T,sequence length,4096,1464.0,1464.0,1464.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:32,0.8.0 +tiled_swiglu,deepspeed_tiled,backward,memory,MB,T,sequence length,8192,2080.0,2080.0,2080.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:32,0.8.0 +tiled_geglu,liger_cutile,forward,speed,ms,T,sequence length,1024,0.4885600060224533,0.4883583903312683,0.4905791997909546,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:44,0.8.0 +tiled_geglu,liger_cutile,forward,speed,ms,T,sequence length,2048,0.9769279956817627,0.975545597076416,0.9783167958259582,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:44,0.8.0 +tiled_geglu,liger_cutile,forward,speed,ms,T,sequence length,4096,1.901535987854004,1.899564838409424,1.90163836479187,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:44,0.8.0 +tiled_geglu,liger_cutile,forward,speed,ms,T,sequence length,8192,3.8667519092559814,3.7745919227600098,3.958911895751953,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:44,0.8.0 +tiled_geglu,liger_cutile,backward,speed,ms,T,sequence length,1024,1.128928005695343,1.1266303777694704,1.12957444190979,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:56,0.8.0 +tiled_geglu,liger_cutile,backward,speed,ms,T,sequence length,2048,2.195504069328308,2.192825698852539,2.1989120006561276,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:56,0.8.0 +tiled_geglu,liger_cutile,backward,speed,ms,T,sequence length,4096,4.184128046035767,4.181708908081055,4.186547183990479,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:56,0.8.0 +tiled_geglu,liger_cutile,backward,speed,ms,T,sequence length,8192,8.19820785522461,8.19820785522461,8.19820785522461,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:49:56,0.8.0 +tiled_geglu,liger_cutile,full,speed,ms,T,sequence length,1024,1.645535945892334,1.6431872129440308,1.6475263595581056,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:09,0.8.0 +tiled_geglu,liger_cutile,full,speed,ms,T,sequence length,2048,3.2633919715881348,3.261664056777954,3.2640640258789064,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:09,0.8.0 +tiled_geglu,liger_cutile,full,speed,ms,T,sequence length,4096,6.712480068206787,6.712480068206787,6.712480068206787,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:09,0.8.0 +tiled_geglu,liger_cutile,full,speed,ms,T,sequence length,8192,13.7042875289917,13.7042875289917,13.7042875289917,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:09,0.8.0 +tiled_geglu,liger_cutile,full,memory,MB,T,sequence length,1024,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:21,0.8.0 +tiled_geglu,liger_cutile,full,memory,MB,T,sequence length,2048,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:21,0.8.0 +tiled_geglu,liger_cutile,full,memory,MB,T,sequence length,4096,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:21,0.8.0 +tiled_geglu,liger_cutile,full,memory,MB,T,sequence length,8192,3152.0,3152.0,3152.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:21,0.8.0 +tiled_geglu,liger_cutile,forward,memory,MB,T,sequence length,1024,600.0,600.0,600.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:32,0.8.0 +tiled_geglu,liger_cutile,forward,memory,MB,T,sequence length,2048,800.0,800.0,800.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:32,0.8.0 +tiled_geglu,liger_cutile,forward,memory,MB,T,sequence length,4096,1200.0,1200.0,1200.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:32,0.8.0 +tiled_geglu,liger_cutile,forward,memory,MB,T,sequence length,8192,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:32,0.8.0 +tiled_geglu,liger_cutile,backward,memory,MB,T,sequence length,1024,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:44,0.8.0 +tiled_geglu,liger_cutile,backward,memory,MB,T,sequence length,2048,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:44,0.8.0 +tiled_geglu,liger_cutile,backward,memory,MB,T,sequence length,4096,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:44,0.8.0 +tiled_geglu,liger_cutile,backward,memory,MB,T,sequence length,8192,3152.0,3152.0,3152.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""gelu_pytorch_tanh"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""geglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:44,0.8.0 +tiled_swiglu,liger_cutile,forward,speed,ms,T,sequence length,1024,0.48131200671195984,0.4803264081478119,0.484524792432785,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:58,0.8.0 +tiled_swiglu,liger_cutile,forward,speed,ms,T,sequence length,2048,0.9625279903411865,0.9615487933158875,0.9961663961410523,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:58,0.8.0 +tiled_swiglu,liger_cutile,forward,speed,ms,T,sequence length,4096,1.946720004081726,1.8699967861175537,2.0279232025146485,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:58,0.8.0 +tiled_swiglu,liger_cutile,forward,speed,ms,T,sequence length,8192,3.6316800117492676,3.613612794876099,3.6497472286224366,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:50:58,0.8.0 +tiled_swiglu,liger_cutile,backward,speed,ms,T,sequence length,1024,1.1366080045700073,1.13230721950531,1.137606406211853,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:10,0.8.0 +tiled_swiglu,liger_cutile,backward,speed,ms,T,sequence length,2048,2.2547839879989624,2.250188779830933,2.2578432083129885,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:10,0.8.0 +tiled_swiglu,liger_cutile,backward,speed,ms,T,sequence length,4096,4.5061280727386475,4.41973762512207,4.592518520355225,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:10,0.8.0 +tiled_swiglu,liger_cutile,backward,speed,ms,T,sequence length,8192,8.435711860656738,8.435711860656738,8.435711860656738,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:10,0.8.0 +tiled_swiglu,liger_cutile,full,speed,ms,T,sequence length,1024,1.62608003616333,1.6250687837600708,1.6274624347686766,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:23,0.8.0 +tiled_swiglu,liger_cutile,full,speed,ms,T,sequence length,2048,3.1243200302124023,3.1236671447753905,3.260755205154419,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:23,0.8.0 +tiled_swiglu,liger_cutile,full,speed,ms,T,sequence length,4096,6.388864040374756,6.388864040374756,6.388864040374756,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:23,0.8.0 +tiled_swiglu,liger_cutile,full,speed,ms,T,sequence length,8192,13.16659164428711,13.16659164428711,13.16659164428711,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:23,0.8.0 +tiled_swiglu,liger_cutile,full,memory,MB,T,sequence length,1024,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:35,0.8.0 +tiled_swiglu,liger_cutile,full,memory,MB,T,sequence length,2048,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:35,0.8.0 +tiled_swiglu,liger_cutile,full,memory,MB,T,sequence length,4096,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:35,0.8.0 +tiled_swiglu,liger_cutile,full,memory,MB,T,sequence length,8192,3152.0,3152.0,3152.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:35,0.8.0 +tiled_swiglu,liger_cutile,forward,memory,MB,T,sequence length,1024,600.0,600.0,600.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:46,0.8.0 +tiled_swiglu,liger_cutile,forward,memory,MB,T,sequence length,2048,800.0,800.0,800.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:46,0.8.0 +tiled_swiglu,liger_cutile,forward,memory,MB,T,sequence length,4096,1200.0,1200.0,1200.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:46,0.8.0 +tiled_swiglu,liger_cutile,forward,memory,MB,T,sequence length,8192,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:46,0.8.0 +tiled_swiglu,liger_cutile,backward,memory,MB,T,sequence length,1024,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:58,0.8.0 +tiled_swiglu,liger_cutile,backward,memory,MB,T,sequence length,2048,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:58,0.8.0 +tiled_swiglu,liger_cutile,backward,memory,MB,T,sequence length,4096,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:58,0.8.0 +tiled_swiglu,liger_cutile,backward,memory,MB,T,sequence length,8192,3152.0,3152.0,3152.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""hidden_act"": ""silu"", ""dtype"": ""torch.bfloat16"", ""activation_type"": ""swiglu"", ""num_shards"": 4, ""bsz"": 2}",NVIDIA B200,2026-06-26 06:51:58,0.8.0 +multi_token_attention,liger_triton,forward,speed,ms,L,sequence length,32,0.08939200267195702,0.08745600283145905,0.09270399808883667,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:03,0.8.0 +multi_token_attention,liger_triton,forward,speed,ms,L,sequence length,64,0.08803199976682663,0.08643200248479843,0.09101440012454987,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:03,0.8.0 +multi_token_attention,liger_triton,forward,speed,ms,L,sequence length,128,0.1011200025677681,0.09964799880981445,0.10396800190210342,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:03,0.8.0 +multi_token_attention,liger_triton,forward,speed,ms,L,sequence length,256,0.15875199437141418,0.15769599378108978,0.159743994474411,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:03,0.8.0 +multi_token_attention,torch,forward,speed,ms,L,sequence length,32,0.11552000045776367,0.11481600254774094,0.11689600348472595,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:10,0.8.0 +multi_token_attention,torch,forward,speed,ms,L,sequence length,64,0.11577600240707397,0.11497599631547928,0.11664000153541565,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:10,0.8.0 +multi_token_attention,torch,forward,speed,ms,L,sequence length,128,0.1528320014476776,0.15207679569721222,0.1539199948310852,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:10,0.8.0 +multi_token_attention,torch,forward,speed,ms,L,sequence length,256,0.12243200093507767,0.12144000083208084,0.12323839962482452,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:10,0.8.0 +multi_token_attention,liger_triton,backward,speed,ms,L,sequence length,32,0.7507839798927307,0.4801279902458191,0.7928640246391296,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:04,0.8.0 +multi_token_attention,liger_triton,backward,speed,ms,L,sequence length,64,0.7721279859542847,0.4530239999294281,0.8342080116271973,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:04,0.8.0 +multi_token_attention,liger_triton,backward,speed,ms,L,sequence length,128,0.7677760124206543,0.3471232056617737,0.8225152015686036,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:04,0.8.0 +multi_token_attention,liger_triton,backward,speed,ms,L,sequence length,256,0.796319991350174,0.47600001096725464,0.8539648056030273,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:04,0.8.0 +multi_token_attention,torch,backward,speed,ms,L,sequence length,32,0.5251839756965637,0.328000009059906,0.5798720121383667,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:11,0.8.0 +multi_token_attention,torch,backward,speed,ms,L,sequence length,64,0.5860799849033356,0.5460479855537415,0.6246399879455566,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:11,0.8.0 +multi_token_attention,torch,backward,speed,ms,L,sequence length,128,0.5822399854660034,0.4240319907665253,0.6189696192741394,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:11,0.8.0 +multi_token_attention,torch,backward,speed,ms,L,sequence length,256,0.581824004650116,0.3613631904125214,0.6393792152404785,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:11,0.8.0 +multi_token_attention,liger_triton,full,speed,ms,L,sequence length,32,1.0248639583587646,0.7380800127983094,1.0675008058547975,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:05,0.8.0 +multi_token_attention,liger_triton,full,speed,ms,L,sequence length,64,1.053104043006897,0.8824383854866028,1.0941567897796631,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:05,0.8.0 +multi_token_attention,liger_triton,full,speed,ms,L,sequence length,128,1.0816960334777832,0.7878527998924255,1.1202944040298461,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:05,0.8.0 +multi_token_attention,liger_triton,full,speed,ms,L,sequence length,256,1.0908799767494202,0.8984128117561341,1.13047034740448,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:05,0.8.0 +multi_token_attention,torch,full,speed,ms,L,sequence length,32,0.7956799864768982,0.6544320225715636,0.8466816186904907,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,torch,full,speed,ms,L,sequence length,64,0.8275200128555298,0.6856639981269836,0.8772479891777039,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,torch,full,speed,ms,L,sequence length,128,0.856224000453949,0.722495973110199,0.900767982006073,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,torch,full,speed,ms,L,sequence length,256,0.8654080033302307,0.7229696035385131,0.9074367880821228,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,liger_triton,full,memory,MB,L,sequence length,32,0.15966796875,0.15966796875,0.15966796875,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:05,0.8.0 +multi_token_attention,liger_triton,full,memory,MB,L,sequence length,64,2.00439453125,2.00439453125,2.00439453125,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:05,0.8.0 +multi_token_attention,liger_triton,full,memory,MB,L,sequence length,128,19.50439453125,19.50439453125,19.50439453125,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:05,0.8.0 +multi_token_attention,liger_triton,full,memory,MB,L,sequence length,256,80.00439453125,80.00439453125,80.00439453125,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:05,0.8.0 +multi_token_attention,torch,full,memory,MB,L,sequence length,32,0.12890625,0.12890625,0.12890625,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,torch,full,memory,MB,L,sequence length,64,1.94921875,1.94921875,1.94921875,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,torch,full,memory,MB,L,sequence length,128,19.53662109375,19.53662109375,19.53662109375,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,torch,full,memory,MB,L,sequence length,256,79.12939453125,79.12939453125,79.12939453125,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,liger_cutile,forward,speed,ms,L,sequence length,32,0.045504000037908554,0.04416000097990036,0.04851200059056282,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:10,0.8.0 +multi_token_attention,liger_cutile,forward,speed,ms,L,sequence length,64,0.044224001467227936,0.04307200014591217,0.045798398554325104,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:10,0.8.0 +multi_token_attention,liger_cutile,forward,speed,ms,L,sequence length,128,0.05471999943256378,0.0541439987719059,0.05575679987668991,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:10,0.8.0 +multi_token_attention,liger_cutile,forward,speed,ms,L,sequence length,256,0.04371200129389763,0.04278400167822838,0.044704001396894455,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:10,0.8.0 +multi_token_attention,liger_cutile,backward,speed,ms,L,sequence length,32,0.6680319905281067,0.44035841226577777,0.7198336005210877,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:11,0.8.0 +multi_token_attention,liger_cutile,backward,speed,ms,L,sequence length,64,0.6890079975128174,0.37939199805259705,0.7395840287208557,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:11,0.8.0 +multi_token_attention,liger_cutile,backward,speed,ms,L,sequence length,128,0.6905919909477234,0.39688960313797,0.7461696147918702,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:11,0.8.0 +multi_token_attention,liger_cutile,backward,speed,ms,L,sequence length,256,0.7213119864463806,0.4784767866134644,0.7704576015472413,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:11,0.8.0 +multi_token_attention,liger_cutile,full,speed,ms,L,sequence length,32,0.8794879913330078,0.7299584031105042,0.921728003025055,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,liger_cutile,full,speed,ms,L,sequence length,64,0.9089279770851135,0.6498112082481384,0.9593087911605835,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,liger_cutile,full,speed,ms,L,sequence length,128,0.9290239810943604,0.6826239824295044,0.9699199795722961,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,liger_cutile,full,speed,ms,L,sequence length,256,0.9290399849414825,0.6150720238685609,0.9707648038864136,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,liger_cutile,full,memory,MB,L,sequence length,32,0.1435546875,0.1435546875,0.1435546875,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,liger_cutile,full,memory,MB,L,sequence length,64,2.00390625,2.00390625,2.00390625,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,liger_cutile,full,memory,MB,L,sequence length,128,19.50439453125,19.50439453125,19.50439453125,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +multi_token_attention,liger_cutile,full,memory,MB,L,sequence length,256,80.00439453125,80.00439453125,80.00439453125,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 diff --git a/src/liger_kernel/ops/cutile/ops/__init__.py b/src/liger_kernel/ops/cutile/ops/__init__.py index 4967c6daf..dd6a36a55 100644 --- a/src/liger_kernel/ops/cutile/ops/__init__.py +++ b/src/liger_kernel/ops/cutile/ops/__init__.py @@ -21,12 +21,25 @@ from liger_kernel.ops.cutile.ops.geglu import LigerGELUMulFunction from liger_kernel.ops.cutile.ops.geglu import geglu_backward from liger_kernel.ops.cutile.ops.geglu import geglu_forward +from liger_kernel.ops.cutile.ops.group_norm import LigerGroupNormFunction +from liger_kernel.ops.cutile.ops.group_norm import group_norm_backward +from liger_kernel.ops.cutile.ops.group_norm import group_norm_forward from liger_kernel.ops.cutile.ops.jsd import LigerJSDFunction from liger_kernel.ops.cutile.ops.jsd import jsd_backward from liger_kernel.ops.cutile.ops.jsd import jsd_forward +from liger_kernel.ops.cutile.ops.kl_div import LigerKLDivLossFunction from liger_kernel.ops.cutile.ops.layer_norm import LigerLayerNormFunction from liger_kernel.ops.cutile.ops.layer_norm import layer_norm_backward from liger_kernel.ops.cutile.ops.layer_norm import layer_norm_forward +from liger_kernel.ops.cutile.ops.llama4_rope import LigerLlama4RopeFunction +from liger_kernel.ops.cutile.ops.multi_token_attention import LigerMultiTokenAttentionFunction +from liger_kernel.ops.cutile.ops.qwen2vl_mrope import LigerQwen2VLMRopeFunction +from liger_kernel.ops.cutile.ops.rope import LigerRopeFunction +from liger_kernel.ops.cutile.ops.rope import rope_backward +from liger_kernel.ops.cutile.ops.rope import rope_forward +from liger_kernel.ops.cutile.ops.sparsemax import LigerSparsemaxFunction +from liger_kernel.ops.cutile.ops.tiled_mlp import LigerTiledMLPFunction +from liger_kernel.ops.cutile.ops.tiled_mlp import apply_tiled_mlp __all__ = [ "LigerCrossEntropyFunction", @@ -38,10 +51,23 @@ "LigerGELUMulFunction", "geglu_backward", "geglu_forward", + "LigerGroupNormFunction", + "group_norm_backward", + "group_norm_forward", "LigerJSDFunction", "jsd_backward", "jsd_forward", + "LigerKLDivLossFunction", "LigerLayerNormFunction", "layer_norm_backward", "layer_norm_forward", + "LigerLlama4RopeFunction", + "LigerMultiTokenAttentionFunction", + "LigerQwen2VLMRopeFunction", + "LigerRopeFunction", + "rope_backward", + "rope_forward", + "LigerSparsemaxFunction", + "LigerTiledMLPFunction", + "apply_tiled_mlp", ] diff --git a/src/liger_kernel/ops/cutile/ops/group_norm.py b/src/liger_kernel/ops/cutile/ops/group_norm.py new file mode 100644 index 000000000..5e15435e0 --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/group_norm.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +Group Normalization kernel (CuTile backend). +""" + +import cuda.tile as ct +import torch + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 +from liger_kernel.ops.utils import ensure_contiguous + +MAX_FUSED_SIZE = 65536 + + +@ct.kernel +def _group_norm_fwd_kernel_ct( + x_input, + y_output, + weight, + bias, + mean_stats, + rstd_stats, + NUM_CHANNELS: ct.Constant[int], + NUM_GROUPS: ct.Constant[int], + CHANNELS_PER_GROUP: ct.Constant[int], + TOTAL_HIDDEN_SIZE: ct.Constant[int], + eps, + BLOCK_SIZE: ct.Constant[int], +): + batch_idx = ct.bid(0) + group_idx = ct.bid(1) + + group_row = batch_idx * NUM_GROUPS + group_idx + + N = CHANNELS_PER_GROUP * TOTAL_HIDDEN_SIZE + inv_N = 1.0 / N + + num_h_chunks = (TOTAL_HIDDEN_SIZE + BLOCK_SIZE - 1) // BLOCK_SIZE + + sum_tile = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + sum_sq_tile = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + + for c_in_group in range(CHANNELS_PER_GROUP): + channel_idx = group_idx * CHANNELS_PER_GROUP + c_in_group + row_idx = batch_idx * NUM_CHANNELS + channel_idx + + for hi in range(num_h_chunks): + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + hi * BLOCK_SIZE + x_tile = ct.astype( + ct.gather(x_input, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + sum_tile = sum_tile + x_tile + sum_sq_tile = sum_sq_tile + x_tile * x_tile + + s = ct.sum(sum_tile, 0, keepdims=False) + sq = ct.sum(sum_sq_tile, 0, keepdims=False) + mean = s * inv_N + variance = sq * inv_N - mean * mean + rstd = ct.rsqrt(variance + eps) + + ct.scatter(mean_stats, group_row, ct.astype(mean, mean_stats.dtype)) + ct.scatter(rstd_stats, group_row, ct.astype(rstd, rstd_stats.dtype)) + + for c_in_group in range(CHANNELS_PER_GROUP): + channel_idx = group_idx * CHANNELS_PER_GROUP + c_in_group + row_idx = batch_idx * NUM_CHANNELS + channel_idx + + w_scalar = ct.astype(ct.load(weight, channel_idx, shape=()), ct.float32) + b_scalar = ct.astype(ct.load(bias, channel_idx, shape=()), ct.float32) + + for hi in range(num_h_chunks): + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + hi * BLOCK_SIZE + x_tile = ct.astype( + ct.gather(x_input, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + y_tile = (x_tile - mean) * rstd * w_scalar + b_scalar + ct.scatter(y_output, (row_idx, col_idx), ct.astype(y_tile, y_output.dtype), check_bounds=True) + + +@ct.kernel +def _group_norm_bwd_kernel_ct( + x_input, + upstream, + weight, + mean_stats, + rstd_stats, + dx_output, + dw_partial, + db_partial, + NUM_CHANNELS: ct.Constant[int], + NUM_GROUPS: ct.Constant[int], + CHANNELS_PER_GROUP: ct.Constant[int], + TOTAL_HIDDEN_SIZE: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], +): + batch_idx = ct.bid(0) + group_idx = ct.bid(1) + + group_row = batch_idx * NUM_GROUPS + group_idx + + mean = ct.astype(ct.load(mean_stats, group_row, shape=()), ct.float32) + rstd = ct.astype(ct.load(rstd_stats, group_row, shape=()), ct.float32) + + N = CHANNELS_PER_GROUP * TOTAL_HIDDEN_SIZE + inv_N = 1.0 / N + + num_h_chunks = (TOTAL_HIDDEN_SIZE + BLOCK_SIZE - 1) // BLOCK_SIZE + + c1_tile = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + c2_tile = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + + for c_in_group in range(CHANNELS_PER_GROUP): + channel_idx = group_idx * CHANNELS_PER_GROUP + c_in_group + row_idx = batch_idx * NUM_CHANNELS + channel_idx + + w_scalar = ct.astype(ct.load(weight, channel_idx, shape=()), ct.float32) + + dW_acc_tile = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + dB_acc_tile = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + + for hi in range(num_h_chunks): + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + hi * BLOCK_SIZE + x_tile = ct.astype( + ct.gather(x_input, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + upstream_tile = ct.astype( + ct.gather(upstream, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + x_hat = (x_tile - mean) * rstd + wdy = w_scalar * upstream_tile + c1_tile = c1_tile + x_hat * wdy + c2_tile = c2_tile + wdy + dW_acc_tile = dW_acc_tile + upstream_tile * x_hat + dB_acc_tile = dB_acc_tile + upstream_tile + + dW_val = ct.sum(dW_acc_tile, 0, keepdims=False) + dB_val = ct.sum(dB_acc_tile, 0, keepdims=False) + ct.scatter(dw_partial, (batch_idx, channel_idx), ct.astype(dW_val, dw_partial.dtype)) + ct.scatter(db_partial, (batch_idx, channel_idx), ct.astype(dB_val, db_partial.dtype)) + + c1 = ct.sum(c1_tile, 0, keepdims=False) * inv_N + c2 = ct.sum(c2_tile, 0, keepdims=False) * inv_N + + for c_in_group in range(CHANNELS_PER_GROUP): + channel_idx = group_idx * CHANNELS_PER_GROUP + c_in_group + row_idx = batch_idx * NUM_CHANNELS + channel_idx + + w_scalar = ct.astype(ct.load(weight, channel_idx, shape=()), ct.float32) + + for hi in range(num_h_chunks): + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + hi * BLOCK_SIZE + x_tile = ct.astype( + ct.gather(x_input, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + upstream_tile = ct.astype( + ct.gather(upstream, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + x_hat = (x_tile - mean) * rstd + wdy = w_scalar * upstream_tile + dx = (wdy - (x_hat * c1 + c2)) * rstd + ct.scatter(dx_output, (row_idx, col_idx), ct.astype(dx, dx_output.dtype), check_bounds=True) + + +def group_norm_forward(X, num_channels, num_groups, W, B, eps): + shape = X.shape + batch_size = shape[0] + channels_per_group = num_channels // num_groups + hidden_size = X.shape[-1] + + BLOCK_SIZE = min(MAX_FUSED_SIZE, _next_power_of_2(hidden_size)) + + X_2d = X.view(batch_size * num_channels, hidden_size).contiguous() + Y_2d = torch.empty_like(X_2d) + # Stats kept in fp32 (matches upstream Liger). bf16 stats round-trip through + # forward → backward and lose precision in the (x - mean) * rstd step. + mean_stats = torch.empty(batch_size * num_groups, dtype=torch.float32, device=X.device) + rstd_stats = torch.empty(batch_size * num_groups, dtype=torch.float32, device=X.device) + + grid = (batch_size, num_groups, 1) + ct.launch( + torch.cuda.current_stream(), + grid, + _group_norm_fwd_kernel_ct, + ( + X_2d, + Y_2d, + W, + B, + mean_stats, + rstd_stats, + int(num_channels), + int(num_groups), + int(channels_per_group), + int(hidden_size), + float(eps), + int(BLOCK_SIZE), + ), + ) + + return Y_2d.view(*shape), X_2d, mean_stats, rstd_stats, BLOCK_SIZE + + +def group_norm_backward(dY, X_2d, W, B, Mean, RSTD, num_channels, num_groups): + shape = dY.shape + batch_size = shape[0] + hidden_size = shape[-1] + channels_per_group = num_channels // num_groups + BLOCK_SIZE = min(MAX_FUSED_SIZE, _next_power_of_2(hidden_size)) + + dY_2d = dY.contiguous().view(batch_size * num_channels, hidden_size).contiguous() + + dx_2d = torch.empty_like(X_2d) + dw_partial = torch.zeros(batch_size, num_channels, dtype=W.dtype, device=W.device) + db_partial = torch.zeros(batch_size, num_channels, dtype=B.dtype, device=B.device) + + grid = (batch_size, num_groups, 1) + ct.launch( + torch.cuda.current_stream(), + grid, + _group_norm_bwd_kernel_ct, + ( + X_2d, + dY_2d, + W, + Mean, + RSTD, + dx_2d, + dw_partial, + db_partial, + int(num_channels), + int(num_groups), + int(channels_per_group), + int(hidden_size), + int(BLOCK_SIZE), + ), + ) + + dw = dw_partial.sum(dim=0) + db = db_partial.sum(dim=0) + return dx_2d.view(*shape), dw, db + + +class LigerGroupNormFunction(torch.autograd.Function): + @staticmethod + @ensure_contiguous + def forward( + ctx, + X, + affine_scaling_weight, + affine_shifting_bias, + num_channels, + num_groups, + eps, + ): + Y, X_2d, Mean, RSTD, BLOCK_SIZE = group_norm_forward( + X, + num_channels, + num_groups, + affine_scaling_weight, + affine_shifting_bias, + eps, + ) + ctx.num_channels = num_channels + ctx.num_groups = num_groups + ctx.save_for_backward(X_2d, affine_scaling_weight, affine_shifting_bias, Mean, RSTD) + return Y + + @staticmethod + @ensure_contiguous + def backward(ctx, dY): + X_2d, W, B, Mean, RSTD = ctx.saved_tensors + DX, DW, DB = group_norm_backward(dY, X_2d, W, B, Mean, RSTD, ctx.num_channels, ctx.num_groups) + return DX, DW, DB, None, None, None diff --git a/src/liger_kernel/ops/cutile/ops/kl_div.py b/src/liger_kernel/ops/cutile/ops/kl_div.py new file mode 100644 index 000000000..995a1c50a --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/kl_div.py @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +KL Divergence loss kernel (CuTile backend). + +Computes KL(y_true || y_pred) where y_pred is in log-space. +""" + +import cuda.tile as ct +import torch + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 + +MAX_FUSED_SIZE = 4096 + +_REDUCTION_MODE_NONE = 0 +_REDUCTION_MODE_SUM = 1 +_REDUCTION_MODE_MEAN = 2 +_REDUCTION_MODE_BATCHMEAN = 3 + +_str_to_reduction_mode = { + "none": _REDUCTION_MODE_NONE, + "sum": _REDUCTION_MODE_SUM, + "mean": _REDUCTION_MODE_MEAN, + "batchmean": _REDUCTION_MODE_BATCHMEAN, +} + + +@ct.kernel +def _kldiv_fwd_none_kernel_ct( + Y, + GT, + LOSS, + n_cols: ct.Constant[int], + eps: ct.Constant[float], + BLOCK_SIZE: ct.Constant[int], + LOG_TARGET: ct.Constant[int], + N_FULL_CHUNKS: ct.Constant[int], +): + row_idx = ct.bid(0) + eps_tile = ct.full((BLOCK_SIZE,), eps, dtype=ct.float32) + + for ci in range(N_FULL_CHUNKS): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + y = ct.astype(ct.gather(Y, (row_idx, col_idx), check_bounds=False), ct.float32) + gt = ct.astype(ct.gather(GT, (row_idx, col_idx), check_bounds=False), ct.float32) + + if LOG_TARGET: + loss = ct.exp(gt) * (gt - y) + else: + gt_clipped = ct.maximum(gt, eps_tile) + loss = gt * (ct.log(gt_clipped) - y) + + ct.scatter(LOSS, (row_idx, col_idx), ct.astype(loss, LOSS.dtype), check_bounds=False) + + if N_FULL_CHUNKS * BLOCK_SIZE < n_cols: + ci = N_FULL_CHUNKS + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + y = ct.astype(ct.gather(Y, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + gt = ct.astype(ct.gather(GT, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + + if LOG_TARGET: + loss = ct.exp(gt) * (gt - y) + else: + gt_clipped = ct.maximum(gt, eps_tile) + loss = gt * (ct.log(gt_clipped) - y) + + ct.scatter(LOSS, (row_idx, col_idx), ct.astype(loss, LOSS.dtype), check_bounds=True) + + +@ct.kernel +def _kldiv_fwd_reduce_kernel_ct( + Y, + GT, + LOSS, + n_cols: ct.Constant[int], + eps: ct.Constant[float], + BLOCK_SIZE: ct.Constant[int], + LOG_TARGET: ct.Constant[int], + N_FULL_CHUNKS: ct.Constant[int], +): + row_idx = ct.bid(0) + + loss_acc = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + eps_tile = ct.full((BLOCK_SIZE,), eps, dtype=ct.float32) + + for ci in range(N_FULL_CHUNKS): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + y = ct.astype(ct.gather(Y, (row_idx, col_idx), check_bounds=False), ct.float32) + gt = ct.astype(ct.gather(GT, (row_idx, col_idx), check_bounds=False), ct.float32) + + if LOG_TARGET: + loss = ct.exp(gt) * (gt - y) + else: + gt_clipped = ct.maximum(gt, eps_tile) + loss = gt * (ct.log(gt_clipped) - y) + + loss_acc = ct.add(loss_acc, loss) + + if N_FULL_CHUNKS * BLOCK_SIZE < n_cols: + ci = N_FULL_CHUNKS + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + y = ct.astype(ct.gather(Y, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + gt = ct.astype(ct.gather(GT, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + + if LOG_TARGET: + loss = ct.exp(gt) * (gt - y) + else: + gt_clipped = ct.maximum(gt, eps_tile) + loss = gt * (ct.log(gt_clipped) - y) + + loss_acc = ct.add(loss_acc, loss) + + row_sum = ct.sum(loss_acc, 0, keepdims=False) + ct.scatter(LOSS, row_idx, ct.astype(row_sum, LOSS.dtype)) + + +@ct.kernel +def _kldiv_bwd_kernel_ct( + GT, + GRADS, + n_cols: ct.Constant[int], + scale, # runtime: depends on grad_output.item() — making this constexpr triggers a + # JIT recompile on every backward when the upstream grad changes (e.g. in autograd + # benchmarks that draw a fresh torch.randn_like(loss) per iteration). + BLOCK_SIZE: ct.Constant[int], + LOG_TARGET: ct.Constant[int], + N_FULL_CHUNKS: ct.Constant[int], +): + row_idx = ct.bid(0) + + for ci in range(N_FULL_CHUNKS): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + gt = ct.astype(ct.gather(GT, (row_idx, col_idx), check_bounds=False), ct.float32) + + if LOG_TARGET: + res = -ct.exp(gt) * scale + else: + res = -gt * scale + + ct.scatter(GRADS, (row_idx, col_idx), ct.astype(res, GRADS.dtype), check_bounds=False) + + if N_FULL_CHUNKS * BLOCK_SIZE < n_cols: + ci = N_FULL_CHUNKS + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + gt = ct.astype(ct.gather(GT, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + + if LOG_TARGET: + res = -ct.exp(gt) * scale + else: + res = -gt * scale + + ct.scatter(GRADS, (row_idx, col_idx), ct.astype(res, GRADS.dtype), check_bounds=True) + + +def _kldiv_forward(y_pred, y_true, log_target, reduction, eps): + BT, V = y_pred.shape + BLOCK_SIZE = min(MAX_FUSED_SIZE, _next_power_of_2(V)) + reduction_int = _str_to_reduction_mode[reduction] + n_full_chunks = V // BLOCK_SIZE + + grid = (BT, 1, 1) + + if reduction_int == _REDUCTION_MODE_NONE: + output_tensor = torch.zeros(BT, V, device=y_pred.device, dtype=torch.float32) + ct.launch( + torch.cuda.current_stream(), + grid, + _kldiv_fwd_none_kernel_ct, + ( + y_pred, + y_true, + output_tensor, + int(V), + float(eps), + int(BLOCK_SIZE), + int(log_target), + int(n_full_chunks), + ), + ) + return output_tensor + else: + row_sums = torch.zeros(BT, device=y_pred.device, dtype=torch.float32) + ct.launch( + torch.cuda.current_stream(), + grid, + _kldiv_fwd_reduce_kernel_ct, + ( + y_pred, + y_true, + row_sums, + int(V), + float(eps), + int(BLOCK_SIZE), + int(log_target), + int(n_full_chunks), + ), + ) + if reduction_int == _REDUCTION_MODE_BATCHMEAN: + return row_sums.sum() / BT + elif reduction_int == _REDUCTION_MODE_SUM: + return row_sums.sum(dim=0) + else: # mean + return row_sums.sum() / (BT * V) + + +def _kldiv_backward(y_true, scale, log_target): + BT, V = y_true.shape + BLOCK_SIZE = min(MAX_FUSED_SIZE, _next_power_of_2(V)) + n_full_chunks = V // BLOCK_SIZE + + new_grads = torch.empty_like(y_true) + grid = (BT, 1, 1) + ct.launch( + torch.cuda.current_stream(), + grid, + _kldiv_bwd_kernel_ct, + ( + y_true, + new_grads, + int(V), + float(scale), + int(BLOCK_SIZE), + int(log_target), + int(n_full_chunks), + ), + ) + + return new_grads + + +class LigerKLDivLossFunction(torch.autograd.Function): + """CuTile autograd wrapper for KL divergence loss.""" + + @staticmethod + def forward(ctx, y_pred, y_true, reduction, log_target, eps): + y_pred = y_pred.contiguous() + y_true = y_true.contiguous() + ctx.save_for_backward(y_true) + ctx.reduction = reduction + ctx.log_target = log_target + return _kldiv_forward(y_pred, y_true, log_target, reduction, eps) + + @staticmethod + def backward(ctx, grad_output): + (y_true,) = ctx.saved_tensors + BT, V = y_true.shape + + if grad_output.numel() == 1: + scale = grad_output.item() + else: + scale = 1.0 + + if ctx.reduction == "batchmean": + scale /= BT + elif ctx.reduction == "mean": + scale /= BT * V + + derivative = _kldiv_backward(y_true, scale, ctx.log_target) + + if grad_output.numel() != 1: + derivative = derivative * grad_output + + return derivative, None, None, None, None diff --git a/src/liger_kernel/ops/cutile/ops/llama4_rope.py b/src/liger_kernel/ops/cutile/ops/llama4_rope.py new file mode 100644 index 000000000..9861e3cc4 --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/llama4_rope.py @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +Llama4-style Rotary Position Embedding (RoPE) kernel (CuTile backend). + +Applies in-place complex multiplication: (q_r + i*q_i) * (f_r + i*f_i). + +Grid: (batch_size, seq_len, n_heads_max) — one block per (batch, seq, head). + +Interleaved layout: q[b, s, h, 2*d] = real part, q[b, s, h, 2*d+1] = imaginary part. +We construct the stride-2 index pairs using: + base = ct.arange(BLOCK_SIZE) + doubled = base + base # [0, 2, 4, ..., 2*(BLOCK_SIZE-1)] + real_idx = doubled + d_start*2 + imag_idx = real_idx + 1 + +q and k are passed as 2D views (B*S*H, head_dim) for simpler indexing. +freqs_cis is passed as (S, head_dim) after view_as_real + reshape. +""" + +import cuda.tile as ct +import torch + + +def _select_block_size(head_dim_half: int) -> int: + if head_dim_half >= 256: + return 128 + if head_dim_half >= 96: + return 128 + if head_dim_half >= 48: + return 64 + if head_dim_half >= 24: + return 32 + return 16 + + +@ct.kernel +def _llama4_rope_kernel_ct( + query, # (B*S*H_q, head_dim) query — modified in-place + key, # (B*S*H_k, head_dim) key — modified in-place + freqs, # (S, head_dim) frequencies (view_as_real, flattened last 2 dims) + seq_len, + HEAD_DIM_HALF: ct.Constant[int], + N_Q_HEADS: ct.Constant[int], + N_K_HEADS: ct.Constant[int], + imag_sign, + BLOCK_SIZE: ct.Constant[int], +): + """ + RoPE kernel. + + Grid: (batch_size, seq_len, n_heads_max). + One block per (batch, seq, head) position. + + For each d-block, loads BLOCK_SIZE (real, imag) pairs from Q/K and FREQS, + computes complex multiplication, and stores back in-place. + + Index construction (stride-2 interleaved): + base = arange(BLOCK_SIZE) # [0, 1, ..., BLOCK_SIZE-1] + doubled = base + base # [0, 2, 4, ..., 2*(BLOCK_SIZE-1)] + real_idx = doubled + d_start*2 # real column indices + imag_idx = real_idx + 1 # imag column indices + """ + batch_idx = ct.bid(0) + seq_idx = ct.bid(1) + pid_h = ct.bid(2) + + # Number of BLOCK_SIZE blocks over head_dim_half + n_d_blocks = (HEAD_DIM_HALF + BLOCK_SIZE - 1) // BLOCK_SIZE + + for di in range(n_d_blocks): + d_start = di * BLOCK_SIZE + + # Build interleaved column indices for real/imag parts + base = ct.arange(BLOCK_SIZE, dtype=ct.int32) + doubled = base + base # [0, 2, 4, ..., 2*(BLOCK_SIZE-1)] + real_idx = doubled + d_start * 2 # real column indices in (seq, head_dim) + imag_idx = real_idx + 1 # imag column indices + + # Load frequencies for this seq position and d-block + f_r = ct.astype( + ct.gather(freqs, (seq_idx, real_idx), check_bounds=True, padding_value=0.0, latency=3), + ct.float32, + ) + f_i = ct.astype( + ct.gather(freqs, (seq_idx, imag_idx), check_bounds=True, padding_value=0.0, latency=3), + ct.float32, + ) + f_i = f_i * imag_sign + + # Process query head + if pid_h < N_Q_HEADS: + q_row = batch_idx * seq_len * N_Q_HEADS + seq_idx * N_Q_HEADS + pid_h + q_r = ct.astype( + ct.gather(query, (q_row, real_idx), check_bounds=True, padding_value=0.0, latency=3), ct.float32 + ) + q_i = ct.astype( + ct.gather(query, (q_row, imag_idx), check_bounds=True, padding_value=0.0, latency=3), ct.float32 + ) + + # Complex multiply: (q_r + i*q_i) * (f_r + i*f_i) + new_q_r = q_r * f_r - q_i * f_i + new_q_i = q_r * f_i + q_i * f_r + + ct.scatter(query, (q_row, real_idx), ct.astype(new_q_r, query.dtype), check_bounds=True) + ct.scatter(query, (q_row, imag_idx), ct.astype(new_q_i, query.dtype), check_bounds=True) + + # Process key head + if pid_h < N_K_HEADS: + k_row = batch_idx * seq_len * N_K_HEADS + seq_idx * N_K_HEADS + pid_h + k_r = ct.astype( + ct.gather(key, (k_row, real_idx), check_bounds=True, padding_value=0.0, latency=3), ct.float32 + ) + k_i = ct.astype( + ct.gather(key, (k_row, imag_idx), check_bounds=True, padding_value=0.0, latency=3), ct.float32 + ) + + new_k_r = k_r * f_r - k_i * f_i + new_k_i = k_r * f_i + k_i * f_r + + ct.scatter(key, (k_row, real_idx), ct.astype(new_k_r, key.dtype), check_bounds=True) + ct.scatter(key, (k_row, imag_idx), ct.astype(new_k_i, key.dtype), check_bounds=True) + + +def _llama4_rope_forward_ct(q, k, freqs_cis, BLOCK_SIZE=None, imag_sign=1.0): + original_dtype = q.dtype + + batch_size, seq_len, n_q_heads, head_dim = q.shape + _, _, n_k_heads, _ = k.shape + head_dim_half = head_dim // 2 + + # Normalize freqs_cis to (seq_len, head_dim) real layout + if freqs_cis.is_complex(): + freqs_cis = freqs_cis.reshape(-1, freqs_cis.shape[-1]) + if freqs_cis.shape[0] > seq_len: + freqs_cis = freqs_cis[:seq_len] + freqs_cis = torch.view_as_real(freqs_cis) # (seq_len, head_dim_half, 2) + + if freqs_cis.ndim == 3: + # (seq_len, head_dim_half, 2) → (seq_len, head_dim) + freqs_cis = freqs_cis.reshape(freqs_cis.shape[0], -1) + + compute_dtype = torch.float32 if q.dtype == torch.float32 else q.dtype + if k.dtype != q.dtype: + k = k.to(q.dtype) + q = q.to(compute_dtype).contiguous() + k = k.to(compute_dtype).contiguous() + freqs_cis = freqs_cis.float().contiguous() + + if BLOCK_SIZE is None: + BLOCK_SIZE = _select_block_size(head_dim_half) + + # Reshape to 2D for the kernel: (B*S*H, head_dim) + q_2d = q.reshape(batch_size * seq_len * n_q_heads, head_dim).contiguous() + k_2d = k.reshape(batch_size * seq_len * n_k_heads, head_dim).contiguous() + + n_heads_max = max(n_q_heads, n_k_heads) + grid = (batch_size, seq_len, n_heads_max) + + ct.launch( + torch.cuda.current_stream(), + grid, + _llama4_rope_kernel_ct, + ( + q_2d, + k_2d, + freqs_cis, + int(seq_len), + int(head_dim_half), + int(n_q_heads), + int(n_k_heads), + float(imag_sign), + int(BLOCK_SIZE), + ), + ) + + q_out = q_2d.reshape(batch_size, seq_len, n_q_heads, head_dim) + k_out = k_2d.reshape(batch_size, seq_len, n_k_heads, head_dim) + + if q_out.dtype != original_dtype: + q_out = q_out.to(original_dtype) + if k_out.dtype != original_dtype: + k_out = k_out.to(original_dtype) + + return q_out, k_out + + +class LigerLlama4RopeFunction(torch.autograd.Function): + """CuTile autograd wrapper for Llama4 RoPE.""" + + @staticmethod + def forward(ctx, q, k, freqs_cis, BLOCK_SIZE=None): + q_out, k_out = _llama4_rope_forward_ct(q, k, freqs_cis, BLOCK_SIZE, imag_sign=1.0) + ctx.save_for_backward(freqs_cis.detach() if isinstance(freqs_cis, torch.Tensor) else freqs_cis) + ctx.BLOCK_SIZE = BLOCK_SIZE + return q_out, k_out + + @staticmethod + def backward(ctx, dq, dk): + (freqs_cis,) = ctx.saved_tensors + BLOCK_SIZE = getattr(ctx, "BLOCK_SIZE", None) + dq_out, dk_out = _llama4_rope_forward_ct(dq, dk, freqs_cis, BLOCK_SIZE, imag_sign=-1.0) + return dq_out, dk_out, None, None diff --git a/src/liger_kernel/ops/cutile/ops/multi_token_attention.py b/src/liger_kernel/ops/cutile/ops/multi_token_attention.py new file mode 100644 index 000000000..7c70c6a62 --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/multi_token_attention.py @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT +import cuda.tile as ct +import torch +import torch.nn.functional as F + +from torch.nn.modules.utils import _pair + +from liger_kernel.ops.cutile.ops.sparsemax import _sparsemax_backward as _sparsemax_backward_ct +from liger_kernel.ops.cutile.ops.sparsemax import _sparsemax_forward as _sparsemax_forward_ct +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 + +_MASK_INF_VAL = -1e9 # large negative; -inf breaks multiply-accumulate pattern ((-inf)*0 = NaN) + + +def _select_block_size(L: int) -> int: + return min(_next_power_of_2(L), 128) + + +@ct.kernel +def _mask_inf_fwd_kernel_ct( + scores_2d, + output_2d, + L: ct.Constant[int], + BLOCK: ct.Constant[int], +): + actual_row = ct.bid(0) + batch_id = ct.bid(1) + row_idx = batch_id * L + actual_row + n_chunks = (L + BLOCK - 1) // BLOCK + + for ci in range(n_chunks): + col_start = ci * BLOCK + col_idx = ct.arange(BLOCK, dtype=ct.int32) + col_start + src_tile = ct.load(scores_2d, index=(row_idx, ci), shape=(1, BLOCK), padding_mode=ct.PaddingMode.ZERO).reshape( + (BLOCK,) + ) + is_future_f = ct.astype(col_idx > actual_row, ct.float32) + is_past_f = ct.astype(col_idx <= actual_row, ct.float32) + out_tile = ( + ct.astype(src_tile, ct.float32) * is_past_f + ct.full((BLOCK,), _MASK_INF_VAL, ct.float32) * is_future_f + ) + ct.store(output_2d, index=(row_idx, ci), tile=ct.astype(out_tile, output_2d.dtype).reshape((1, BLOCK))) + + +@ct.kernel +def _mask_zero_fwd_kernel_ct( + scores_2d, + output_2d, + L: ct.Constant[int], + BLOCK: ct.Constant[int], +): + actual_row = ct.bid(0) + batch_id = ct.bid(1) + row_idx = batch_id * L + actual_row + n_chunks = (L + BLOCK - 1) // BLOCK + + for ci in range(n_chunks): + col_start = ci * BLOCK + col_idx = ct.arange(BLOCK, dtype=ct.int32) + col_start + src_tile = ct.load(scores_2d, index=(row_idx, ci), shape=(1, BLOCK), padding_mode=ct.PaddingMode.ZERO).reshape( + (BLOCK,) + ) + is_past_f = ct.astype(col_idx <= actual_row, ct.float32) + out_tile = ct.astype(src_tile, ct.float32) * is_past_f + ct.store(output_2d, index=(row_idx, ci), tile=ct.astype(out_tile, output_2d.dtype).reshape((1, BLOCK))) + + +@ct.kernel +def _mask_bwd_kernel_ct( + grad_2d, + output_2d, + L: ct.Constant[int], + BLOCK: ct.Constant[int], +): + actual_row = ct.bid(0) + batch_id = ct.bid(1) + row_idx = batch_id * L + actual_row + n_chunks = (L + BLOCK - 1) // BLOCK + + for ci in range(n_chunks): + col_start = ci * BLOCK + col_idx = ct.arange(BLOCK, dtype=ct.int32) + col_start + grad_tile = ct.load(grad_2d, index=(row_idx, ci), shape=(1, BLOCK), padding_mode=ct.PaddingMode.ZERO).reshape( + (BLOCK,) + ) + is_past_f = ct.astype(col_idx <= actual_row, ct.float32) + out_tile = ct.astype(grad_tile, ct.float32) * is_past_f + ct.store(output_2d, index=(row_idx, ci), tile=ct.astype(out_tile, output_2d.dtype).reshape((1, BLOCK))) + + +@ct.kernel +def _fused_softmax_zeromask_bwd_kernel_ct( + probs_2d, + grad_probs_2d, + output_2d, + L: ct.Constant[int], + BLOCK: ct.Constant[int], +): + """Fused softmax backward + causal zero-mask: dx = p*(dp - dot(p,dp)); zero col>row.""" + actual_row = ct.bid(0) + batch_id = ct.bid(1) + row_idx = batch_id * L + actual_row + n_chunks = (L + BLOCK - 1) // BLOCK + + dot_tile = ct.full((BLOCK,), 0.0, dtype=ct.float32) + for ci in range(n_chunks): + col_idx = ct.arange(BLOCK, dtype=ct.int32) + ci * BLOCK + p_tile = ct.astype( + ct.gather(probs_2d, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + dp_tile = ct.astype( + ct.gather(grad_probs_2d, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + dot_tile = dot_tile + p_tile * dp_tile + dot = ct.sum(dot_tile, 0, keepdims=False) + + for ci in range(n_chunks): + col_idx = ct.arange(BLOCK, dtype=ct.int32) + ci * BLOCK + p_tile = ct.astype( + ct.gather(probs_2d, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + dp_tile = ct.astype( + ct.gather(grad_probs_2d, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + dx_tile = p_tile * (dp_tile - dot) + is_past_f = ct.astype(col_idx <= actual_row, ct.float32) + ct.scatter(output_2d, (row_idx, col_idx), ct.astype(dx_tile * is_past_f, output_2d.dtype), check_bounds=True) + + +def _mask_launch(tensor: torch.Tensor, kernel) -> torch.Tensor: + *batch, L, _ = tensor.shape + N = int(torch.prod(torch.tensor(batch))) if batch else 1 + t_f = tensor.reshape(N * L, L).contiguous() + out = torch.empty_like(t_f) + BLOCK = _select_block_size(L) + ct.launch(torch.cuda.current_stream(), (L, N, 1), kernel, (t_f, out, int(L), int(BLOCK))) + return out.reshape(*batch, L, L) + + +def _mask_inf_forward_ct(scores: torch.Tensor) -> torch.Tensor: + return _mask_launch(scores, _mask_inf_fwd_kernel_ct) + + +def _mask_zero_forward_ct(scores: torch.Tensor) -> torch.Tensor: + return _mask_launch(scores, _mask_zero_fwd_kernel_ct) + + +def _mask_backward_ct(grad: torch.Tensor) -> torch.Tensor: + return _mask_launch(grad, _mask_bwd_kernel_ct) + + +def _fused_softmax_zeromask_bwd_ct_launch(probs: torch.Tensor, grad_probs: torch.Tensor) -> torch.Tensor: + *batch, L, _ = probs.shape + N = int(torch.prod(torch.tensor(batch))) if batch else 1 + p_f = probs.reshape(N * L, L).contiguous() + dp_f = grad_probs.reshape(N * L, L).contiguous() + out = torch.empty_like(p_f) + + BLOCK = _select_block_size(L) + grid = (L, N, 1) + ct.launch( + torch.cuda.current_stream(), + grid, + _fused_softmax_zeromask_bwd_kernel_ct, + (p_f, dp_f, out, int(L), int(BLOCK)), + ) + return out.reshape(*batch, L, L) + + +def _conv1x1_backward(grad_out: torch.Tensor, inp: torch.Tensor, weight: torch.Tensor): + """mm-based 1x1 conv backward -- bypasses cuDNN dispatch overhead. + + For a kernel_size=1 conv: + grad_input[b,cin,h,w] = sum_cout(W[cout,cin] * dout[b,cout,h,w]) + grad_weight[cout,cin] = sum_{b,h,w}(dout[b,cout,h,w] * inp[b,cin,h,w]) + + Both reduce to matrix multiplications on the (B*H*W, C) reshape, letting + cuBLAS SGEMM handle the compute. On B200 for CH=1, L=128 this is ~1.69x + faster than F.conv_transpose2d + torch.nn.grad.conv2d_weight because it + bypasses cuDNN's per-call dispatch overhead for this tiny shape. + """ + B, C_out, H, W = grad_out.shape + C_in = inp.shape[1] + N = B * H * W + go_2d = grad_out.permute(0, 2, 3, 1).reshape(N, C_out) + in_2d = inp.permute(0, 2, 3, 1).reshape(N, C_in) + w_2d = weight.view(C_out, C_in) + grad_input = torch.mm(go_2d, w_2d).reshape(B, H, W, C_in).permute(0, 3, 1, 2).contiguous() + grad_weight = torch.mm(go_2d.t(), in_2d).view(weight.shape) + return grad_input, grad_weight + + +class LigerMultiTokenAttentionFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, scores, weight, bias=None, stride=1, padding=0, dilation=1, groups=1, sparse=False): + scores = scores.contiguous() + weight = weight.contiguous() + if bias is not None: + bias = bias.contiguous() + + ctx.sparse = sparse + + if sparse: + if scores.dtype != torch.float32: + raise RuntimeError( + f"CuTile sparse multi-token attention only supports fp32 input scores. Got dtype={scores.dtype}." + ) + compute_dtype = torch.float32 + weight_c, bias_c = weight, bias + + scores_inf = _mask_inf_forward_ct(scores) + probs, out_flat_sparse = _sparsemax_forward_ct(scores_inf, dim=-1) + out_conv = F.conv2d( + probs, weight_c, bias_c, stride=stride, padding=padding, dilation=dilation, groups=groups + ) + out = _mask_zero_forward_ct(out_conv) + ctx.save_for_backward(scores_inf, probs, out_flat_sparse, weight_c, bias_c) + else: + compute_dtype = scores.dtype + # fp16: promote to float32 for TF32 conv+softmax — avoids backward regression on small shapes (L≤128). + if compute_dtype == torch.float16: + scores = scores.float() + weight_c = weight.float() + bias_c = bias.float() if bias is not None else None + else: + weight_c, bias_c = weight, bias + + scores_inf = _mask_inf_forward_ct(scores) + probs = torch.softmax(scores_inf, dim=-1) + out_conv = F.conv2d( + probs, weight_c, bias_c, stride=stride, padding=padding, dilation=dilation, groups=groups + ) + out = _mask_zero_forward_ct(out_conv) + ctx.save_for_backward(scores_inf, probs, weight_c, bias_c) + + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.groups = groups + ctx.compute_dtype = compute_dtype + + return out.to(compute_dtype) + + @staticmethod + def backward(ctx, grad_out): + stride, padding, dilation, groups = (ctx.stride, ctx.padding, ctx.dilation, ctx.groups) + sparse = ctx.sparse + + if sparse: + scores_inf, probs, out_flat_sparse, weight, bias = ctx.saved_tensors + else: + scores_inf, probs, weight, bias = ctx.saved_tensors + + # .contiguous() is required: PyTorch's sum().backward() passes a broadcast + # tensor (strides=0), which would cause CuTile gather to read invalid offsets. + grad_out_c = grad_out.to(probs.dtype).contiguous() + + grad_conv = _mask_backward_ct(grad_out_c) + + # conv backward: mm-based 1x1 shortcut or cuDNN fallback + if stride == (1, 1) and padding == (0, 0) and dilation == (1, 1) and groups == 1: + grad_probs, grad_weight = _conv1x1_backward(grad_conv, probs, weight) + else: + # NOTE: we intentionally do NOT force torch.backends.cudnn.flags(benchmark=True) + # here, so this conv-backward runs under the same cuDNN heuristic as the Triton + # reference path (apples-to-apples comparison). For large spatial dims (e.g. + # L>=4096), enabling cudnn.benchmark=True may perform better here: cuDNN's + # default heuristic can otherwise pick a slower weight-gradient algorithm. If + # you target long sequences, consider enabling it globally (or wrapping this call). + grad_probs, grad_weight, _ = torch.ops.aten.convolution_backward( + grad_conv, + probs, + weight, + None, + list(stride), + list(padding), + list(dilation), + False, + [0, 0], + groups, + [True, True, False], + ) + + grad_bias = None + if bias is not None: + grad_bias = grad_conv.sum(dim=(0, 2, 3)) + + if sparse: + grad_scores_inf = _sparsemax_backward_ct(grad_probs.contiguous(), out_flat_sparse, dim=-1) + grad_scores = _mask_backward_ct(grad_scores_inf.to(probs.dtype).contiguous()) + else: + grad_scores = _fused_softmax_zeromask_bwd_ct_launch(probs, grad_probs) + + orig = ctx.compute_dtype + return ( + grad_scores.to(orig), + grad_weight.to(orig), + grad_bias.to(orig) if grad_bias is not None else None, + None, + None, + None, + None, + None, + ) diff --git a/src/liger_kernel/ops/cutile/ops/qwen2vl_mrope.py b/src/liger_kernel/ops/cutile/ops/qwen2vl_mrope.py new file mode 100644 index 000000000..c164355a9 --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/qwen2vl_mrope.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +# Adapted from https://github.com/linkedin/Liger-Kernel/blob/main/src/liger_kernel/ops/qwen2vl_mrope.py + +""" +Qwen2VL Multimodal Rotary Position Embedding (M-RoPE) kernel (CuTile backend). + +Half-split layout: left half of head_dim = real part, right half = imaginary part. +Three RoPE sections: temporal [0, t_end), height [t_end, h_end), width [h_end, hd//2). +cos/sin shape: (3, bsz, seq_len, head_dim). +Grid: (bsz, seq_len) — one program per token (2D grid avoids divmod on pid). +""" + +import cuda.tile as ct +import torch + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 + +ConstInt = ct.Constant[int] +PAD_ZERO = ct.PaddingMode.ZERO + + +@ct.kernel +def _qwen2vl_mrope_kernel_ct( + query, # 1D flat, len = bsz*sl*n_qh*hd + key, # 1D flat, len = bsz*sl*n_kh*hd + cos, # 1D flat, len = 3*bsz*sl*hd + sin, # 1D flat + sl, + BS_SL, # bsz * sl (slab stride = BS_SL * HEAD_DIM, computed in-kernel) + N_QH: ConstInt, + N_KH: ConstInt, + MROPE_SECTION_T: ConstInt, + MROPE_SECTION_H: ConstInt, + BACKWARD: ct.Constant[bool], + HEAD_DIM: ConstInt, + HEAD_DIM_HALF: ConstInt, + TILE_HD: ConstInt, + TILE_QH: ConstInt, + TILE_KH: ConstInt, + HD_POW2: ct.Constant[bool], +): + batch_idx = ct.bid(0) + seq_idx = ct.bid(1) + + t_end = MROPE_SECTION_T + h_end = t_end + MROPE_SECTION_H + + FLAT = TILE_QH * TILE_HD + row_1d = ct.arange(TILE_QH, dtype=ct.int32) + col_1d = ct.arange(TILE_HD, dtype=ct.int32) + flat_row = ct.broadcast_to(row_1d[:, None], (TILE_QH, TILE_HD)).reshape((FLAT,)) + flat_col = ct.broadcast_to(col_1d[None, :], (TILE_QH, TILE_HD)).reshape((FLAT,)) + + # Issue Q+K gathers first to start DRAM fetch, then cos/sin gathers can overlap. + q_token_off = (batch_idx * sl + seq_idx) * (N_QH * HEAD_DIM) + q_r_idx = q_token_off + flat_row * HEAD_DIM + flat_col + q_i_idx = q_r_idx + HEAD_DIM_HALF + if HD_POW2: + q_mask = flat_row < N_QH + else: + q_mask = (flat_row < N_QH) & (flat_col < HEAD_DIM_HALF) + q_r = ct.gather(query, q_r_idx, mask=q_mask, check_bounds=False, latency=2) + q_i = ct.gather(query, q_i_idx, mask=q_mask, check_bounds=False, latency=2) + + # K indices (computed early so K gathers can fire right after Q's) + FLAT_K = TILE_KH * TILE_HD + krow_1d = ct.arange(TILE_KH, dtype=ct.int32) + k_flat_row = ct.broadcast_to(krow_1d[:, None], (TILE_KH, TILE_HD)).reshape((FLAT_K,)) + k_flat_col = ct.broadcast_to(col_1d[None, :], (TILE_KH, TILE_HD)).reshape((FLAT_K,)) + k_token_off = (batch_idx * sl + seq_idx) * (N_KH * HEAD_DIM) + k_r_idx = k_token_off + k_flat_row * HEAD_DIM + k_flat_col + k_i_idx = k_r_idx + HEAD_DIM_HALF + if HD_POW2: + k_mask = k_flat_row < N_KH + else: + k_mask = (k_flat_row < N_KH) & (k_flat_col < HEAD_DIM_HALF) + k_r = ct.gather(key, k_r_idx, mask=k_mask, check_bounds=False, latency=2) + k_i = ct.gather(key, k_i_idx, mask=k_mask, check_bounds=False, latency=2) + + token_cs_off = (batch_idx * sl + seq_idx) * HEAD_DIM + slab_stride = BS_SL * HEAD_DIM + t_idx = flat_col + token_cs_off + h_idx = t_idx + slab_stride + w_idx = h_idx + slab_stride + t_cos = ct.gather(cos, t_idx, check_bounds=False, latency=2) + t_sin = ct.gather(sin, t_idx, check_bounds=False, latency=2) + h_cos = ct.gather(cos, h_idx, check_bounds=False, latency=2) + h_sin = ct.gather(sin, h_idx, check_bounds=False, latency=2) + w_cos = ct.gather(cos, w_idx, check_bounds=False, latency=2) + w_sin = ct.gather(sin, w_idx, check_bounds=False, latency=2) + + in_t = flat_col < t_end + in_h = flat_col < h_end + cos_row = ct.where(in_t, t_cos, ct.where(in_h, h_cos, w_cos)) + sin_row = ct.where(in_t, t_sin, ct.where(in_h, h_sin, w_sin)) + if BACKWARD: + sin_row = -sin_row + + cos_q = cos_row.astype(query.dtype) + sin_q = sin_row.astype(query.dtype) + new_q_r = q_r * cos_q - q_i * sin_q + new_q_i = q_i * cos_q + q_r * sin_q + + # Reuse Q's cos_row when FLAT_K <= FLAT (common case: TILE_KH <= TILE_QH). + if TILE_KH <= TILE_QH: + cos_k = ct.extract(cos_row, (0,), shape=(FLAT_K,)).astype(key.dtype) + sin_k = ct.extract(sin_row, (0,), shape=(FLAT_K,)).astype(key.dtype) + else: + t_idx_k = k_flat_col + token_cs_off + h_idx_k = t_idx_k + slab_stride + w_idx_k = h_idx_k + slab_stride + t_cos_k = ct.gather(cos, t_idx_k, check_bounds=False, latency=2) + t_sin_k = ct.gather(sin, t_idx_k, check_bounds=False, latency=2) + h_cos_k = ct.gather(cos, h_idx_k, check_bounds=False, latency=2) + h_sin_k = ct.gather(sin, h_idx_k, check_bounds=False, latency=2) + w_cos_k = ct.gather(cos, w_idx_k, check_bounds=False, latency=2) + w_sin_k = ct.gather(sin, w_idx_k, check_bounds=False, latency=2) + in_t_k = k_flat_col < t_end + in_h_k = k_flat_col < h_end + cos_k_raw = ct.where(in_t_k, t_cos_k, ct.where(in_h_k, h_cos_k, w_cos_k)) + sin_k_raw = ct.where(in_t_k, t_sin_k, ct.where(in_h_k, h_sin_k, w_sin_k)) + if BACKWARD: + sin_k_raw = -sin_k_raw + cos_k = cos_k_raw.astype(key.dtype) + sin_k = sin_k_raw.astype(key.dtype) + new_k_r = k_r * cos_k - k_i * sin_k + new_k_i = k_i * cos_k + k_r * sin_k + ct.scatter(query, q_r_idx, new_q_r, mask=q_mask, check_bounds=False, latency=1) + ct.scatter(query, q_i_idx, new_q_i, mask=q_mask, check_bounds=False, latency=1) + ct.scatter(key, k_r_idx, new_k_r, mask=k_mask, check_bounds=False, latency=1) + ct.scatter(key, k_i_idx, new_k_i, mask=k_mask, check_bounds=False, latency=1) + + +def _qwen2vl_mrope_forward(q, k, cos, sin, mrope_section): + q = q.transpose(1, 2).contiguous() + k = k.transpose(1, 2).contiguous() + + batch_size, seq_len, n_q_head, head_dim = q.shape + n_kv_head = k.shape[2] + head_dim_half = head_dim // 2 + TILE_HD = _next_power_of_2(head_dim_half) + TILE_QH = _next_power_of_2(n_q_head) + TILE_KH = _next_power_of_2(n_kv_head) + bs_sl = batch_size * seq_len + + cos = cos.contiguous() + sin = sin.contiguous() + + grid = (batch_size, seq_len) + ct.launch( + torch.cuda.current_stream(), + grid, + _qwen2vl_mrope_kernel_ct, + ( + q.view(-1), + k.view(-1), + cos.view(-1), + sin.view(-1), + int(seq_len), + int(bs_sl), + int(n_q_head), + int(n_kv_head), + int(mrope_section[0]), + int(mrope_section[1]), + False, + int(head_dim), + int(head_dim_half), + int(TILE_HD), + int(TILE_QH), + int(TILE_KH), + bool(TILE_HD == head_dim_half), + ), + ) + + return q.transpose(1, 2), k.transpose(1, 2), cos, sin + + +def _qwen2vl_mrope_backward(dq, dk, cos, sin, mrope_section): + dq = dq.transpose(1, 2).contiguous() + dk = dk.transpose(1, 2).contiguous() + + batch_size, seq_len, n_q_head, head_dim = dq.shape + n_kv_head = dk.shape[2] + head_dim_half = head_dim // 2 + TILE_HD = _next_power_of_2(head_dim_half) + TILE_QH = _next_power_of_2(n_q_head) + TILE_KH = _next_power_of_2(n_kv_head) + bs_sl = batch_size * seq_len + + grid = (batch_size, seq_len) + ct.launch( + torch.cuda.current_stream(), + grid, + _qwen2vl_mrope_kernel_ct, + ( + dq.view(-1), + dk.view(-1), + cos.view(-1), + sin.view(-1), + int(seq_len), + int(bs_sl), + int(n_q_head), + int(n_kv_head), + int(mrope_section[0]), + int(mrope_section[1]), + True, + int(head_dim), + int(head_dim_half), + int(TILE_HD), + int(TILE_QH), + int(TILE_KH), + bool(TILE_HD == head_dim_half), + ), + ) + + return dq.transpose(1, 2), dk.transpose(1, 2) + + +class LigerQwen2VLMRopeFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, q, k, cos, sin, mrope_section, unsqueeze_dim=1): + q, k, cos, sin = _qwen2vl_mrope_forward(q, k, cos, sin, mrope_section) + ctx.save_for_backward(cos, sin) + ctx.mrope_section = mrope_section + return q, k + + @staticmethod + def backward(ctx, dq, dk): + cos, sin = ctx.saved_tensors + mrope_section = ctx.mrope_section + dq, dk = _qwen2vl_mrope_backward(dq, dk, cos, sin, mrope_section) + return dq, dk, None, None, None, None diff --git a/src/liger_kernel/ops/cutile/ops/rope.py b/src/liger_kernel/ops/cutile/ops/rope.py new file mode 100644 index 000000000..01b6945cf --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/rope.py @@ -0,0 +1,312 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +Rotary Positional Embedding (RoPE) kernel (CuTile backend). +""" + +import cuda.tile as ct +import torch + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 + +ConstInt = ct.Constant[int] +PAD_ZERO = ct.PaddingMode.ZERO + + +@ct.kernel +def _rope_4d_kernel_ct( + Q, + K, + COS, + SIN, + seq_len: ConstInt, + sin_sign: ct.Constant[float], + TILE_QH: ConstInt, + TILE_KH: ConstInt, + TILE_HD: ConstInt, +): + cos_bs = COS.shape[0] + + pid = ct.bid(0) + batch_idx = pid // seq_len + seq_idx = pid % seq_len + cos_batch_idx = 0 if cos_bs == 1 else batch_idx + + # Stay in the input dtype throughout the math — triton's rope does the same. + # The fp32 cast doubles register pressure and arithmetic cost on bf16 inputs + # without meaningful precision gain for RoPE (rotation has unit magnitude). + cos_row = ct.load(COS, index=(cos_batch_idx, seq_idx, 0), shape=(1, 1, TILE_HD)).reshape((1, TILE_HD)) + sin_row = ct.load(SIN, index=(cos_batch_idx, seq_idx, 0), shape=(1, 1, TILE_HD)).reshape((1, TILE_HD)) * sin_sign + + q_r = ct.load(Q, index=(batch_idx, 0, seq_idx, 0), shape=(1, TILE_QH, 1, TILE_HD)).reshape((TILE_QH, TILE_HD)) + q_i = ct.load(Q, index=(batch_idx, 0, seq_idx, 1), shape=(1, TILE_QH, 1, TILE_HD)).reshape((TILE_QH, TILE_HD)) + new_q_r = q_r * cos_row - q_i * sin_row + new_q_i = q_i * cos_row + q_r * sin_row + ct.store(Q, index=(batch_idx, 0, seq_idx, 0), tile=new_q_r.reshape((1, TILE_QH, 1, TILE_HD))) + ct.store(Q, index=(batch_idx, 0, seq_idx, 1), tile=new_q_i.reshape((1, TILE_QH, 1, TILE_HD))) + + k_r = ct.load(K, index=(batch_idx, 0, seq_idx, 0), shape=(1, TILE_KH, 1, TILE_HD)).reshape((TILE_KH, TILE_HD)) + k_i = ct.load(K, index=(batch_idx, 0, seq_idx, 1), shape=(1, TILE_KH, 1, TILE_HD)).reshape((TILE_KH, TILE_HD)) + new_k_r = k_r * cos_row - k_i * sin_row + new_k_i = k_i * cos_row + k_r * sin_row + ct.store(K, index=(batch_idx, 0, seq_idx, 0), tile=new_k_r.reshape((1, TILE_KH, 1, TILE_HD))) + ct.store(K, index=(batch_idx, 0, seq_idx, 1), tile=new_k_i.reshape((1, TILE_KH, 1, TILE_HD))) + + +@ct.kernel +def _rope_general_kernel_ct( + Q, # (bsz, seq_len, n_q_heads, head_dim) -- 4D natural layout + K, # (bsz, seq_len, n_k_heads, head_dim) + COS, # (cos_bs, seq_len, head_dim) + SIN, # (cos_bs, seq_len, head_dim) + cos_bs: ConstInt, + seq_len: ConstInt, + N_Q_HEADS: ConstInt, + N_K_HEADS: ConstInt, + HEAD_DIM_HALF: ConstInt, + sin_sign: ct.Constant[float], + TILE_HD: ConstInt, +): + """General gather/scatter-based rope kernel. + + Works for arbitrary head_dim (including odd) and arbitrary n_heads: + - tile size is padded via _next_power_of_2 but original data is NOT padded + - ct.gather uses padding_value=0.0 for OOB reads (when TILE_HD > head_dim_half) + - elements past 2*head_dim_half (tail when head_dim is odd) are preserved unchanged + """ + pid = ct.bid(0) + batch_idx = pid // seq_len + seq_idx = pid % seq_len + cos_batch_idx = 0 if cos_bs == 1 else batch_idx + + dim_idx = ct.arange(TILE_HD, dtype=ct.int32) # [0, TILE_HD) + + cos_row = ct.astype( + ct.gather(COS, (cos_batch_idx, seq_idx, dim_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + sin_row = ( + ct.astype( + ct.gather(SIN, (cos_batch_idx, seq_idx, dim_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + * sin_sign + ) + + valid_mask = dim_idx < HEAD_DIM_HALF + + for h in range(N_Q_HEADS): + q_r = ct.astype( + ct.gather(Q, (batch_idx, seq_idx, h, dim_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + q_i = ct.astype( + ct.gather(Q, (batch_idx, seq_idx, h, dim_idx + HEAD_DIM_HALF), check_bounds=True, padding_value=0.0), + ct.float32, + ) + new_q_r = q_r * cos_row - q_i * sin_row + new_q_i = q_i * cos_row + q_r * sin_row + new_q_r = ct.where(valid_mask, new_q_r, q_r) + new_q_i = ct.where(valid_mask, new_q_i, q_i) + ct.scatter(Q, (batch_idx, seq_idx, h, dim_idx), ct.astype(new_q_r, Q.dtype), check_bounds=True) + ct.scatter(Q, (batch_idx, seq_idx, h, dim_idx + HEAD_DIM_HALF), ct.astype(new_q_i, Q.dtype), check_bounds=True) + + for h in range(N_K_HEADS): + k_r = ct.astype( + ct.gather(K, (batch_idx, seq_idx, h, dim_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + k_i = ct.astype( + ct.gather(K, (batch_idx, seq_idx, h, dim_idx + HEAD_DIM_HALF), check_bounds=True, padding_value=0.0), + ct.float32, + ) + new_k_r = k_r * cos_row - k_i * sin_row + new_k_i = k_i * cos_row + k_r * sin_row + new_k_r = ct.where(valid_mask, new_k_r, k_r) + new_k_i = ct.where(valid_mask, new_k_i, k_i) + ct.scatter(K, (batch_idx, seq_idx, h, dim_idx), ct.astype(new_k_r, K.dtype), check_bounds=True) + ct.scatter(K, (batch_idx, seq_idx, h, dim_idx + HEAD_DIM_HALF), ct.astype(new_k_i, K.dtype), check_bounds=True) + + +def rope_forward(q, k, cos, sin): + bsz, n_q_heads, seq_len, head_dim = q.shape + n_k_heads = k.shape[1] + head_dim_half = head_dim // 2 + original_dtype = q.dtype + + TILE_HD = _next_power_of_2(head_dim_half) + TILE_QH = _next_power_of_2(n_q_heads) + TILE_KH = _next_power_of_2(n_k_heads) + # ALIGNED: shapes are all power-of-2 → _rope_4d_kernel_ct's block-indexed ct.load + # works (its TILE_HD must equal head_dim_half exactly, which must be pow2). + # Otherwise we fall back to _rope_general_kernel_ct (ct.gather/scatter on element + # indices). Both are cuTile-native; we cannot reuse the 4D kernel for non-pow2 + # head_dim_half because ct.load requires the load shape to be power-of-2. + # Note: ct.load handles non-contig tensors natively via strides — no .contiguous() + # call needed on q/k for the fast path. + ALIGNED = (TILE_HD == head_dim_half) and (TILE_QH == n_q_heads) and (TILE_KH == n_k_heads) + + cos_3d = cos.contiguous() + sin_3d = sin.contiguous() + cos_bs = cos_3d.shape[0] + grid = (bsz * seq_len,) + + if ALIGNED: + ct.launch( + torch.cuda.current_stream(), + grid, + _rope_4d_kernel_ct, + ( + q, + k, + cos_3d, + sin_3d, + int(seq_len), + float(1.0), + int(TILE_QH), + int(TILE_KH), + int(TILE_HD), + ), + ) + return q, k, cos_3d, sin_3d, cos_bs, ALIGNED, TILE_QH, TILE_KH, TILE_HD, original_dtype + else: + # General path: handles arbitrary head_dim (including odd) and n_heads. + # Uses gather/scatter directly on the 4D layout — no data padding. + q_t = q.transpose(1, 2).contiguous() # (bsz, seq_len, n_q_heads, head_dim) + k_t = k.transpose(1, 2).contiguous() + cos_3d = cos.contiguous() + sin_3d = sin.contiguous() + cos_bs = cos_3d.shape[0] + ct.launch( + torch.cuda.current_stream(), + grid, + _rope_general_kernel_ct, + ( + q_t, + k_t, + cos_3d, + sin_3d, + int(cos_bs), + int(seq_len), + int(n_q_heads), + int(n_k_heads), + int(head_dim_half), + float(1.0), + int(TILE_HD), + ), + ) + q_out = q_t.transpose(1, 2).to(original_dtype) + k_out = k_t.transpose(1, 2).to(original_dtype) + return q_out, k_out, cos_3d, sin_3d, cos_bs, ALIGNED, TILE_QH, TILE_KH, TILE_HD, original_dtype + + +def rope_backward( + dq, + dk, + cos, + sin, + cos_bs, + ALIGNED, + TILE_QH, + TILE_KH, + TILE_HD, + original_dtype, + bsz, + seq_len, + n_q_heads, + n_k_heads, + head_dim, +): + head_dim_half = head_dim // 2 + n_row = bsz * seq_len + grid = (n_row,) + + if ALIGNED: + ct.launch( + torch.cuda.current_stream(), + grid, + _rope_4d_kernel_ct, + ( + dq, + dk, + cos, + sin, + int(seq_len), + float(-1.0), + int(TILE_QH), + int(TILE_KH), + int(TILE_HD), + ), + ) + return dq, dk + else: + dq_t = dq.transpose(1, 2).contiguous() + dk_t = dk.transpose(1, 2).contiguous() + ct.launch( + torch.cuda.current_stream(), + grid, + _rope_general_kernel_ct, + ( + dq_t, + dk_t, + cos, + sin, + int(cos_bs), + int(seq_len), + int(n_q_heads), + int(n_k_heads), + int(head_dim_half), + float(-1.0), + int(TILE_HD), + ), + ) + dq_out = dq_t.transpose(1, 2).to(original_dtype) + dk_out = dk_t.transpose(1, 2).to(original_dtype) + return dq_out, dk_out + + +class LigerRopeFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + bsz, n_q_heads, seq_len, head_dim = q.shape + n_k_heads = k.shape[1] + q_out, k_out, saved_cos, saved_sin, cos_bs, ALIGNED, TILE_QH, TILE_KH, TILE_HD, original_dtype = rope_forward( + q, k, cos, sin + ) + ctx.save_for_backward(saved_cos, saved_sin) + ctx.bsz = bsz + ctx.seq_len = seq_len + ctx.n_q_heads = n_q_heads + ctx.n_k_heads = n_k_heads + ctx.head_dim = head_dim + ctx.cos_bs = cos_bs + ctx.original_dtype = original_dtype + ctx.ALIGNED = ALIGNED + ctx.TILE_QH = TILE_QH + ctx.TILE_KH = TILE_KH + ctx.TILE_HD = TILE_HD + return q_out, k_out + + @staticmethod + def backward(ctx, dq, dk): + saved_cos, saved_sin = ctx.saved_tensors + dq_out, dk_out = rope_backward( + dq, + dk, + saved_cos, + saved_sin, + ctx.cos_bs, + ctx.ALIGNED, + ctx.TILE_QH, + ctx.TILE_KH, + ctx.TILE_HD, + ctx.original_dtype, + ctx.bsz, + ctx.seq_len, + ctx.n_q_heads, + ctx.n_k_heads, + ctx.head_dim, + ) + return dq_out, dk_out, None, None, None, None diff --git a/src/liger_kernel/ops/cutile/ops/sparsemax.py b/src/liger_kernel/ops/cutile/ops/sparsemax.py new file mode 100644 index 000000000..a051a47ad --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/sparsemax.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +import cuda.tile as ct +import torch + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 +from liger_kernel.ops.utils import ensure_contiguous + +# 20 bisections give fp32-scale tau precision (~1e-6 relative interval). +_BSEARCH_ITER = 20 + + +def _select_block_size(n_cols: int) -> int: + return min(_next_power_of_2(n_cols), 4096) + + +@ct.kernel(occupancy=4) +def _sparsemax_bsearch_kernel_ct( + y_output, + x_input, + N_COLS: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], + BSEARCH_ITER: ct.Constant[int], +): + row_idx = ct.bid(0) + n_chunks = (N_COLS + BLOCK_SIZE - 1) // BLOCK_SIZE + + x_max = ct.full((1,), -1e38, dtype=ct.float32) + + for ci in range(n_chunks): + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + ci * BLOCK_SIZE + x_tile = ct.astype( + ct.gather(x_input, (row_idx, col_idx), check_bounds=True, padding_value=-1e38), + ct.float32, + ) + x_max = ct.maximum(x_max, ct.max(x_tile, 0, keepdims=True)) + + # tau_lo = x_max - 1 is the tightest universally-valid lower bound: + # f(x_max - 1) = sum_{x > x_max - 1}(x - (x_max - 1)) >= (x_max - (x_max - 1)) = 1. + # Using (sum_x - 1)/n_cols breaks when some entries are large-negative mask sentinels + # (e.g. -1e9): the lower bound falls below the true tau on rows with few finite entries, + # and the bisection range becomes too wide for 20 iterations to converge precisely. + tau_lo = x_max - ct.full((1,), 1.0, ct.float32) + tau_hi = x_max + + one = ct.full((1,), 1.0, ct.float32) + half = ct.full((1,), 0.5, ct.float32) + + for _ in range(BSEARCH_ITER): + tau_mid = half * (tau_lo + tau_hi) + f = ct.full((1,), 0.0, ct.float32) + + for ci in range(n_chunks): + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + ci * BLOCK_SIZE + x_tile = ct.astype( + ct.gather(x_input, (row_idx, col_idx), check_bounds=True, padding_value=-1e38), + ct.float32, + ) + valid_mask = ct.astype(col_idx < N_COLS, ct.float32) + in_supp = ct.astype(x_tile > tau_mid, ct.float32) * valid_mask + f = f + ct.sum(in_supp * (x_tile - tau_mid), 0, keepdims=True) + + tau_lo = ct.where(f >= one, tau_mid, tau_lo) + tau_hi = ct.where(f < one, tau_mid, tau_hi) + + tau = half * (tau_lo + tau_hi) + + zero = ct.full((BLOCK_SIZE,), 0.0, ct.float32) + for ci in range(n_chunks): + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + ci * BLOCK_SIZE + x_tile = ct.astype( + ct.gather(x_input, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.float32, + ) + y_tile = ct.maximum(x_tile - tau, zero) + ct.scatter(y_output, (row_idx, col_idx), ct.astype(y_tile, y_output.dtype), check_bounds=True) + + +# Low-occupancy variant for large N (>16384): at high occupancy 7×128KB/SM thrashes L2; +# occ=2 keeps each row resident in L2 across bisection passes. Generated via replace_hints +# from the same kernel definition to avoid duplicating ~60 lines of bisection code. +_sparsemax_bsearch_kernel_large_ct = _sparsemax_bsearch_kernel_ct.replace_hints(occupancy=2) + + +@ct.kernel +def _sparsemax_bwd_kernel_ct( + grad_input, + output, + grad_output, + N_COLS: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], +): + row_idx = ct.bid(0) + n_chunks = (N_COLS + BLOCK_SIZE - 1) // BLOCK_SIZE + + go_sum_tile = ct.full((BLOCK_SIZE,), 0.0, ct.float32) + supp_cnt_tile = ct.full((BLOCK_SIZE,), 0.0, ct.float32) + + for ci in range(n_chunks): + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + ci * BLOCK_SIZE + o_tile = ct.astype(ct.gather(output, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + go_tile = ct.astype( + ct.gather(grad_output, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32 + ) + supp_f = ct.astype(o_tile > ct.full((BLOCK_SIZE,), 0.0, ct.float32), ct.float32) + go_sum_tile = go_sum_tile + supp_f * go_tile + supp_cnt_tile = supp_cnt_tile + supp_f + + go_sum = ct.sum(go_sum_tile, 0, keepdims=False) + supp_cnt = ct.sum(supp_cnt_tile, 0, keepdims=False) + mean_go = go_sum / (supp_cnt + 1e-6) + + for ci in range(n_chunks): + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + ci * BLOCK_SIZE + o_tile = ct.astype(ct.gather(output, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + go_tile = ct.astype( + ct.gather(grad_output, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32 + ) + supp_f = ct.astype(o_tile > ct.full((BLOCK_SIZE,), 0.0, ct.float32), ct.float32) + gi_tile = supp_f * (go_tile - mean_go) + ct.scatter(grad_input, (row_idx, col_idx), ct.astype(gi_tile, grad_input.dtype), check_bounds=True) + + +def _sparsemax_forward(x: torch.Tensor, dim: int): + if dim < 0: + dim += x.dim() + x_sw = x.transpose(dim, -1).contiguous() + n_cols = x_sw.size(-1) + n_rows = x_sw.numel() // n_cols + x_flat = x_sw.view(n_rows, n_cols) + + BLOCK_SIZE = _select_block_size(n_cols) + out_flat = torch.empty_like(x_flat) + kernel = _sparsemax_bsearch_kernel_large_ct if n_cols > 16384 else _sparsemax_bsearch_kernel_ct + ct.launch( + torch.cuda.current_stream(), + (n_rows, 1, 1), + kernel, + (out_flat, x_flat, int(n_cols), int(BLOCK_SIZE), int(_BSEARCH_ITER)), + ) + + return out_flat.view_as(x_sw).transpose(dim, -1).contiguous(), out_flat + + +def _sparsemax_backward(grad_out: torch.Tensor, out_flat: torch.Tensor, dim: int): + grad_sw = grad_out.transpose(dim, -1).contiguous() + n_cols = grad_sw.size(-1) + n_rows = grad_sw.numel() // n_cols + go_flat = grad_sw.view(n_rows, n_cols).contiguous() + + BLOCK_SIZE = _select_block_size(n_cols) + dx_flat = torch.empty_like(go_flat) + ct.launch( + torch.cuda.current_stream(), + (n_rows, 1, 1), + _sparsemax_bwd_kernel_ct, + (dx_flat, out_flat, go_flat, int(n_cols), int(BLOCK_SIZE)), + ) + + return dx_flat.view_as(grad_sw).transpose(dim, -1) + + +class LigerSparsemaxFunction(torch.autograd.Function): + @staticmethod + @ensure_contiguous + def forward(ctx, x: torch.Tensor, dim: int): + y, out_flat = _sparsemax_forward(x, dim) + ctx.save_for_backward(out_flat) + ctx.dim = dim + return y + + @staticmethod + @ensure_contiguous + def backward(ctx, grad_out: torch.Tensor): + (out_flat,) = ctx.saved_tensors + dx = _sparsemax_backward(grad_out, out_flat, ctx.dim) + return dx, None diff --git a/src/liger_kernel/ops/cutile/ops/tiled_mlp.py b/src/liger_kernel/ops/cutile/ops/tiled_mlp.py new file mode 100644 index 000000000..06a1c7a79 --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/tiled_mlp.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +Tiled MLP (cuTile backend). + +Pure Python implementation — no GPU kernel. +Shards input along sequence dimension (dim=-2), applies fn on each shard, +and concatenates. Backward re-computes forward per shard to save memory. +""" + +import math + +from typing import Callable +from typing import List +from typing import Optional + +import torch + + +class LigerTiledMLPFunction(torch.autograd.Function): + """Tiled MLP computation (no GPU kernel, memory-efficient via re-computation).""" + + @staticmethod + def forward(ctx, fn, mlp_module, x, shards, compute_params=None): + # compute_params is part of the upstream API (intended for DeepSpeed ZeRO + # weight registration); we accept and forward it for parity but don't + # consume it here — the autograd machinery already tracks fn's weights. + del compute_params + ctx.fn = fn + ctx.mlp_module = mlp_module + ctx.shards = shards + ctx.save_for_backward(x) + + x_shards = list(torch.chunk(x, chunks=shards, dim=-2)) + with torch.no_grad(): + output_shards = [fn(mlp_module, x_shard) for x_shard in x_shards] + return torch.cat(output_shards, dim=-2) + + @staticmethod + def backward(ctx, *grads): + fn = ctx.fn + (x,) = ctx.saved_tensors + mlp_module = ctx.mlp_module + shards = ctx.shards + + x_requires_grad = x.requires_grad + + x_detached = x.detach() + x_shards = list(torch.chunk(x_detached, chunks=shards, dim=-2)) + grad_shards = list(torch.chunk(grads[0], chunks=shards, dim=-2)) + + if x_requires_grad: + x_grad = torch.zeros_like(x_detached) + # torch.chunk on dim=-2 of a contiguous tensor returns views that + # share storage with x_grad. We assign the matching view as each + # shard leaf's .grad so autograd writes the per-shard gradients + # in-place into x_grad without an extra copy. + assert x_detached.is_contiguous(), "x must be contiguous for x_grad view chunking" + x_grad_shards = list(torch.chunk(x_grad, chunks=shards, dim=-2)) + else: + x_grad = None + + for i, (x_shard, grad_shard) in enumerate(zip(x_shards, grad_shards)): + x_shard_leaf = x_shard.detach().requires_grad_(x_requires_grad) + if x_requires_grad: + x_shard_leaf.grad = x_grad_shards[i] + with torch.enable_grad(): + output = fn(mlp_module, x_shard_leaf) + torch.autograd.backward(output, grad_shard) + + return None, None, x_grad, None, None + + +def apply_tiled_mlp( + fn: Callable, + mlp_module: torch.nn.Module, + x: torch.Tensor, + num_shards: Optional[int] = None, + compute_params: Optional[List[torch.nn.Parameter]] = None, +) -> torch.Tensor: + if num_shards is None: + hidden_size = x.shape[-1] + seqlen = x.shape[-2] + num_shards = math.ceil(seqlen / hidden_size) + num_shards = max(1, num_shards) + return LigerTiledMLPFunction.apply(fn, mlp_module, x, num_shards, compute_params)