From 1e22599522269c9d31f4ec3c4b0b75d01f07f9a7 Mon Sep 17 00:00:00 2001 From: Sachin Sharma Date: Fri, 31 Jul 2026 12:10:52 +0530 Subject: [PATCH 01/98] ggml-zendnn : group matmul direct API for mul_mat_id (#25918) * ggml-zendnn : group matmul API for mul_mat_id * ggml-zendnn : scale MUL_MAT_ID fallback threshold by expert count --- ggml/src/ggml-zendnn/CMakeLists.txt | 2 +- ggml/src/ggml-zendnn/ggml-zendnn.cpp | 296 +++++++++++++++++++-------- 2 files changed, 216 insertions(+), 82 deletions(-) diff --git a/ggml/src/ggml-zendnn/CMakeLists.txt b/ggml/src/ggml-zendnn/CMakeLists.txt index e4ba9cfbd0fc..87d721f6d78a 100644 --- a/ggml/src/ggml-zendnn/CMakeLists.txt +++ b/ggml/src/ggml-zendnn/CMakeLists.txt @@ -28,7 +28,7 @@ if (NOT ZENDNN_ROOT OR ZENDNN_ROOT STREQUAL "" OR ZENDNN_ROOT STREQUAL "OFF") ExternalProject_Add( zendnn GIT_REPOSITORY https://github.com/amd/ZenDNN.git - GIT_TAG 253b94ce0d7e9284c265fefb485714944caff9d3 # ZenDNN-2026-WW19 + GIT_TAG 1f399a75cc0993778374a51bea49b64a57879595 # ZenDNN-2026-WW28 PREFIX ${ZENDNN_PREFIX} SOURCE_DIR ${ZENDNN_SOURCE_DIR} BINARY_DIR ${ZENDNN_BUILD_DIR} diff --git a/ggml/src/ggml-zendnn/ggml-zendnn.cpp b/ggml/src/ggml-zendnn/ggml-zendnn.cpp index 3c33dcb11a03..e6a9b51b7925 100644 --- a/ggml/src/ggml-zendnn/ggml-zendnn.cpp +++ b/ggml/src/ggml-zendnn/ggml-zendnn.cpp @@ -30,6 +30,29 @@ zendnnl::common::data_type_t ggml_to_zendnn_type() { } } +/** + * Builds the matmul_params shared by ggml_zendnn_matmul() and ggml_zendnn_group_matmul(): + * dtype selection plus, for Q8_0 weights, dynamic-quant setup. Callers still need to set + * quant_params.src_scale.dims themselves, since that depends on the batch size(s) in use. + */ +template +static zendnnl::lowoha::matmul::matmul_params ggml_zendnn_make_matmul_params(ggml_backend_zendnn_context * ctx) { + zendnnl::lowoha::matmul::matmul_params params; + params.dtypes.src = ggml_to_zendnn_type(); + params.dtypes.wei = ggml_to_zendnn_type(); + params.dtypes.dst = ggml_to_zendnn_type(); + params.num_threads = ctx->n_threads; + + if constexpr (std::is_same_v) { + params.dtypes.compute = zendnnl::common::data_type_t::s8; + params.dynamic_quant = true; + params.quant_params.src_scale.buff = nullptr; + params.quant_params.src_scale.dt = zendnnl::common::data_type_t::bf16; + params.packing.pack_format_b = 1; + } + return params; +} + /** * ZenDNN matmul: computes C = B * A. * @@ -47,22 +70,12 @@ static bool ggml_zendnn_matmul(ggml_backend_zendnn_context * ctx, int64_t m, int const TA * A, int64_t lda, const TB * B, int64_t ldb, TC * C, int64_t ldc) { - zendnnl::lowoha::matmul::matmul_params params; - params.dtypes.src = ggml_to_zendnn_type(); - params.dtypes.wei = ggml_to_zendnn_type(); - params.dtypes.dst = ggml_to_zendnn_type(); - params.num_threads = ctx->n_threads; + zendnnl::lowoha::matmul::matmul_params params = ggml_zendnn_make_matmul_params(ctx); zendnnl::lowoha::matmul::matmul_batch_params_t batch_params; if constexpr (std::is_same_v) { - params.dtypes.compute = zendnnl::common::data_type_t::s8; - const int64_t num_groups = k / QK8_0; - params.dynamic_quant = true; - params.quant_params.src_scale.buff = nullptr; - params.quant_params.src_scale.dt = zendnnl::common::data_type_t::bf16; - params.quant_params.src_scale.dims = {n, num_groups}; - params.packing.pack_format_b = 1; + params.quant_params.src_scale.dims = {n, k / QK8_0}; } zendnnl::error_handling::status_t status = zendnnl::lowoha::matmul::matmul_direct( @@ -223,6 +236,99 @@ struct mmid_row_mapping { int32_t i2; }; +/** + * ZenDNN batched matmul: computes C[i] = B[i] * A[i] for every active expert i via a single + * group_matmul_direct() call. Batched analogue of ggml_zendnn_matmul() - see its docs for the + * per-expert A/B/C shape convention. m and k are shared by every expert; n (batch size) varies + * per expert, hence the vector. + */ +template +static bool ggml_zendnn_group_matmul(ggml_backend_zendnn_context * ctx, int64_t m, int64_t k, + const std::vector & n, + const std::vector & A, int64_t lda, + const std::vector & B, int64_t ldb, + const std::vector & C, int64_t ldc) { + + const int n_experts = n.size(); + + zendnnl::lowoha::matmul::matmul_params base_params = ggml_zendnn_make_matmul_params(ctx); + + std::vector layout(n_experts, 'r'); + std::vector trans_a(n_experts, false); + std::vector trans_b(n_experts, true); + std::vector batch_m(n_experts); + std::vector batch_n(n_experts, m); + std::vector batch_k(n_experts, k); + std::vector alpha(n_experts, 1.0f); + std::vector beta(n_experts, 0.0f); + std::vector bias(n_experts, nullptr); + std::vector lda_v(n_experts, lda); + std::vector ldb_v(n_experts, ldb); + std::vector ldc_v(n_experts, ldc); + std::vector is_wei_const(n_experts, true); + std::vector params(n_experts, base_params); + + for (int i = 0; i < n_experts; i++) { + batch_m[i] = n[i]; + + // src_scale.dims depends on this expert's row count, unlike the rest of base_params + if constexpr (std::is_same_v) { + params[i].quant_params.src_scale.dims = {n[i], k / QK8_0}; + } + } + + zendnnl::error_handling::status_t status = zendnnl::lowoha::matmul::group_matmul_direct( + layout, trans_a, trans_b, batch_m, batch_n, batch_k, alpha, + B, ldb_v, A, lda_v, bias, beta, + C, ldc_v, is_wei_const, params); + + if (status != zendnnl::error_handling::status_t::success) { + GGML_LOG_ERROR("%s, ZenDNN group matmul failed: status=%d\n", __func__, static_cast(status)); + return false; + } + return true; +} + +static bool ggml_zendnn_group_gemm(ggml_backend_zendnn_context * ctx, int64_t m, int64_t k, + const std::vector & n, + const std::vector & A, int64_t lda, + const std::vector & B, int64_t ldb, + const std::vector & C, int64_t ldc, + int Atype, int Btype, int Ctype) { + + assert(m >= 0); + for (size_t i = 0; i < n.size(); i++) { + assert(n[i] >= 0); + } + assert(k >= 0); + assert(lda >= k); + assert(ldb >= k); + assert(ldc >= m); + + // categorize types + switch (Atype) { + case GGML_TYPE_F32: + if (Btype != GGML_TYPE_F32 || Ctype != GGML_TYPE_F32) + return false; + return ggml_zendnn_group_matmul(ctx, m, k, n, A, lda, B, ldb, C, ldc); + case GGML_TYPE_BF16: + if (Btype != GGML_TYPE_BF16) + return false; + if (Ctype == GGML_TYPE_BF16) + return ggml_zendnn_group_matmul( + ctx, m, k, n, A, lda, B, ldb, C, ldc); + if (Ctype == GGML_TYPE_F32) + return ggml_zendnn_group_matmul(ctx, m, k, n, A, lda, B, ldb, C, ldc); + return false; + case GGML_TYPE_Q8_0: + if (Btype != GGML_TYPE_F32 || Ctype != GGML_TYPE_F32) + return false; + return ggml_zendnn_group_matmul(ctx, m, k, n, A, lda, B, ldb, C, ldc); + default: + return false; // unsupported type + } +} + static void ggml_zendnn_compute_forward_mul_mat_id( ggml_backend_zendnn_context * ctx, ggml_tensor * dst) { @@ -262,7 +368,8 @@ static void ggml_zendnn_compute_forward_mul_mat_id( std::vector matrix_row_counts(n_as, 0); std::vector> matrix_rows(n_as); - int64_t max_rows = 0; + int64_t total_rows = 0; + int n_active_experts = 0; // group rows by expert (preprocessing step) for (int64_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { for (int id = 0; id < n_ids; ++id) { @@ -270,66 +377,74 @@ static void ggml_zendnn_compute_forward_mul_mat_id( GGML_ASSERT(i02 >= 0 && i02 < n_as); + if (matrix_row_counts[i02] == 0) { + n_active_experts++; + } matrix_rows[i02].push_back({id, iid1}); matrix_row_counts[i02]++; - if (matrix_row_counts[i02] > max_rows) { - max_rows = matrix_row_counts[i02]; - } + total_rows++; } } - if (max_rows == 0) { + if (total_rows == 0) { return; // no rows to process } const size_t row_size = ggml_row_size(vec_dot_type, ne10); - // size for converting src1 rows to vec_dot_type if needed - const size_t nbw1 = row_size; - const size_t nbw2 = nbw1 * ne11; - const size_t nbw3 = nbw2 * ne12; - const size_t src1_conv_size = (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) ? ne13 * nbw3 : 0; - // For Q8_0, src1 is always F32; the gather buffer must hold F32 rows (ne10*4 bytes), // not Q8_0-encoded rows (row_size ≈ ne10/32*34 bytes) — they differ by ~4x. const size_t f32_row_size = (size_t)ne10 * sizeof(float); const size_t gather_row_size = (src0->type == GGML_TYPE_Q8_0) ? f32_row_size : row_size; + if (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) { + GGML_ASSERT(src1->type == GGML_TYPE_F32); + } + // size for MoE gather/scatter buffers - const size_t wdata_cur_size = max_rows * gather_row_size; - const size_t dst_cur_size = max_rows * ggml_row_size(dst->type, ne01); + const size_t wdata_cur_size = total_rows * gather_row_size; + const size_t dst_cur_size = total_rows * ggml_row_size(dst->type, ne01); // allocate single buffer for all needs - const size_t total_size = src1_conv_size + wdata_cur_size + dst_cur_size; + const size_t total_size = wdata_cur_size + dst_cur_size; if (ctx->work_size < total_size) { ctx->work_data.reset(new char[total_size]); ctx->work_size = total_size; } // partition the buffer - char * work_data = ctx->work_data.get(); - char * wdata_cur = work_data + src1_conv_size; + char * wdata_cur = ctx->work_data.get(); char * dst_cur = wdata_cur + wdata_cur_size; - // ZenDNN requires FP32 for dynamic quantization, so conversion is skipped - if (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) { - GGML_ASSERT(src1->type == GGML_TYPE_F32); - - #pragma omp parallel for collapse(3) num_threads(ctx->n_threads) schedule(static) - for (int64_t i13 = 0; i13 < ne13; ++i13) { - for (int64_t i12 = 0; i12 < ne12; ++i12) { - for (int64_t i11 = 0; i11 < ne11; ++i11) { - const float * src1_f32 = (float *)((char *)src1->data + i11*nb11 + i12*nb12 + i13*nb13); - void * src1_conv = (char *)work_data + i11*nbw1 + i12*nbw2 + i13*nbw3; - from_float(src1_f32, src1_conv, ne10); - } + // per-expert data collected during gather, handed to ggml_zendnn_group_gemm() as one batch + std::vector expert_row_count(n_active_experts); + std::vector batch_src(n_active_experts); + std::vector batch_wei(n_active_experts); + std::vector batch_dst(n_active_experts); + + // precompute per-expert buffer offsets and batch indices for the parallel loop below + std::vector expert_wdata_off(n_as, 0); + std::vector expert_dst_off(n_as, 0); + std::vector expert_batch_idx(n_as, -1); + { + int64_t w_off = 0; + int64_t d_off = 0; + int batch_idx = 0; + for (int64_t cur_a = 0; cur_a < n_as; ++cur_a) { + if (matrix_row_counts[cur_a] == 0) { + continue; } + expert_wdata_off[cur_a] = w_off; + expert_dst_off[cur_a] = d_off; + expert_batch_idx[cur_a] = batch_idx; + w_off += matrix_row_counts[cur_a] * gather_row_size; + d_off += matrix_row_counts[cur_a] * ggml_row_size(dst->type, ne01); + batch_idx++; } } - const void * wdata = (src1->type == vec_dot_type || src0->type == GGML_TYPE_Q8_0) ? src1->data : work_data; - - // process each expert with gather -> gemm -> scatter pattern + // gather + inline-convert input rows into each expert's batch slot + #pragma omp parallel for num_threads(ctx->n_threads) schedule(static) for (int64_t cur_a = 0; cur_a < n_as; ++cur_a) { const int64_t cne1 = matrix_row_counts[cur_a]; @@ -337,42 +452,57 @@ static void ggml_zendnn_compute_forward_mul_mat_id( continue; } - const char * src0_cur = (const char *) src0->data + cur_a*nb02; + const int64_t w_off = expert_wdata_off[cur_a]; + const int64_t d_off = expert_dst_off[cur_a]; + const int batch_idx = expert_batch_idx[cur_a]; - // gather input rows for this expert - #pragma omp parallel for num_threads(ctx->n_threads) schedule(static) for (int64_t ir1 = 0; ir1 < cne1; ++ir1) { const mmid_row_mapping & row_mapping = matrix_rows[cur_a][ir1]; - const int64_t id = row_mapping.i1; + const int64_t id = row_mapping.i1; const int64_t i11 = id % ne11; const int64_t i12 = row_mapping.i2; - std::memcpy( - wdata_cur + ir1 * gather_row_size, - (const char *) wdata + (i11 + i12*ne11) * gather_row_size, - gather_row_size - ); + const char * src_row = (const char *) src1->data + i11*nb11 + i12*nb12; + void * dst_row = wdata_cur + w_off + ir1 * gather_row_size; + + if (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) { + from_float((const float *) src_row, dst_row, ne10); + } else { + // no conversion: src1 already matches vec_dot_type, or src0 is Q8_0, whose + // ZenDNN dynamic quantization requires the row to stay in F32 + std::memcpy(dst_row, src_row, gather_row_size); + } } - // batched gemm for all tokens in this expert - if (!ggml_zendnn_gemm(ctx, - ne01, // m - cne1, // n - ne10, // k - src0_cur, - ne00, // lda - wdata_cur, - ne10, // ldb - dst_cur, - ne01, // ldc - src0->type, - src0->type == GGML_TYPE_Q8_0 ? GGML_TYPE_F32 : vec_dot_type, - dst->type)) { - GGML_ABORT("%s: ZenDNN gemm failed\n", __func__); + expert_row_count[batch_idx] = cne1; + batch_src[batch_idx] = wdata_cur + w_off; + batch_wei[batch_idx] = (const char *) src0->data + cur_a * nb02; + batch_dst[batch_idx] = dst_cur + d_off; + } + + if (!ggml_zendnn_group_gemm(ctx, + ne01, // m + ne10, // k + expert_row_count, // n (per expert) + batch_wei, ne00, // A: weights, lda + batch_src, ne10, // B: input, ldb + batch_dst, ne01, // C: output, ldc + src0->type, + src0->type == GGML_TYPE_Q8_0 ? GGML_TYPE_F32 : vec_dot_type, + dst->type)) + GGML_ABORT("%s: ZenDNN group gemm failed\n", __func__); + + // scatter output rows to destination + #pragma omp parallel for num_threads(ctx->n_threads) schedule(static) + for (int64_t cur_a = 0; cur_a < n_as; ++cur_a) { + const int64_t cne1 = matrix_row_counts[cur_a]; + + if (cne1 == 0) { + continue; } - // scatter output rows to destination - #pragma omp parallel for num_threads(ctx->n_threads) schedule(static) + const int64_t d_off = expert_dst_off[cur_a]; + for (int64_t ir1 = 0; ir1 < cne1; ++ir1) { const mmid_row_mapping & row_mapping = matrix_rows[cur_a][ir1]; const int64_t id = row_mapping.i1; @@ -381,7 +511,7 @@ static void ggml_zendnn_compute_forward_mul_mat_id( std::memcpy( (char *) dst->data + i1*nb1 + i2*nb2, - dst_cur + ir1 * ggml_row_size(dst->type, ne01), + dst_cur + d_off + ir1 * ggml_row_size(dst->type, ne01), ggml_row_size(dst->type, ne01) ); } @@ -591,22 +721,26 @@ static bool ggml_backend_zendnn_device_supports_op(ggml_backend_dev_t dev, const if(K <= 256 || N <= 128 || M <= 96) { return false; } + + // MUL_MAT_ID's gather+matmul+scatter approach favors a moderate expert count + if (op->op == GGML_OP_MUL_MAT_ID) { + const int64_t n_experts = weights->ne[2]; + const int64_t max_experts = 32; + if (n_experts > max_experts) { + return false; + } + + // fall back once the average rows per expert (N / n_experts) is too thin + // to amortize each per-expert GEMM's overhead + if (N / n_experts <= 32) { + return false; + } + } } else if (ne0 < min_batch || ne1 < min_batch || ne10 < min_batch) { return false; } - // MUL_MAT_ID performs best with a moderate number of experts due to its - // gather + batched matmul + scatter approach. Future versions will leverage - // ZenDNN's grouped_gemm for better scalability with larger expert counts: - // https://github.com/amd/ZenDNN/blob/main/docs/operator/lowoha_group_gemm_operator.md - if (op->op == GGML_OP_MUL_MAT_ID) { - const int64_t n_experts = weights->ne[2]; - const int64_t max_experts = 32; - if (n_experts > max_experts) { - return false; - } - } switch (weights->type) { case GGML_TYPE_F32: case GGML_TYPE_BF16: From 69e62fc77c911da169cc8726b490028d53bb90fe Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:03:30 +0200 Subject: [PATCH 02/98] llama : enforce the same K and V cache types for DeepSeek V4; enable FA if V cache is quantized (#25871) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * llama : enforce the same K and V cache types for DeepSeek V4; enable FA if V cache is quantized * llama : enforce the same K and V cache types for MLA models --------- Co-authored-by: Stanisław Szymczyk --- src/llama-context.cpp | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 3e0af3f317f6..5ef7becf6f29 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -3553,6 +3553,22 @@ llama_context * llama_init_from_model( } } + if ((model->hparams.is_mla() || model->arch == LLM_ARCH_DEEPSEEK4) && params.type_k != params.type_v) { + LLAMA_LOG_ERROR("%s: model does not support different K (%s) and V (%s) cache types\n", __func__, ggml_type_name(params.type_k), ggml_type_name(params.type_v)); + return nullptr; + } + + if (ggml_is_quantized(params.type_v) && params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_ENABLED) { + if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO) { + LLAMA_LOG_INFO("%s: enabling flash_attn since it is required for quantized V cache\n", __func__); + params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; + } + if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) { + LLAMA_LOG_ERROR("%s: quantized V cache requires flash_attn to be enabled\n", __func__); + return nullptr; + } + } + if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_k)) { const uint32_t blck_size = ggml_blck_size(params.type_k); for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) { @@ -3575,11 +3591,6 @@ llama_context * llama_init_from_model( } } - if (ggml_is_quantized(params.type_v) && params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) { - LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__); - return nullptr; - } - if (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != model->hparams.pooling_type) { //user-specified pooling-type is different from the model default From d5d3e05bf8d2a29b71270b1af9d538dc2545b02a Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Fri, 31 Jul 2026 15:25:16 +0800 Subject: [PATCH 03/98] [SYCL] support the missed types in cpy (#26005) * support the missed types in cpy * use correct funct * rm unused code --- ggml/src/ggml-sycl/cpy.cpp | 183 ++++++++++++++++++++++++++----------- ggml/src/ggml-sycl/cpy.hpp | 33 +++++++ 2 files changed, 164 insertions(+), 52 deletions(-) diff --git a/ggml/src/ggml-sycl/cpy.cpp b/ggml/src/ggml-sycl/cpy.cpp index 5d0f9a89fd33..55e076172237 100644 --- a/ggml/src/ggml-sycl/cpy.cpp +++ b/ggml/src/ggml-sycl/cpy.cpp @@ -8,7 +8,6 @@ #include "ggml-sycl/presets.hpp" #include "ggml.h" - static void cpy_1_f32_f32(const char * cxi, char * cdsti) { const float * xi = (const float *) cxi; float * dsti = (float *) cdsti; @@ -151,6 +150,20 @@ static void cpy_blck_q8_0_f32(const char * cxi, char * cdsti) { } } +static void cpy_blck_q2_0_f32(const char * cxi, char * cdsti) { + const block_q2_0 * xi = (const block_q2_0 *) cxi; + float * cdstf = (float *) cdsti; + + const float d = xi->d; + + for (int j = 0; j < QK2_0; ++j) { + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + const int q = (xi->qs[byte_index] >> bit_offset) & 0x3; + cdstf[j] = (float) (q - 1) * d; + } +} + template static void cpy_blck_q_f32(const char * cxi, char * cdsti) { @@ -256,7 +269,7 @@ static void ggml_cpy_f16_f32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -274,7 +287,7 @@ static void ggml_cpy_f32_f32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -292,7 +305,7 @@ static void ggml_cpy_f32_f16_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -308,7 +321,7 @@ static void ggml_cpy_f32_i32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -324,7 +337,7 @@ static void ggml_cpy_i32_f32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -338,7 +351,7 @@ static void ggml_cpy_f32_q8_0_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK8_0 == 0); const int num_blocks = ne / QK8_0; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -350,12 +363,25 @@ static void ggml_cpy_q8_0_f32_sycl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } +static void ggml_cpy_q2_0_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ne; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cpy_q_f32(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, + ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + static void ggml_cpy_f32_q4_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, @@ -363,7 +389,7 @@ static void ggml_cpy_f32_q4_0_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK4_0 == 0); const int num_blocks = ne / QK4_0; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -375,7 +401,8 @@ static void ggml_cpy_q4_0_f32_sycl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK4_0>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -389,7 +416,7 @@ static void ggml_cpy_f32_q4_1_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK4_1 == 0); const int num_blocks = ne / QK4_1; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -401,7 +428,8 @@ static void ggml_cpy_q4_1_f32_sycl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK4_1>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -415,7 +443,7 @@ static void ggml_cpy_f32_q5_0_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK5_0 == 0); const int num_blocks = ne / QK5_0; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -427,7 +455,8 @@ static void ggml_cpy_q5_0_f32_sycl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK5_0>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -441,7 +470,7 @@ static void ggml_cpy_f32_q5_1_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK5_1 == 0); const int num_blocks = ne / QK5_1; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -453,7 +482,8 @@ static void ggml_cpy_q5_1_f32_sycl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK5_1>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -466,7 +496,8 @@ static void ggml_cpy_mxfp4_f32_sycl(const char * cx, char * cdst, const int ne, const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_q_f32, QK_MXFP4>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -480,7 +511,8 @@ static void ggml_cpy_f32_iq4_nl_sycl(const char * cx, char * cdst, const int ne, GGML_ASSERT(ne % QK4_NL == 0); const int num_blocks = ne / QK4_NL; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -526,7 +558,7 @@ static void ggml_cpy_f16_q4_0_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK4_0 == 0); const int num_blocks = ne / QK4_0; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -540,7 +572,7 @@ static void ggml_cpy_f16_q4_1_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK4_1 == 0); const int num_blocks = ne / QK4_1; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -554,7 +586,7 @@ static void ggml_cpy_f16_q5_0_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK5_0 == 0); const int num_blocks = ne / QK5_0; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -564,6 +596,7 @@ static void ggml_cpy_f16_q5_0_sycl(const char * cx, char * cdst, const int ne, c static bool ggml_sycl_is_quantized_type(enum ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -594,6 +627,7 @@ static bool ggml_sycl_is_quantized_type(enum ggml_type type) { static bool ggml_sycl_can_quantize_rows_sycl(enum ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -651,7 +685,8 @@ static void ggml_sycl_quantize_rows_q(const char * cx, char * cdst, const int64_ constexpr int block_size = 256; const int64_t grid_size = ceil_div(total_blocks, (int64_t) block_size); - stream->parallel_for(sycl::nd_range<1>(grid_size * block_size, block_size), [=](sycl::nd_item<1> item_ct1) { + stream->parallel_for(sycl::nd_range<1>(grid_size * block_size, block_size), + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { const int64_t block_idx = item_ct1.get_global_linear_id(); if (block_idx >= total_blocks) { return; @@ -708,6 +743,11 @@ static void ggml_sycl_quantize_rows_sycl(const char * cx, char * cdst, const ggm nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, stream); break; + case GGML_TYPE_Q2_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; case GGML_TYPE_Q5_1: ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, @@ -760,7 +800,7 @@ static void ggml_cpy_f16_f16_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -779,7 +819,7 @@ static void ggml_cpy_i16_i16_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -798,7 +838,7 @@ static void ggml_cpy_i32_i32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -812,7 +852,8 @@ static void ggml_cpy_q8_0_q8_0(const char * cx, char * cdst, const int ne, const const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -825,7 +866,8 @@ static void ggml_cpy_q5_0_q5_0(const char * cx, char * cdst, const int ne, const const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -839,7 +881,8 @@ static void ggml_cpy_q5_1_q5_1(const char * cx, char * cdst, const int ne, const stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -851,7 +894,8 @@ static void ggml_cpy_q4_0_q4_0(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -864,7 +908,8 @@ static void ggml_cpy_q4_1_q4_1(const char * cx, char * cdst, const int ne, const const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -875,18 +920,32 @@ static void ggml_cpy_q1_0_q1_0(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } +static void ggml_cpy_q2_0_q2_0(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + static void ggml_cpy_mxfp4_mxfp4(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -897,7 +956,8 @@ static void ggml_cpy_nvfp4_nvfp4(const char * cx, char * cdst, const int ne, con const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -908,7 +968,8 @@ static void ggml_cpy_q2_K_q2_K(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -919,7 +980,8 @@ static void ggml_cpy_q3_K_q3_K(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -930,7 +992,8 @@ static void ggml_cpy_q4_K_q4_K(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -941,7 +1004,8 @@ static void ggml_cpy_q5_K_q5_K(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -952,7 +1016,8 @@ static void ggml_cpy_q6_K_q6_K(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -963,7 +1028,8 @@ static void ggml_cpy_iq2_xxs_iq2_xxs(const char * cx, char * cdst, const int ne, const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -974,7 +1040,8 @@ static void ggml_cpy_iq2_xs_iq2_xs(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -985,7 +1052,8 @@ static void ggml_cpy_iq2_s_iq2_s(const char * cx, char * cdst, const int ne, con const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -996,7 +1064,8 @@ static void ggml_cpy_iq3_xxs_iq3_xxs(const char * cx, char * cdst, const int ne, const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1007,7 +1076,8 @@ static void ggml_cpy_iq1_s_iq1_s(const char * cx, char * cdst, const int ne, con const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1018,7 +1088,8 @@ static void ggml_cpy_iq1_m_iq1_m(const char * cx, char * cdst, const int ne, con const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1029,7 +1100,8 @@ static void ggml_cpy_iq4_nl_iq4_nl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1040,7 +1112,8 @@ static void ggml_cpy_iq3_s_iq3_s(const char * cx, char * cdst, const int ne, con const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1051,7 +1124,8 @@ static void ggml_cpy_iq4_xs_iq4_xs(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1065,7 +1139,7 @@ static void ggml_cpy_f32_bf16_sycl(const char * cx, char * cdst, const int ne, c stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -1079,7 +1153,7 @@ static void ggml_cpy_bf16_f32_sycl(const char * cx, char * cdst, const int ne, c stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -1093,7 +1167,7 @@ static void ggml_cpy_bf16_bf16_sycl(const char * cx, char * cdst, const int ne, stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -1107,7 +1181,7 @@ static void ggml_cpy_f16_bf16_sycl(const char * cx, char * cdst, const int ne, c stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -1121,7 +1195,7 @@ static void ggml_cpy_bf16_f16_sycl(const char * cx, char * cdst, const int ne, c stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -1213,6 +1287,9 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co } else if (src0->type == GGML_TYPE_Q8_0 && src1->type == GGML_TYPE_F32) { ggml_cpy_q8_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q2_0 && src1->type == GGML_TYPE_F32) { + ggml_cpy_q2_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) { ggml_cpy_f32_q5_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -1243,6 +1320,8 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co ggml_cpy_q4_1_q4_1(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_Q1_0 && src1->type == GGML_TYPE_Q1_0) { ggml_cpy_q1_0_q1_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q2_0 && src1->type == GGML_TYPE_Q2_0) { + ggml_cpy_q2_0_q2_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_MXFP4 && src1->type == GGML_TYPE_MXFP4) { ggml_cpy_mxfp4_mxfp4(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_NVFP4 && src1->type == GGML_TYPE_NVFP4) { diff --git a/ggml/src/ggml-sycl/cpy.hpp b/ggml/src/ggml-sycl/cpy.hpp index c4cfd961da2a..34bae1b2dd18 100644 --- a/ggml/src/ggml-sycl/cpy.hpp +++ b/ggml/src/ggml-sycl/cpy.hpp @@ -70,6 +70,39 @@ inline void cpy_blck_f32_q1_0(const char * cxi, char * cdsti) { } } +inline int round_nearest_int(float x) { + return (int)(x >= 0.0f ? x + 0.5f : x - 0.5f); +} + +inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + block_q2_0 * dsti = (block_q2_0 *) cdsti; + + float amax = 0.0f; + + for (int j = 0; j < QK2_0; ++j) { + amax = sycl::fmax(amax, sycl::fabs((float) xi[j])); + } + + const float d = amax; + const float id = d > 0.0f ? 1.0f / d : 0.0f; + + dsti->d = d; + + for (int j = 0; j < QK2_0 / 4; ++j) { + dsti->qs[j] = 0; + } + + for (int j = 0; j < QK2_0; ++j) { + int q = round_nearest_int(xi[j] * id) + 1; + q = dpct::max(0, dpct::min(3, q)); + + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + dsti->qs[byte_index] |= (uint8_t) q << bit_offset; + } +} + inline int best_index_mxfp4(const float x, const float e) { int best_index = 0; float best_err = sycl::fabs((float) (kvalues_mxfp4[0] * e - x)); From 9d9a6d29f6b981cc7f41983d26e56485c6af1811 Mon Sep 17 00:00:00 2001 From: Ozymandias_EBON <112784549+johnkarlhill@users.noreply.github.com> Date: Fri, 31 Jul 2026 02:43:16 -0500 Subject: [PATCH 04/98] =?UTF-8?q?SYCL:=20add=20oneMKL=20GEMM=20flash=20att?= =?UTF-8?q?ention=20for=20XMX-accelerated=20prompt=20proc=E2=80=A6=20(#250?= =?UTF-8?q?25)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * SYCL: add oneMKL GEMM flash attention for XMX-accelerated prompt processing * fattn-mkl: fix interleaved dst layout in normalize kernel - Fix mkl_fa_normalize_head: use interleaved dst layout ((query * n_q_heads + head) * DV) matching TILE's flash_attn_combine_results. Previously used dense head-major layout which wrote head outputs to wrong addresses, corrupting attention for all models except Qwen3.6-27B (where GQA=6 heads were sparse enough to avoid visible overlap). - Remove 7 redundant stream->wait() calls — SYCL in-order queue already serializes pure SYCL kernel dependencies. Retain only the 4 MKL GEMM ↔ SYCL handshake barriers (oneMKL GEMM uses its own internal queue that does not respect SYCL in-order). - Remove unused dst_row_stride, diagnostic clutter, and dead K/V hex dump (fa_diag block in fattn-mkl.cpp). - Add MKL_FA_DISABLE=1 env var for A/B testing. - Add FA-DISP watchdog (MKL_FA_DEBUG=1) and FA-DIAG output fingerprint (MKL_FA_DIAG=1) in fattn.cpp. Tested: Gemma-4-26B, Gemma-4-31B, Qwen3.6-27B, Qwen3.6-35B-A3B Perf (B70/Battlemage, 32K, q8_0 KV): Gemma-4-26B: 1473 t/s MKL vs 746 TILE (1.97x) Qwen3.6-27B: 609 t/s MKL vs 330 TILE (1.85x) Co-Authored-By: Claude Code on DeepSeek-v4-Pro * Thank you for the review feedback: rename env vars, use GGML_LOG_INFO, document in SYCL.md Completed the following: - Rename MKL_FA_DISABLE → GGML_SYCL_ENABLE_MKL_FA (inverted: 0 to disable) - Rename MKL_FA_DEBUG → GGML_SYCL_MKL_FA_DEBUG - Rename MKL_FA_DIAG → GGML_SYCL_MKL_FA_DIAG - Replace fprintf(stderr, ...) / fflush(stderr) with GGML_LOG_INFO() macro - Document all three env vars in docs/backend/SYCL.md under Runtime - Add comment explaining MKL FA activation trigger (flash-attn + quantized KV cache + batch-size >= 1024 + n_kv >= 1024) Resolves review feedback from arthw. Again, thank you!!! Co-Authored-By: Claude Code on DeepSeek-v4-Pro * Thank you for the review feedback round 2: use ggml_sycl_get_env, remove dup waits, gate perf macros - Replace raw getenv() with ggml_sycl_get_env() in all 4 env-var checks (fattn.cpp: GGML_SYCL_ENABLE_MKL_FA, GGML_SYCL_MKL_FA_DEBUG, GGML_SYCL_MKL_FA_DIAG; fattn-mkl.cpp: GGML_SYCL_MKL_FA_DEBUG) - Remove duplicated stream->wait() before ev.wait_and_throw() in GEMM KQ and GEMM VKQ — ev.wait_and_throw() already waits for completion - Gate MKL_ACCUM macro behind do_print so timing accumulators are no-ops in normal operation - Remove redundant MIT/Intel copyright header from fattn-mkl.cpp - Remove unused #include - Expand SYCL.md MKL FA docs with step-by-step activation trigger and example llama-cli command Again, thank you!!! Co-Authored-By: Claude Code on DeepSeek-v4-Pro * fattn-mkl: enable MKL FA for all KV cache types Remove the quantized-only restriction on MKL activation — the MKL kernel converts any non-F16 K/V to F16 via to_fp16_sycl before GEMM, so F16 (default), BF16, and F32 caches all benefit from XMX hardware acceleration. The type restriction was an unnecessary gate. Before (F16/BF16 default cache + FA on at 32K prefill): ~356 t/s (TILE path) After: ~670 t/s (MKL path, matching quantized-cache baseline) Minimal change: two conditions removed, one comment updated in fattn.cpp. No kernel or conversion code changes — the dequant pipeline already covers all types. * fattn-mkl: rename mkl_disable -> mkl_enable for clarity * fattn-mkl: refine MKL FA dispatch gates Three changes: 1. Remove quantized-only restriction - MKL FA activates for all KV cache types (F16 default, BF16, F32, quantized). The MKL kernel converts non-F16 K/V via to_fp16_sycl before GEMM. 2. Rename mkl_disable -> mkl_enable to match env var (GGML_SYCL_ENABLE_MKL_FA). 3. Replace batch-size threshold with Q->ne[1] >= 32 gate. Keeps TG (Q=1) and MTP drafts (Q=3-8) on VEC path where fused kernel beats MKL launch overhead. Routes all multi-token prefill through XMX-accelerated GEMM. Production data confirms Q patterns: 1-8 TG, 32-127 cache reuse, 128+ full reprocess. At 32K F16/BF16 FA-on: 356 -> 670 t/s. * ggml-sycl: fix F16 cache + MKL FA multi-turn corruption; add gate guards Two changes: 1. Always copy F16 K/V to dense row-major buffers before MKL GEMM. Previously F16 was read in-place with raw tensor strides. During multi-turn conversations, the accumulated KV cache had different stride properties than a fresh prefill, producing corrupted outputs. Now dense F16 gets a fast memcpy; interleaved (Gemma) gets a strided copy kernel. This matches what the quantized paths already did through to_fp16_sycl. 2. Gate MKL FA on unsupported op params (max_bias, logit_softcap, batch dim mismatch) and pathological F16 strides (nb[1] not a multiple of ne[0]*2). These conditions would previously crash inside the MKL kernel. Pathological strides (test-only) and ALiBi/softcap fall through to TILE/VEC which handle them correctly. The stride check uses modulo rather than equality, so both dense (nb1 == ne0*2) and interleaved (nb1 == H * ne0*2) pass — all real models use these layouts. Only test cases with overlapping rows (nb1=32 or nb1=75 for ne0=40) are blocked. Thanks to hmscider for the oneDNN FA PR (#25222) which surfaced the same insight: always normalize inputs to contiguous F16 before GEMM. Co-Authored-By: Claude Code using DeepSeek-V4-Pro * fattn-mkl: fix quant+GQA KV strides, tighten MKL gate, add K>=1024 tests Adding K>=1024 flash-attn test cases surfaced several MKL bugs: - Quant K/V with a padded seq-view (real KV cache) used the wrong strides in the dequant path... only the true Gemma interleave layout should reconstruct strides. nb[2] vs ne[1]*nb[1] - Gate was firing on shapes the kernel doesn't handle: head_dim < 64 or not a multiple of 64, MHA, attention sinks, and bf16 decode... fell through to vec which no bf16 case. Gate MKL to the validated envelope: gqa>=2, head_dim 64 through 512 (has to be a multiple of 64) with matching K/V head size, mask, no sinks/alibi/softcap... everything else falls back to tile. Covers Qwen Dense/MoE and Gemma4 Dense/MoE Ran test-backend-ops -o FLASH_ATTN_EXT: 3641/3641 pass. Perplexity unchanged... 6.7267 MKL vs 6.7290 stock using Qwen 27b q5_k_xl * Update ggml/src/ggml-sycl/fattn.cpp Co-authored-by: Neo Zhang * Update ggml/src/ggml-sycl/fattn.cpp Co-authored-by: Neo Zhang * Update ggml/src/ggml-sycl/fattn.cpp Co-authored-by: Neo Zhang * fattn-mkl: bound attention scratch so it doesn't grow with batch or context... also dropped the bf16 comment in fattn.cpp per arthw review. * Update ggml/src/ggml-sycl/fattn-mkl.cpp Co-authored-by: Neo Zhang * Update ggml/src/ggml-sycl/fattn-mkl.cpp Co-authored-by: Neo Zhang * apply arthw suggestions: enum for dequant modes, macro for wg_size, env-var one-liners --------- Co-authored-by: Claude Code using DeepSeek-V4-Pro Co-authored-by: Neo Zhang --- docs/backend/SYCL.md | 3 + ggml/src/ggml-sycl/fattn-mkl.cpp | 690 +++++++++++++++++++++++++++++++ ggml/src/ggml-sycl/fattn.cpp | 121 ++++++ ggml/src/ggml-sycl/fattn.hpp | 2 + tests/test-backend-ops.cpp | 12 + 5 files changed, 828 insertions(+) create mode 100644 ggml/src/ggml-sycl/fattn-mkl.cpp diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 914d63af22db..814e541e1a1b 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -797,6 +797,9 @@ use 1 SYCL GPUs: [0] with Max compute units:512 | GGML_SYCL_FA_ONEDNN | 1 (default) or 0 | Enable the oneDNN fused SDPA (flash-attention) path on supported GPUs. Set to 0 to always use the native SYCL flash-attention kernel. | | GGML_SYCL_FA_ONEDNN_MAX_KV | 0 (default, disabled) or positive integer | By default (0), all sequences are handled by the oneDNN fused SDPA path, regardless of KV length; a positive value caps that length, past which sequences fall back to the native kernel. If GPU driver watchdog resets (DEVICE_LOST) occur during long-context inference, set this near the context depth where they start, e.g. 24576. | | GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. | +| GGML_SYCL_ENABLE_MKL_FA | 1 (default) or 0 | Enable oneMKL GEMM flash attention for XMX-accelerated prompt processing with quantized KV cache. Automatically activates during prefill (prompt processing) when all conditions are met: (1) flash-attn enabled (`-fa` or `--flash-attn on`), (2) KV cache quantized (`--cache-type-k q8_0 --cache-type-v q8_0` or other `*_0/*_1` types), (3) batch size ≥ 1024 (`--batch-size 1024`), (4) prompt length ≥ 1024 tokens. Set to 0 to force the TILE kernel for A/B testing. Example minimum command: `llama-cli -m model.gguf -fa -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 --batch-size 1024 -p "your prompt"` | +| GGML_SYCL_MKL_FA_DEBUG | 0 (default) or 1 | Enable per-call diagnostic logging for MKL flash attention: GEMM/softmax timings, interleaved-head detection, and buffer memory usage. | +| GGML_SYCL_MKL_FA_DIAG | 0 (default) or 1 | Enable output fingerprinting for MKL flash attention. Dumps the first 64 float output values for the first 6 FA calls with n_kv ≥ 1024, labeled with kernel type (MKL/TILE/VEC) for cross-kernel comparison. | | GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute (currently top-k MoE gating). | | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.
Recommended to use when --split-mode = layer | | UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Allow SYCL/Unified Runtime Level Zero device allocations larger than 4 GiB. llama.cpp's direct Level Zero allocation path requests the relaxed maximum-size limit itself when GGML_SYCL_ENABLE_LEVEL_ZERO=1. | diff --git a/ggml/src/ggml-sycl/fattn-mkl.cpp b/ggml/src/ggml-sycl/fattn-mkl.cpp new file mode 100644 index 000000000000..fc22b7bdb8c5 --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-mkl.cpp @@ -0,0 +1,690 @@ +// Flash attention via oneMKL GEMM (XMX-accelerated). +// Uses column_major::gemm for Q*K^T and S*V matmuls +// with an online softmax SYCL kernel. +// +// All GQA query heads sharing a KV head are batched into single +// GEMM calls, amortizing MKL launch overhead across K and V reuse. +// + +#include "common.hpp" +#include "fattn-common.hpp" +#include "fattn-buffers.hpp" +#include "convert.hpp" +#include "fattn.hpp" + +#include +#include +#include + +#define MKL_FA_CHUNK_SIZE_KV 8192 + +// Number of query rows processed per tile. The score buffers (KQ_f32, S_f16) +// are sized q_tile_rows * chunk_size, so this bounds their footprint +// regardless of batch size (n_query_rows = n_queries * gqa_ratio). A typical +// single-ubatch prefill (e.g. ubatch 1024 * gqa 8 = 8192 rows) is exactly one +// tile, so it runs with no extra iterations. Larger batches tile and stay +// bounded. Override with GGML_SYCL_MKL_FA_Q_TILE. +#define MKL_FA_Q_TILE 8192 + +#define MKL_FA_WG_SIZE 256 + +using oneapi::mkl::transpose; +using oneapi::mkl::blas::column_major::gemm; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// Pack all GQA Q heads for one KV head into fp16, applying q_scale. +// Launches one kernel per GQA group — each kernel copies exactly +// n_queries * DKQ elements using the per-group dst offset and +// per-head source stride. +static void mkl_fa_pack_q_fp16( + dpct::queue_ptr stream, + sycl::half * __restrict dst, + const float * __restrict q_src, + int n_queries, int n_query_rows, int DKQ, + int gqa_ratio, int kvh_base_head, + float q_scale, int64_t q_row_stride, int64_t q_head_stride, + int64_t wg_size) { + + for (int iqg = 0; iqg < gqa_ratio; iqg++) { + int iqh = kvh_base_head + iqg; + sycl::half * dst_g = dst + (int64_t)iqg * n_queries * DKQ; + + const int64_t n_elem = (int64_t)n_queries * DKQ; + const int64_t wg = ((n_elem + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int64_t e = item.get_global_id(0); + if (e >= n_elem) return; + + int64_t q = e / DKQ; + int64_t d = e - q * DKQ; + + // Stride-aware source offset: handles permuted, + // sliced, or contiguous Q tensor layouts. + int64_t src_off = d + + q * q_row_stride + + (int64_t)iqh * q_head_stride; + + dst_g[e] = sycl::half( + q_src[src_off] * q_scale); + }); + }); + } +} + +// Zero-initialize the online softmax state arrays. +// KQ_max → -inf, KQ_sum → 0, VKQ_accum → 0. +// Merged into one kernel to avoid per-array launch overhead. +static void mkl_fa_init_softmax_state( + dpct::queue_ptr stream, + float * kmax, float * ksum, float * vacc, + int n_query_rows, int DV, int64_t wg_size) { + + const float neg_inf = -1e30f; + const int64_t n_maxsum = n_query_rows; + const int64_t n_vacc = (int64_t)n_query_rows * DV; + const int64_t total = (n_vacc > n_maxsum) ? n_vacc : n_maxsum; + const int64_t wg = ((total + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int64_t i = item.get_global_id(0); + if (i < n_maxsum) { + kmax[i] = neg_inf; + ksum[i] = 0.0f; + } + if (i < n_vacc) { + vacc[i] = 0.0f; + } + }); + }); +} + +// Online softmax over one KV chunk for a tile of GQA query rows. +// The tile spans absolute rows [q0, q0 + q_rows). Score buffers +// (KQ_f32/S_f16) are indexed RELATIVE to the tile; the persistent state +// (VKQ_accum/KQ_max/KQ_sum) and mask are indexed by ABSOLUTE row. +// For each row: find local max → rescale previous VKQ_accum → +// compute exp(s - max) → write S_f16 → update running max/sum. +static void mkl_fa_online_softmax_chunk( + dpct::queue_ptr stream, + float * __restrict KQ_f32, + sycl::half * __restrict S_f16, + float * __restrict KQ_max, + float * __restrict KQ_sum, + float * __restrict VKQ_accum, + int q0, int q_rows, int n_queries, int DV, + int chunk_size, int chunk_start, + int kvh_head, int gqa_ratio, + const sycl::half * mask_data, int64_t mask_head_stride, + int64_t mask_row_stride, int mask_n_heads, + float logit_softcap, int64_t wg_size) { + + const int64_t wg = ((q_rows + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int jc_rel = item.get_global_id(0); + if (jc_rel >= q_rows) return; + int jc_abs = q0 + jc_rel; + + const int gqa_group = jc_abs / n_queries; + const int q_row = jc_abs % n_queries; + + // Score buffers are tile-local (relative index). + const float * __restrict KQ_row = KQ_f32 + + jc_rel * (int64_t)chunk_size; + // Persistent accumulator is full-sized (absolute index). + float * __restrict vkq = VKQ_accum + + jc_abs * (int64_t)DV; + + const sycl::half * mask_h = nullptr; + int64_t m_stride = 0; + if (mask_data) { + int m_head = (mask_n_heads > 1) + ? (kvh_head + gqa_group) : 0; + mask_h = mask_data + (int64_t)m_head * mask_head_stride; + m_stride = mask_row_stride; + } + + // Row-wise local maximum (softcap before mask) + float local_max = -1e30f; + for (int i = 0; i < chunk_size; i++) { + float s = KQ_row[i]; + if (logit_softcap != 0.0f) { + s = logit_softcap * sycl::tanh(s); + } + if (mask_h) { + s += (float)mask_h[q_row * m_stride + + (chunk_start + i)]; + } + if (s > local_max) local_max = s; + } + + // Rescale previous accumulator by exp(old_max - new_max) + float old_max = KQ_max[jc_abs]; + float new_max = (old_max > local_max) ? old_max : local_max; + float rescale = (old_max < -1e29f) ? 1.0f + : sycl::native::exp(old_max - new_max); + + for (int v = 0; v < DV; v++) { + vkq[v] *= rescale; + } + + // Softmax and write S_f16 (tile-local index) + float local_sum = 0.0f; + sycl::half * __restrict S_row = S_f16 + + jc_rel * (int64_t)chunk_size; + + for (int i = 0; i < chunk_size; i++) { + float s = KQ_row[i]; + if (logit_softcap != 0.0f) { + s = logit_softcap * sycl::tanh(s); + } + if (mask_h) { + s += (float)mask_h[q_row * m_stride + + (chunk_start + i)]; + } + float val = sycl::native::exp(s - new_max); + S_row[i] = sycl::half(val); + local_sum += val; + } + + KQ_sum[jc_abs] = KQ_sum[jc_abs] * rescale + local_sum; + KQ_max[jc_abs] = new_max; + }); + }); +} + +// Write one GQA group's normalized output to its destination head. +static void mkl_fa_normalize_head( + dpct::queue_ptr stream, + float * __restrict dst_batch, + const float * __restrict VKQ_accum, + const float * __restrict KQ_sum, + int iqh, int n_queries, int DV, int n_q_heads, + int64_t src_offset, int64_t wg_size) { + + const int64_t wg = ((n_queries + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int jc = item.get_global_id(0); + if (jc >= n_queries) return; + + int ksum_idx = (int)(src_offset / DV) + jc; + float inv_sum = 1.0f / KQ_sum[ksum_idx]; + const float * __restrict src = VKQ_accum + + src_offset + jc * (int64_t)DV; + // Interleaved dst layout (matching TILE): + // rows alternate between heads, then increment query. + // offset = (query * n_q_heads + head) * DV + float * __restrict dst_row = dst_batch + + ((int64_t)jc * n_q_heads + iqh) * (int64_t)DV; + + for (int v = 0; v < DV; v++) { + dst_row[v] = src[v] * inv_sum; + } + }); + }); +} + +// --------------------------------------------------------------------------- +// Per-chunk dequant +// +// Rather than dequantizing all of K/V up front (footprint scales with +// context), we dequant one KV-head chunk at a time into a dense +// [this_chunk x D] fp16 buffer (row-major, lda = D). The source address of +// element (head=ikvh, row=chunk_start+r, col=c) decomposes into independent +// linear terms head_off(ikvh) + row_off(chunk_start) + (r,c), so slicing a +// chunk is a clean pointer offset in every layout case. The true-Gemma- +// interleave vs padded-seq-view distinction is resolved once when the +// descriptor is built; slicing does not reintroduce it. +// --------------------------------------------------------------------------- +enum mkl_fa_kv_desc_mode { + MKL_FA_KV_MODE_F16_DENSE = 0, + MKL_FA_KV_MODE_F16_INTERLEAVED = 1, + MKL_FA_KV_MODE_QUANT_CONTIG = 2, + MKL_FA_KV_MODE_QUANT_NC = 3, +}; + +struct mkl_fa_kv_desc { + const char * data = nullptr; + ggml_type type = GGML_TYPE_F16; + int64_t D = 0; // ne[0] + int64_t nb1 = 0; // byte stride, seq dim + int64_t nb2 = 0; // byte stride, head dim + mkl_fa_kv_desc_mode mode = MKL_FA_KV_MODE_F16_DENSE; + int64_t ts = 0; // type size (mode 3 base offset) + int64_t s01 = 0; // nc row stride in blocks (mode 3) + int64_t s02 = 0; // nc head stride in blocks (mode 3) +}; + +static mkl_fa_kv_desc mkl_fa_make_desc(const ggml_tensor * T, bool interleaved, int n_kv_heads) { + mkl_fa_kv_desc d; + d.data = (const char *)T->data; + d.type = T->type; + d.D = T->ne[0]; + d.nb1 = (int64_t)T->nb[1]; + d.nb2 = (int64_t)T->nb[2]; + d.ts = (int64_t)ggml_type_size(T->type); + + if (T->type == GGML_TYPE_F16) { + d.mode = interleaved ? MKL_FA_KV_MODE_F16_INTERLEAVED + : MKL_FA_KV_MODE_F16_DENSE; + } else if (ggml_is_contiguously_allocated(T) && !interleaved) { + d.mode = MKL_FA_KV_MODE_QUANT_CONTIG; + } else { + d.mode = MKL_FA_KV_MODE_QUANT_NC; + const int64_t bs = (int64_t)ggml_blck_size(T->type); + const int64_t blk_per_row = T->ne[0] / bs; + // True Gemma interleave packs heads within a row (nb[2] < ne[1]*nb[1]) + // → reconstruct physical strides. Padded seq-views (nb[2] > ne[1]*nb[1]) + // already have correct physical strides. + const bool gemma = interleaved && + ((int64_t)T->nb[2] < (int64_t)T->ne[1] * (int64_t)T->nb[1]); + if (gemma) { + d.s01 = (int64_t)n_kv_heads * blk_per_row; + d.s02 = blk_per_row; + } else { + d.s01 = d.nb1 / d.ts; + d.s02 = d.nb2 / d.ts; + } + } + return d; +} + +// Dequant one KV-head chunk into a dense [this_chunk x D] fp16 buffer. +static void mkl_fa_dequant_chunk( + dpct::queue_ptr stream, const mkl_fa_kv_desc & d, ggml_tensor * dst_ctx, + sycl::half * out, int ikvh, int chunk_start, int this_chunk) { + + const int64_t D = d.D; + switch (d.mode) { + case MKL_FA_KV_MODE_F16_DENSE: { + const char * base = d.data + (int64_t)ikvh * d.nb2 + + (int64_t)chunk_start * d.nb1; + stream->memcpy(out, base, (size_t)this_chunk * D * sizeof(sycl::half)); + break; + } + case MKL_FA_KV_MODE_F16_INTERLEAVED: { + const char * base = d.data + (int64_t)ikvh * d.nb2 + + (int64_t)chunk_start * d.nb1; + const int64_t row_halfs = d.nb1 / (int64_t)sizeof(sycl::half); + const sycl::half * src = (const sycl::half *)base; + stream->parallel_for( + sycl::range<2>((size_t)this_chunk, (size_t)D), + [=](sycl::item<2> it) { + int64_t r = it.get_id(0); + int64_t c = it.get_id(1); + out[r * D + c] = src[r * row_halfs + c]; + }); + break; + } + case MKL_FA_KV_MODE_QUANT_CONTIG: { + const char * base = d.data + (int64_t)ikvh * d.nb2 + + (int64_t)chunk_start * d.nb1; + to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(d.type, dst_ctx); + to_fp16(base, out, (int64_t)this_chunk * D, stream); + break; + } + default: { // MKL_FA_KV_MODE_QUANT_NC + to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(d.type); + const int64_t base_blocks = (int64_t)ikvh * d.s02 + + (int64_t)chunk_start * d.s01; + const char * base = d.data + base_blocks * d.ts; + // ne02 = ne03 = 1 → s02/s03 inert; head+chunk offset carried by base. + to_fp16(base, out, D, this_chunk, 1, 1, d.s01, d.s02, d.s02, stream); + break; + } + } +} + +// --------------------------------------------------------------------------- +// MKL Flash Attention orchestrator +// +// Pipeline: dequantize K/V → for each KV head: +// pack GQA Q heads → MKL GEMM KQ → online softmax → +// MKL GEMM VKQ → accumulate → normalize → scatter to dst +// --------------------------------------------------------------------------- +void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + ggml_tensor * KQV = dst; + + GGML_ASSERT(Q->type == GGML_TYPE_F32); + GGML_ASSERT(KQV->type == GGML_TYPE_F32); + + // --- Op params --- + float scale = 1.0f, max_bias = 0.0f, logit_softcap = 0.0f; + memcpy(&scale, (const float *)KQV->op_params + 0, sizeof(float)); + memcpy(&max_bias, (const float *)KQV->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *)KQV->op_params + 2, sizeof(float)); + + const float q_scale = scale; + + // --- Dimensions --- + const int DKQ = (int)K->ne[0]; + const int DV = (int)V->ne[0]; + const int n_queries = (int)Q->ne[1]; + const int n_q_heads = (int)Q->ne[2]; + const int n_kv_heads = (int)K->ne[2]; + const int n_batch = (int)Q->ne[3]; + const int n_kv = (int)K->ne[1]; + const int gqa_ratio = n_q_heads / n_kv_heads; + const int n_query_rows = n_queries * gqa_ratio; + + GGML_ASSERT(n_q_heads % n_kv_heads == 0); + GGML_ASSERT(max_bias == 0.0f); // ALiBi not supported + GGML_ASSERT(Q->ne[3] == K->ne[3] || K->ne[3] == 1); + + const int chunk_size = std::min(MKL_FA_CHUNK_SIZE_KV, n_kv); + + // Query rows are processed in tiles of q_tile_rows so the score buffers + // (KQ_f32/S_f16 = q_tile_rows * chunk_size) stay bounded regardless of + // batch size. n_query_rows <= Q_TILE is a single tile (no extra work). + static int q_tile_env = ggml_sycl_get_env("GGML_SYCL_MKL_FA_Q_TILE", MKL_FA_Q_TILE); + const int q_tile_rows = std::max(1, std::min(q_tile_env, n_query_rows)); + + const int64_t wg_size = MKL_FA_WG_SIZE; + + // --- Debug output (gated by GGML_SYCL_MKL_FA_DEBUG=1) --- + static int mkl_call_count = 0; + mkl_call_count++; + static int mkl_debug = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DEBUG", 0); + const bool do_print = (mkl_debug == 1); + + const int64_t q_row_stride = Q->nb[1] / sizeof(float); + const int64_t q_head_stride = Q->nb[2] / sizeof(float); + + const bool V_is_K_view = V->view_src + && (V->view_src == K || (V->view_src == K->view_src + && V->view_offs == K->view_offs)); + + // Early interleaved detection for debug output. + // True interleaved detection happens after dequant (nb12_fp16 == nb11_fp16), + // but we can pre-detect on the original tensor strides. + const bool k_early_interleaved = + ((int64_t)K->ne[1] * K->nb[1] != K->nb[2]); + const bool v_early_interleaved = + !V_is_K_view && ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]); + + if (do_print) { + GGML_LOG_INFO("[MKL-FA] #%d D=%d DV=%d n_q=%d n_kv=%d " + "n_qh=%d n_kvh=%d gqa=%d batch=%d K=%s V=%s " + "chunk=%d buf=%.1fMB%s%s\n", + mkl_call_count, DKQ, DV, n_queries, n_kv, + n_q_heads, n_kv_heads, gqa_ratio, n_batch, + ggml_type_name(K->type), ggml_type_name(V->type), + chunk_size, + (double)((int64_t)n_query_rows * chunk_size * sizeof(float)) + / (1024.0 * 1024.0), + k_early_interleaved ? " K_ILV" : "", + v_early_interleaved ? " V_ILV" : ""); + GGML_LOG_INFO("[MKL-FA] #%d Q-nb1=%lld Q-nb2=%lld " + "q_rs=%lld q_hs=%lld dst_rs=%lld dst_hs=%lld\n", + mkl_call_count, + (long long)Q->nb[1], (long long)Q->nb[2], + (long long)q_row_stride, (long long)q_head_stride, + (long long)(KQV->nb[1] / sizeof(float)), + (long long)(KQV->nb[2] / sizeof(float))); + } + + // --- Stream and allocators --- + dpct::queue_ptr stream = ctx.stream(); + +#define MKL_TAKE_TIME(t0) auto t0 = std::chrono::steady_clock::now() +#define MKL_ACCUM(acc, t0) do { if (do_print) { \ + acc += (int64_t)std::chrono::duration_cast \ + (std::chrono::steady_clock::now() - (t0)).count(); \ +} } while(0) + + int64_t gemm_kq_time_us = 0; + int64_t gemm_vkq_time_us = 0; + int64_t softmax_time_us = 0; + int64_t dequant_time_us = 0; + + MKL_TAKE_TIME(t_deq); + + // --- K/V dequant descriptors --- + // Dequant is done per-chunk inside the KV loop (footprint independent of + // context). Output is always dense row-major fp16 [this_chunk x D], lda=D. + // Interleaved detection: ne[1]*nb[1] != nb[2] means heads are interleaved. + const bool k_interleaved = + ((int64_t)K->ne[1] * K->nb[1] != K->nb[2]) && K->ne[2] > 1; + const bool v_interleaved = + ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]) && V->ne[2] > 1; + + const mkl_fa_kv_desc K_desc = mkl_fa_make_desc(K, k_interleaved, n_kv_heads); + const mkl_fa_kv_desc V_desc = V_is_K_view + ? K_desc : mkl_fa_make_desc(V, v_interleaved, n_kv_heads); + + MKL_ACCUM(dequant_time_us, t_deq); + + // --- Resolve mask pointers --- + const sycl::half * mask_data = nullptr; + int64_t mask_head_stride = 0; + int64_t mask_row_stride = 0; + int mask_n_heads = 0; + + if (mask) { + // Use actual fp16 device size (2 bytes), NOT sizeof(sycl::half) + // which may be 4 on the host in oneAPI. + mask_head_stride = mask->nb[2] / 2; + mask_row_stride = mask->nb[1] / 2; + mask_n_heads = (int)mask->ne[2]; + } + + // --- Allocate intermediates from pool --- + ggml_sycl_pool & pool = ctx.pool(); + + ggml_sycl_pool_alloc KQ_f32(pool); // [q_tile_rows x chunk] + ggml_sycl_pool_alloc S_f16(pool); // [q_tile_rows x chunk] + ggml_sycl_pool_alloc VKQ_chunk(pool); // [q_tile_rows x DV] + ggml_sycl_pool_alloc VKQ_accum(pool); // [n_query_rows x DV] (full) + ggml_sycl_pool_alloc KQ_max(pool); // [n_query_rows] (full) + ggml_sycl_pool_alloc KQ_sum(pool); // [n_query_rows] (full) + ggml_sycl_pool_alloc Q_head_f16(pool); // [n_query_rows x DKQ] (full) + ggml_sycl_pool_alloc K_chunk_f16(pool); // [chunk x DKQ] (per-chunk dequant) + ggml_sycl_pool_alloc V_chunk_f16(pool); // [chunk x DV] (per-chunk dequant) + + KQ_f32.alloc((size_t)q_tile_rows * chunk_size); + S_f16.alloc((size_t)q_tile_rows * chunk_size); + VKQ_chunk.alloc((size_t)q_tile_rows * DV); + VKQ_accum.alloc((size_t)n_query_rows * DV); + KQ_max.alloc(n_query_rows); + KQ_sum.alloc(n_query_rows); + Q_head_f16.alloc((size_t)n_query_rows * DKQ); + K_chunk_f16.alloc((size_t)chunk_size * DKQ); + + sycl::half * V_chunk_f16_ptr; + if (V_is_K_view) { + V_chunk_f16_ptr = K_chunk_f16.ptr; // V aliases K (DV == DKQ) + } else { + V_chunk_f16.alloc((size_t)chunk_size * DV); + V_chunk_f16_ptr = V_chunk_f16.ptr; + } + + sycl::half * Q_head_f16_ptr = Q_head_f16.ptr; + float * KQ_f32_ptr = KQ_f32.ptr; + sycl::half * S_f16_ptr = S_f16.ptr; + float * VKQ_chunk_ptr = VKQ_chunk.ptr; + float * VKQ_accum_ptr = VKQ_accum.ptr; + float * KQ_max_ptr = KQ_max.ptr; + float * KQ_sum_ptr = KQ_sum.ptr; + sycl::half * K_chunk_f16_ptr = K_chunk_f16.ptr; + + const float alpha = 1.0f; + const float beta = 0.0f; + + for (int ib = 0; ib < n_batch; ib++) { + const float * Q_batch = (const float *)Q->data + + ib * (Q->nb[3] / sizeof(float)); + float * dst_batch = (float *)KQV->data + + ib * (KQV->nb[3] / sizeof(float)); + + const sycl::half * mask_batch = nullptr; + if (mask) { + int m_batch = (mask->ne[3] > 1) ? ib : 0; + mask_batch = (const sycl::half *)mask->data + + m_batch * (mask->nb[3] / 2); // 2 = actual fp16 device size + } + + for (int ikvh = 0; ikvh < n_kv_heads; ikvh++) { + int kvh_base_head = ikvh * gqa_ratio; + + // 1. Pack all GQA Q heads into fp16 (full n_query_rows) + mkl_fa_pack_q_fp16(stream, + Q_head_f16_ptr, Q_batch, + n_queries, n_query_rows, DKQ, + gqa_ratio, kvh_base_head, + q_scale, q_row_stride, q_head_stride, wg_size); + + // 2. Initialize softmax state (full n_query_rows) + mkl_fa_init_softmax_state(stream, + KQ_max_ptr, KQ_sum_ptr, VKQ_accum_ptr, + n_query_rows, DV, wg_size); + + // Sync before MKL GEMM (MKL may use an internal queue) + stream->wait(); + + // 3. KV chunk loop (OUTER): dequant each chunk once, then tile queries. + for (int chunk_start = 0; chunk_start < n_kv; chunk_start += chunk_size) { + int this_chunk = std::min(chunk_size, n_kv - chunk_start); + + // 3a. Dequant this KV chunk to dense fp16 (once per chunk) + { + MKL_TAKE_TIME(t0); + mkl_fa_dequant_chunk(stream, K_desc, KQV, + K_chunk_f16_ptr, ikvh, chunk_start, this_chunk); + if (!V_is_K_view) { + mkl_fa_dequant_chunk(stream, V_desc, KQV, + V_chunk_f16_ptr, ikvh, chunk_start, this_chunk); + } + stream->wait(); // dequant must be ready before MKL GEMM + MKL_ACCUM(dequant_time_us, t0); + } + + // 3b. Query tile loop (INNER) — bounds KQ_f32/S_f16 footprint. + for (int q0 = 0; q0 < n_query_rows; q0 += q_tile_rows) { + int q_rows = std::min(q_tile_rows, n_query_rows - q0); + + // GEMM: KQ = Q_tile × K_chunk^T + { + MKL_TAKE_TIME(t0); + sycl::event ev = gemm(*stream, + transpose::trans, transpose::nontrans, + this_chunk, q_rows, DKQ, + alpha, + K_chunk_f16_ptr, DKQ, + Q_head_f16_ptr + (int64_t)q0 * DKQ, DKQ, + beta, + KQ_f32_ptr, this_chunk); + try { ev.wait_and_throw(); } catch (sycl::exception & e) { + GGML_LOG_INFO("[MKL-FA] GEMM KQ: %s\n", e.what()); + GGML_ABORT("MKL GEMM KQ failed"); + } + MKL_ACCUM(gemm_kq_time_us, t0); + } + // Online softmax over this chunk for this query tile + { + MKL_TAKE_TIME(t0); + mkl_fa_online_softmax_chunk(stream, + KQ_f32_ptr, S_f16_ptr, + KQ_max_ptr, KQ_sum_ptr, VKQ_accum_ptr, + q0, q_rows, n_queries, DV, + this_chunk, chunk_start, + kvh_base_head, gqa_ratio, + mask_batch, mask_head_stride, + mask_row_stride, mask_n_heads, + logit_softcap, wg_size); + stream->wait(); // S_f16 must be ready for GEMM + MKL_ACCUM(softmax_time_us, t0); + } + + // GEMM: VKQ_chunk = S × V_chunk + { + MKL_TAKE_TIME(t0); + sycl::event ev = gemm(*stream, + transpose::nontrans, transpose::nontrans, + DV, q_rows, this_chunk, + alpha, + V_chunk_f16_ptr, DV, + S_f16_ptr, this_chunk, + beta, + VKQ_chunk_ptr, DV); + try { ev.wait_and_throw(); } catch (sycl::exception & e) { + GGML_LOG_INFO("[MKL-FA] GEMM VKQ: %s\n", e.what()); + GGML_ABORT("MKL GEMM VKQ failed"); + } + MKL_ACCUM(gemm_vkq_time_us, t0); + } + // VKQ_accum[q0..] += VKQ_chunk + { + const int64_t n_total = (int64_t)q_rows * DV; + const int64_t wg = ((n_total + wg_size - 1) / wg_size) + * wg_size; + float * accum = VKQ_accum_ptr + (int64_t)q0 * DV; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int64_t i = item.get_global_id(0); + if (i < n_total) { + accum[i] += VKQ_chunk_ptr[i]; + } + }); + }); + } + } + } + + // 4. Normalize and scatter each GQA head to dst + for (int iqg = 0; iqg < gqa_ratio; iqg++) { + int iqh = kvh_base_head + iqg; + int64_t src_offset = (int64_t)iqg * n_queries * DV; + mkl_fa_normalize_head(stream, + dst_batch, VKQ_accum_ptr, KQ_sum_ptr, + iqh, n_queries, DV, n_q_heads, + src_offset, wg_size); + } + } + } + +#undef MKL_TAKE_TIME +#undef MKL_ACCUM + + if (do_print) { + const int64_t v_chunk_elems = V_is_K_view ? 0 : (int64_t)chunk_size * DV; + double total_mb = (double)( + (int64_t)q_tile_rows * chunk_size * sizeof(float) // KQ_f32 + + (int64_t)q_tile_rows * chunk_size * sizeof(sycl::half) // S_f16 + + (int64_t)q_tile_rows * DV * sizeof(float) // VKQ_chunk + + (int64_t)n_query_rows * DV * sizeof(float) // VKQ_accum + + (int64_t)n_query_rows * sizeof(float) // KQ_max + + (int64_t)n_query_rows * sizeof(float) // KQ_sum + + (int64_t)n_query_rows * DKQ * sizeof(sycl::half) // Q_head_f16 + + (int64_t)chunk_size * DKQ * sizeof(sycl::half) // K_chunk_f16 + + v_chunk_elems * (int64_t)sizeof(sycl::half) // V_chunk_f16 + ) / (1024.0 * 1024.0); + GGML_LOG_INFO("[MKL-FA] #%d n_kv=%d n_q=%d q_tile=%d time_us: " + "dequant=%lld GEMM_KQ=%lld softmax=%lld GEMM_VKQ=%lld " + "buf_mb=%.1f\n", + mkl_call_count, n_kv, n_queries, q_tile_rows, + (long long)dequant_time_us, + (long long)gemm_kq_time_us, + (long long)softmax_time_us, + (long long)gemm_vkq_time_us, + total_mb); + } +} diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp index 1772b9c8584d..f8efa53acdeb 100644 --- a/ggml/src/ggml-sycl/fattn.cpp +++ b/ggml/src/ggml-sycl/fattn.cpp @@ -99,8 +99,10 @@ enum best_fattn_kernel { BEST_FATTN_KERNEL_VEC = 100, BEST_FATTN_KERNEL_ONEDNN = 150, // added enum for onednn==150 BEST_FATTN_KERNEL_TILE = 200, + BEST_FATTN_KERNEL_MKL = 300, }; + static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const ggml_tensor * dst) { GGML_UNUSED(device); #ifndef SYCL_FLASH_ATTN @@ -115,6 +117,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; const int gqa_ratio = Q->ne[2] / K->ne[2]; GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); @@ -122,7 +125,49 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const float max_bias = 0.0f; memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); + float logit_softcap = 0.0f; + memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); + bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; + + // MKL path: XMX-accelerated GEMM for prompt processing (all KV cache types). + // The MKL kernel converts non-F16 K/V to F16 via to_fp16_sycl before GEMM, + // so quantized, F16, BF16, and F32 caches all benefit from XMX acceleration. + // Activates automatically when flash-attn is enabled (--flash-attn on or -fa) + // and n_kv >= 1024. Falls through to TILE/VEC for ALiBi, logit softcap, + // and mismatched batch dimensions (unsupported by the MKL kernel). + // Set GGML_SYCL_ENABLE_MKL_FA=0 to force TILE/VEC path for A/B testing. + // Example: GGML_SYCL_ENABLE_MKL_FA=0 llama-cli -m model.gguf -fa -ngl 99 ... + // Note: MKL GEMM calls are incompatible with SYCL graph capture replay. + static int mkl_enable = ggml_sycl_get_env("GGML_SYCL_ENABLE_MKL_FA", 1); + // MKL is validated for the mainstream GQA envelope: grouped-query + // (gqa_ratio >= 2), head_dim a multiple of 64 in [64,512] with matching + // K/V head size, mask, no sinks/ALiBi/softcap. Gemma's global layers use + // head_dim 512, so the cap must include it. Head sizes not a multiple of + // 64 (72/80/96), MHA (gqa_ratio == 1), and MLA (DKQ != DV, e.g. 576/512) + // fall through to TILE/VEC; see follow-up work. + if (mkl_enable == 1 && mask && !sinks && gqa_ratio >= 2 && + Q->ne[0] >= 64 && Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && + Q->ne[0] == V->ne[0] && + Q->ne[1] >= 32 && K->ne[1] >= 1024 && + max_bias == 0.0f && logit_softcap == 0.0f && + (Q->ne[3] == K->ne[3] || K->ne[3] == 1)) { + // F16 K/V strides must be a multiple of ne[0]*2 (the natural row size + // in bytes). This passes both dense (nb1 == ne0*2) and interleaved + // (nb1 == H * ne0*2). Only pathological test strides like nb1=32 or + // nb1=75 for ne0=40 fall through to TILE. + bool kv_strides_ok = true; + for (const ggml_tensor * t : {K, V}) { + if (t->type == GGML_TYPE_F16 && t->nb[1] % (t->ne[0] * 2) != 0) { + kv_strides_ok = false; + break; + } + } + if (kv_strides_ok) { + return BEST_FATTN_KERNEL_MKL; + } + } + for (const ggml_tensor * t : {Q, K, V, mask}) { if (t == nullptr || ggml_is_quantized(t->type)) { continue; @@ -216,6 +261,37 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_sycl_set_device(ctx.device); + + // n_kv watchdog: log when n_kv differs from the last FA call with + // the same D — helps detect cache-truncation issues. + static int nkv_debug = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DEBUG", 0); + if (nkv_debug == 1) { + const ggml_tensor * K_dbg = dst->src[1]; + const ggml_tensor * V_dbg = dst->src[2]; + static int64_t last_nkv_d256 = 0, last_nkv_d512 = 0; + static int fa_call_seq = 0; + fa_call_seq++; + int64_t cur_nkv = K_dbg->ne[1]; + int Dk = (int)K_dbg->ne[0]; + const char * kname = "TILE"; + best_fattn_kernel k = ggml_sycl_get_best_fattn_kernel(ctx.device, dst); + if (k == BEST_FATTN_KERNEL_MKL) kname = "MKL"; + if (k == BEST_FATTN_KERNEL_VEC) kname = "VEC"; + int64_t delta = 0; + if (Dk == 256) { + delta = cur_nkv - last_nkv_d256; + last_nkv_d256 = cur_nkv; + } else if (Dk == 512) { + delta = cur_nkv - last_nkv_d512; + last_nkv_d512 = cur_nkv; + } + GGML_LOG_INFO("[FA-DISP] #%d %s D=%d n_kv=%lld delta=%lld " + "V_ne1=%lld\n", + fa_call_seq, kname, Dk, + (long long)cur_nkv, (long long)delta, + (long long)V_dbg->ne[1]); + } + switch (ggml_sycl_get_best_fattn_kernel(ggml_sycl_get_device(), dst)) { case BEST_FATTN_KERNEL_NONE: GGML_ABORT("Not support Flash-Attention"); @@ -232,6 +308,51 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst case BEST_FATTN_KERNEL_VEC: ggml_sycl_flash_attn_ext_vec(ctx, dst); break; + case BEST_FATTN_KERNEL_MKL: + ggml_sycl_flash_attn_ext_mkl(ctx, dst); + break; + } + + // --- Output fingerprint (GGML_SYCL_MKL_FA_DIAG=1) --- + // Copy first 64 float output values to host for fingerprinting. + // Compare MKL vs TILE (GGML_SYCL_ENABLE_MKL_FA=0) to detect divergence. + // Only fingerprints the first 6 FA calls with n_kv >= 1024. + static int fa_diag = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DIAG", 0); + static int fa_diag_count = 0; + if (fa_diag == 1 && fa_diag_count < 6) { + const ggml_tensor * K_diag = dst->src[1]; + const ggml_tensor * V_diag = dst->src[2]; + const ggml_tensor * Q_diag = dst->src[0]; + if (K_diag->ne[1] >= 1024) { + fa_diag_count++; + float diag_buf[64]; + dpct::queue_ptr q = ctx.stream(); + q->memcpy(diag_buf, dst->data, 64 * sizeof(float)); + q->wait(); + const char * kname = "???"; + best_fattn_kernel kb = ggml_sycl_get_best_fattn_kernel(ctx.device, dst); + if (kb == BEST_FATTN_KERNEL_MKL) kname = "MKL"; + if (kb == BEST_FATTN_KERNEL_TILE) kname = "TILE"; + if (kb == BEST_FATTN_KERNEL_VEC) kname = "VEC"; + GGML_LOG_INFO("[FA-DIAG] #%d %s D=%d n_kv=%lld n_q=%lld " + "n_qh=%lld n_kvh=%lld K=%s V=%s " + "nb1=%zu nb2=%zu first 64 floats:\n", + fa_diag_count, kname, + (int)K_diag->ne[0], (long long)K_diag->ne[1], + (long long)Q_diag->ne[1], + (long long)Q_diag->ne[2], (long long)K_diag->ne[2], + ggml_type_name(K_diag->type), + ggml_type_name(V_diag->type), + K_diag->nb[1], K_diag->nb[2]); + for (int i = 0; i < 64; i += 8) { + GGML_LOG_INFO(" [%2d] %08x %08x %08x %08x %08x %08x %08x %08x\n", + i, + *(unsigned *)&diag_buf[i+0], *(unsigned *)&diag_buf[i+1], + *(unsigned *)&diag_buf[i+2], *(unsigned *)&diag_buf[i+3], + *(unsigned *)&diag_buf[i+4], *(unsigned *)&diag_buf[i+5], + *(unsigned *)&diag_buf[i+6], *(unsigned *)&diag_buf[i+7]); + } + } } } diff --git a/ggml/src/ggml-sycl/fattn.hpp b/ggml/src/ggml-sycl/fattn.hpp index f2a8ffc97dee..c093970a3fed 100644 --- a/ggml/src/ggml-sycl/fattn.hpp +++ b/ggml/src/ggml-sycl/fattn.hpp @@ -19,4 +19,6 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst bool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst); +void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + #endif // GGML_SYCL_FATTN_HPP diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 162bfc98c8b3..e101ca390526 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9513,6 +9513,18 @@ static std::vector> make_test_cases_eval() { } } + // prefill-shaped cases with long KV (nb >= 32, kv >= 1024): covers the + // XMX/GEMM-accelerated SYCL FA path which only activates for these shapes. + for (int kv : { 1024, 2048, }) { + for (int hs : { 64, 128, 256, }) { + for (int nb : { 32, 64, }) { + for (ggml_type type_KV : { GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, }) { + test_cases.emplace_back(new test_flash_attn_ext(hs, hs, 8, {4, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, type_KV, type_KV)); + } + } + } + } + for (int hsk : { 40, 64, 72, 80, 96, 128, 192, 256, 320, 512, 576 }) { for (int hsv : { 40, 64, 72, 80, 96, 128, 192, 256, 512 }) { if (hsk != 192 && hsk != 320 && hsk != 576 && hsk != hsv) continue; From 15e755f30d3deb75fea2c9cc8cd129a8d7cf634d Mon Sep 17 00:00:00 2001 From: David Friehs Date: Fri, 31 Jul 2026 10:15:44 +0200 Subject: [PATCH 05/98] cuda: extract Q2_0 elements via __byte_perm (#25603) --- ggml/src/ggml-cuda/mmq-load-tiles.cuh | 48 +++++++++---------------- ggml/src/ggml-cuda/vecdotq.cuh | 50 ++++++++------------------- 2 files changed, 31 insertions(+), 67 deletions(-) diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh index 8c9070c6bebf..8ed704c281a4 100644 --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -130,43 +130,27 @@ template static __device__ __forceinline_ } const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + kbx; - // Each 32-element chunk occupies 8 bytes of qs (32 elements * 2 bits = 64 bits) - const int qs_offset = 8*kqsx; - const int qs0 = bxi->qs[qs_offset + 0] | (bxi->qs[qs_offset + 1] << 8) | - (bxi->qs[qs_offset + 2] << 16) | (bxi->qs[qs_offset + 3] << 24); - const int qs1 = bxi->qs[qs_offset + 4] | (bxi->qs[qs_offset + 5] << 8) | - (bxi->qs[qs_offset + 6] << 16) | (bxi->qs[qs_offset + 7] << 24); - - // Unpack 32 2-bit codes into 8 int32s, each holding 4 signed int8s in {-1,0,1,2}. - int unpacked_bytes[8]; -#pragma unroll - for (int j = 0; j < 4; ++j) { - const int shift = j * 8; - const int codes = (qs0 >> shift) & 0xFF; - const int c0 = ((codes >> 0) & 0x3) - 1; - const int c1 = ((codes >> 2) & 0x3) - 1; - const int c2 = ((codes >> 4) & 0x3) - 1; - const int c3 = ((codes >> 6) & 0x3) - 1; - unpacked_bytes[j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24); - } -#pragma unroll - for (int j = 0; j < 4; ++j) { - const int shift = j * 8; - const int codes = (qs1 >> shift) & 0xFF; - const int c0 = ((codes >> 0) & 0x3) - 1; - const int c1 = ((codes >> 2) & 0x3) - 1; - const int c2 = ((codes >> 4) & 0x3) - 1; - const int c3 = ((codes >> 6) & 0x3) - 1; - unpacked_bytes[4 + j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24); - } + const int16_t * qxi = (const int16_t *) bxi->qs + kqsx * 4; const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0; + #pragma unroll - for (int j = 0; j < 8; ++j) { + for (int j = 0; j < 4; ++j) { + const int q = qxi[j]; + + // unpack even and odd crumbs into byte values + const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); + const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2); + // unshuffle values + const int qx = __byte_perm(qe, qo, 0x5140); + const int qy = __byte_perm(qe, qo, 0x7362); + #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*sram_stride + dst_offset + j] = unpacked_bytes[j]; + x_qs[i*sram_stride + dst_offset + j*2+0] = qx; + x_qs[i*sram_stride + dst_offset + j*2+1] = qy; #else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j] = unpacked_bytes[j]; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+0] = qx; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+1] = qy; #endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) } } diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index 36d89c89e320..0f039c735b6b 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -734,48 +734,28 @@ static __device__ __forceinline__ float vec_dot_q2_0_q8_1( // Q8_1: 32 elements per block with individual scales // iqs selects which of the 2 chunks of 32 elements to process (0-1) - const float d2 = bq2_0->d; + const float d2 = bq2_0->d; + const int16_t * qs = (const int16_t *) bq2_0->qs + iqs * 4; // Process only the chunk specified by iqs const block_q8_1 * bq8_1_chunk = bq8_1 + iqs; - // Load 64 bits (8 bytes) for this chunk from Q2_0: bytes [8*iqs, 8*iqs+8) - const int offset = iqs * 8; - const int v0 = bq2_0->qs[offset + 0] | (bq2_0->qs[offset + 1] << 8) | - (bq2_0->qs[offset + 2] << 16) | (bq2_0->qs[offset + 3] << 24); - const int v1 = bq2_0->qs[offset + 4] | (bq2_0->qs[offset + 5] << 8) | - (bq2_0->qs[offset + 6] << 16) | (bq2_0->qs[offset + 7] << 24); - - // Unpack 32 2-bit codes into 8 int32s, each holding 4 signed int8 symbols in {-1,0,1,2}. - // Stored code c in {0,1,2,3} -> symbol s = c - 1. - int vi_bytes[8]; -#pragma unroll - for (int j = 0; j < 4; ++j) { - const int shift = j * 8; - const int codes = (v0 >> shift) & 0xFF; - const int c0 = ((codes >> 0) & 0x3) - 1; - const int c1 = ((codes >> 2) & 0x3) - 1; - const int c2 = ((codes >> 4) & 0x3) - 1; - const int c3 = ((codes >> 6) & 0x3) - 1; - vi_bytes[j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24); - } + int sumi = 0; #pragma unroll for (int j = 0; j < 4; ++j) { - const int shift = j * 8; - const int codes = (v1 >> shift) & 0xFF; - const int c0 = ((codes >> 0) & 0x3) - 1; - const int c1 = ((codes >> 2) & 0x3) - 1; - const int c2 = ((codes >> 4) & 0x3) - 1; - const int c3 = ((codes >> 6) & 0x3) - 1; - vi_bytes[4 + j] = (c0 & 0xFF) | ((c1 & 0xFF) << 8) | ((c2 & 0xFF) << 16) | ((c3 & 0xFF) << 24); - } + const int q = qs[j]; + const int u = get_int_b4(bq8_1_chunk->qs, j*2+0); + const int v = get_int_b4(bq8_1_chunk->qs, j*2+1); - // Compute dot product for this 32-element chunk - int sumi = 0; -#pragma unroll - for (int j = 0; j < 8; ++j) { - const int u = get_int_b4(bq8_1_chunk->qs, j); - sumi = ggml_cuda_dp4a(vi_bytes[j], u, sumi); + // unpack even and odd crumbs into byte values + const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); + const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2); + // unshuffle values + const int qx = __byte_perm(qe, qo, 0x5140); + const int qy = __byte_perm(qe, qo, 0x7362); + + sumi = ggml_cuda_dp4a(u, qx, sumi); + sumi = ggml_cuda_dp4a(v, qy, sumi); } // Apply Q2_0's single scale and this chunk's Q8_1 scale From 000547513f1530346ecd163db8b3e13962949961 Mon Sep 17 00:00:00 2001 From: Ruixiang Wang Date: Fri, 31 Jul 2026 10:16:17 +0200 Subject: [PATCH 06/98] server: correct accepted tokens when need draft token replay (#26320) * spec: correct accepted tokens when need draft token replay * cont : naming --------- Co-authored-by: Georgi Gerganov --- tools/server/server-context.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 0f7ada435a29..4655b518e21f 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -212,6 +212,7 @@ struct server_slot { llama_tokens spec_prompt; std::vector spec_i_batch; common_prompt_checkpoint spec_ckpt; + bool spec_is_replay = false; // TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state // see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837 @@ -332,6 +333,8 @@ struct server_slot { void reset() { SLT_DBG(*this, "%s", "\n"); + spec_is_replay = false; + n_prompt_tokens_cache = 0; last_nl_pos = 0; @@ -3875,6 +3878,7 @@ struct server_context_impl { } // partial acceptance is not supported by the context -> truncate the draft and restore the state + slot.spec_is_replay = true; slot.spec_draft = std::move(accepted); const auto & ckpt = slot.spec_ckpt; @@ -3909,16 +3913,22 @@ struct server_context_impl { const auto ids = std::move(slot.spec_draft); + size_t n_accepted = ids.size() - 1; + if (slot.spec_is_replay && n_accepted > 0) { + n_accepted--; + } + slot.spec_is_replay = false; + slot.t_token_generation = std::max(1, t_now - slot.t_start_generation) / 1e3; // update how many tokens out of those tested were accepted - slot.n_draft_accepted += ids.size() - 1; + slot.n_draft_accepted += n_accepted; slot.n_draft_verif_steps += 1; if (slot.n_accepted_per_pos.empty()) { slot.n_accepted_per_pos.resize(common_speculative_n_max(¶ms_base.speculative), 0); } - for (size_t i = 0; i < ids.size() - 1 && i < slot.n_accepted_per_pos.size(); ++i) { + for (size_t i = 0; i < n_accepted && i < slot.n_accepted_per_pos.size(); ++i) { slot.n_accepted_per_pos[i]++; } @@ -3954,7 +3964,7 @@ struct server_context_impl { slot.print_timings_tg(); - SLT_DBG(slot, "accepted %d/%d draft tokens, new n_tokens = %d\n", (int) ids.size() - 1, (int) n_draft, slot.prompt.n_tokens()); + SLT_DBG(slot, "accepted %d/%d draft tokens, new n_tokens = %d\n", (int) n_accepted, (int) n_draft, slot.prompt.n_tokens()); }); } From 6f3c0a790be2c8599663a98f03b95fb3eae767f8 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Fri, 31 Jul 2026 13:27:03 +0100 Subject: [PATCH 07/98] vulkan: update vulkan sdk to 1.4.357.0 (#26303) --- .github/workflows/build-cpu.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml index 8f62e1a177cd..30b07ce7882c 100644 --- a/.github/workflows/build-cpu.yml +++ b/.github/workflows/build-cpu.yml @@ -121,7 +121,7 @@ jobs: env: OPENBLAS_VERSION: 0.3.23 SDE_VERSION: 9.33.0-2024-01-07 - VULKAN_VERSION: 1.4.313.2 + VULKAN_VERSION: 1.4.357.0 strategy: matrix: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 778574cf9ff8..3460485af388 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -759,7 +759,7 @@ jobs: env: OPENBLAS_VERSION: 0.3.23 - VULKAN_VERSION: 1.4.313.2 + VULKAN_VERSION: 1.4.357.0 strategy: matrix: From 82dbc4f017a7b005f993ac2e7af9c048ad686c04 Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:57:02 +0200 Subject: [PATCH 08/98] llama : load MTP tensors only if they are really used (#26296) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * llama : load MTP tensors only if they are really used * llama : skip loading MTP (if not used) in remaining models that support MTP --------- Co-authored-by: Stanisław Szymczyk --- common/common.cpp | 1 + include/llama.h | 1 + src/llama-model-loader.cpp | 2 ++ src/llama-model-loader.h | 2 ++ src/llama-model.cpp | 1 + src/llama-quant.cpp | 2 +- src/llama.cpp | 2 +- src/models/cohere2moe.cpp | 6 +++++- src/models/glm-dsa.cpp | 6 +++++- src/models/hy-v3.cpp | 6 +++++- src/models/qwen35.cpp | 31 +++++++++++++++--------------- src/models/qwen35moe.cpp | 39 +++++++++++++++++++------------------- src/models/step35.cpp | 6 +++++- 13 files changed, 65 insertions(+), 40 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 1a0aeef4ec6f..ff27d392fb2e 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1620,6 +1620,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) { mparams.progress_callback = params.load_progress_callback; mparams.progress_callback_user_data = params.load_progress_callback_user_data; mparams.no_alloc = params.no_alloc; + mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); return mparams; } diff --git a/include/llama.h b/include/llama.h index a3ceecf5ef47..6e53e2297235 100644 --- a/include/llama.h +++ b/include/llama.h @@ -337,6 +337,7 @@ extern "C" { bool use_extra_bufts; // use extra buffer types (used for weight repacking) bool no_host; // bypass host buffer allowing extra buffers to be used bool no_alloc; // only load metadata and simulate memory allocations + bool load_mtp; // whether to load MTP layers }; struct llama_sampler_seq_config { diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 510586e96c20..df8313e81976 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -526,6 +526,7 @@ llama_model_loader::llama_model_loader( llama_load_mode load_mode, bool check_tensors, bool no_alloc, + bool load_mtp, const llama_model_kv_override * param_overrides_p, const llama_model_tensor_buft_override * param_tensor_buft_overrides_p) : metadata(meta), set_tensor_data(set_tensor_data), set_tensor_data_ud(set_tensor_data_ud) { @@ -812,6 +813,7 @@ llama_model_loader::llama_model_loader( this->check_tensors = check_tensors; this->no_alloc = no_alloc; + this->load_mtp = load_mtp; } std::string llama_model_loader::get_arch_name() const { diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index 75a3652d06bd..7ad380782291 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -79,6 +79,7 @@ struct llama_model_loader { bool use_direct_io = false; bool check_tensors; bool no_alloc; + bool load_mtp; llama_files files; llama_ftype ftype; @@ -129,6 +130,7 @@ struct llama_model_loader { llama_load_mode load_mode, bool check_tensors, bool no_alloc, + bool load_mtp, const llama_model_kv_override * param_overrides_p, const llama_model_tensor_buft_override * param_tensor_buft_overrides_p); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 5e501f983bc0..91c2fc9c008d 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2390,6 +2390,7 @@ llama_model_params llama_model_default_params() { /*.use_extra_bufts =*/ true, /*.no_host =*/ false, /*.no_alloc =*/ false, + /*.load_mtp =*/ false, }; return result; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 92ebc11b99f3..fd6e787bd7d2 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -893,7 +893,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const llama_model_kv_override * kv_overrides = params->kv_overrides; std::vector splits = {}; llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr, - fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, kv_overrides, nullptr); + fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, /*load_mtp*/ true, kv_overrides, nullptr); ml.init_mappings(false); // no prefetching auto mparams = llama_model_default_params(); diff --git a/src/llama.cpp b/src/llama.cpp index d22e4c81a9e3..d6e0bbfefa72 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -305,7 +305,7 @@ static std::pair llama_model_load(struct gguf_context * meta const std::string & fname, std::vector & splits, FILE * file, llama_model_params & params) { try { llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode, - params.check_tensors, params.no_alloc, params.kv_overrides, params.tensor_buft_overrides); + params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides); ml.print_info(); std::unique_ptr model_ptr(llama_model_create(ml, params)); diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp index 499c73a1c49c..3acb7e77af80 100644 --- a/src/models/cohere2moe.cpp +++ b/src/models/cohere2moe.cpp @@ -55,7 +55,11 @@ void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) { const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; - const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp index bd1c4df21281..360c2ee773f0 100644 --- a/src/models/glm-dsa.cpp +++ b/src/models/glm-dsa.cpp @@ -91,7 +91,11 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader & ml) { const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; - const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } const bool is_mla = hparams.is_mla(); if (!is_mla) { diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cpp index 47a0beaf217f..61db93af85ce 100644 --- a/src/models/hy-v3.cpp +++ b/src/models/hy-v3.cpp @@ -33,7 +33,11 @@ void llama_model_hy_v3::load_arch_tensors(llama_model_loader & ml) { const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; - const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index d8ffe43ae76c..309dd432447c 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -39,6 +39,7 @@ void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) { const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0; tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); @@ -97,25 +98,25 @@ void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) { auto & layer = layers[il]; // MTP block looks like a full-attention Qwen3.5 decoder block. - layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, 0); - layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, 0); + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, mtp_flags); + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, mtp_flags); - create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0); - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, 0); - layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, 0); + create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, mtp_flags); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, mtp_flags); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, mtp_flags); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, mtp_flags); - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", il), {n_embd, n_ff}, 0); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", il), { n_ff, n_embd}, 0); - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", il), {n_embd, n_ff}, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", il), {n_embd, n_ff}, mtp_flags); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", il), { n_ff, n_embd}, mtp_flags); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", il), {n_embd, n_ff}, mtp_flags); // NextN-specific tensors that define the MTP block. - layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, 0); - layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, 0); - layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, 0); - layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); - layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); - layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, TENSOR_NOT_REQUIRED); + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, mtp_flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, mtp_flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, mtp_flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, mtp_flags|TENSOR_NOT_REQUIRED); }; for (int i = 0; i < n_layer; ++i) { diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index 7b0876cbb04b..38f2a57985a9 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -42,6 +42,7 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) { const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0; tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); @@ -113,32 +114,32 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) { const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff; // MTP block looks like a full-attention Qwen3.5 decoder block with MoE FFN. - layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, 0); - layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, 0); + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, mtp_flags); + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, mtp_flags); - create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0); - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, 0); - layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, 0); + create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, mtp_flags); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, mtp_flags); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, mtp_flags); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, mtp_flags); // Routed experts - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, 0); - create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, 0); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, mtp_flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, mtp_flags); + create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, mtp_flags); // Shared experts - layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, 0); - layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, 0); - layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, 0); - layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, 0); + layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, mtp_flags); + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, mtp_flags); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, mtp_flags); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, mtp_flags); // NextN-specific tensors that define the MTP block. - layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, 0); - layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, 0); - layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, 0); - layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); - layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); - layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, TENSOR_NOT_REQUIRED); + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, mtp_flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, mtp_flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, mtp_flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, mtp_flags|TENSOR_NOT_REQUIRED); }; for (int i = 0; i < n_layer; ++i) { diff --git a/src/models/step35.cpp b/src/models/step35.cpp index 9b7b18a3678b..5b1d902581e6 100644 --- a/src/models/step35.cpp +++ b/src/models/step35.cpp @@ -48,7 +48,11 @@ void llama_model_step35::load_arch_tensors(llama_model_loader & ml) { const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; - const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); From a09d8abf8c926be74c040674e86c69955deac09f Mon Sep 17 00:00:00 2001 From: timkhronos Date: Fri, 31 Jul 2026 15:06:40 +0200 Subject: [PATCH 09/98] Support rotated kv cache quant (#26180) --- src/models/minimax-m3.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 3e7bada64bc9..0773ad5435c9 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -271,8 +271,6 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ } else { const int64_t n_idx_dim = hparams.indexer_head_size; // 128 - GGML_ASSERT(!inp_attn->self_k_rot && !inp_attn->self_v_rot && "MSA: attn-rot not supported"); - // Index Branch, project, norm, partial RoPE, cache ggml_tensor * iq = build_lora_mm(model.layers[il].index_q_proj, cur); ggml_tensor * ik = build_lora_mm(model.layers[il].index_k_proj, cur); @@ -289,6 +287,14 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ggml_build_forward_expand(gf, mctx_cur->cpy_k_idx(ctx0, ik, inp_attn->get_k_idxs(), il)); ggml_tensor * ik_kv = mctx_cur->get_k_idx(ctx0, il); + if (inp_attn->self_k_rot) { + Qcur = llama_mul_mat_hadamard(ctx0, Qcur, inp_attn->self_k_rot); + Kcur = llama_mul_mat_hadamard(ctx0, Kcur, inp_attn->self_k_rot); + } + if (inp_attn->self_v_rot) { + Vcur = llama_mul_mat_hadamard(ctx0, Vcur, inp_attn->self_v_rot); + } + // Main branch: store K/V, take cache views ggml_build_forward_expand(gf, Qcur); ggml_build_forward_expand(gf, Kcur); @@ -431,7 +437,9 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ cur = ggml_concat(ctx0, cur, outs[st], 1); } } - + if (inp_attn->self_v_rot) { + cur = llama_mul_mat_hadamard(ctx0, cur, inp_attn->self_v_rot); + } cb(cur, "kqv_out", il); if (model.layers[il].wo) { cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s); From db7d8b24b5197ca39435cf47b3c1ba039b53605b Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Fri, 31 Jul 2026 15:30:19 +0200 Subject: [PATCH 10/98] mtmd: add n_embd_head (#26342) Co-authored-by: Daniel Han --- gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 +++ tools/mtmd/clip-impl.h | 1 + tools/mtmd/clip-model.h | 2 ++ tools/mtmd/clip.cpp | 7 ++++--- 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 650f1c8a56a6..d86e614d8fd4 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -353,6 +353,7 @@ class ClipVision: class Attention: HEAD_COUNT = "clip.vision.attention.head_count" HEAD_COUNT_KV = "clip.vision.attention.head_count_kv" # used by mimovl (GQA) + HEAD_DIM = "clip.vision.attention.head_dim" # set when qkv width != n_embd LAYERNORM_EPS = "clip.vision.attention.layer_norm_epsilon" class Projector: diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 3aa4f049f2bb..c5905164c356 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1226,6 +1226,9 @@ def add_vision_head_count(self, value: int) -> None: def add_vision_head_count_kv(self, value: int) -> None: self.add_uint32(Keys.ClipVision.Attention.HEAD_COUNT_KV, value) + def add_vision_head_dim(self, value: int) -> None: + self.add_uint32(Keys.ClipVision.Attention.HEAD_DIM, value) + def add_vision_attention_layernorm_eps(self, value: float) -> None: self.add_float32(Keys.ClipVision.Attention.LAYERNORM_EPS, value) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 589fc724ed08..d42b38222c27 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -41,6 +41,7 @@ #define KEY_PROJ_DIM "clip.%s.projection_dim" #define KEY_N_HEAD "clip.%s.attention.head_count" #define KEY_N_HEAD_KV "clip.%s.attention.head_count_kv" +#define KEY_N_EMBD_HEAD "clip.%s.attention.head_dim" #define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon" #define KEY_FEATURE_LAYERS "clip.%s.feature_layer" diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index fec2b01802f8..8b9db5101d2c 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -54,6 +54,8 @@ struct clip_hparams { int32_t projection_dim = 0; int32_t n_head = 0; int32_t n_head_kv = 0; + // 0 = derive from n_embd; set when qkv width != n_embd + int32_t n_embd_head = 0; int32_t n_layer = 0; int32_t n_merge = 1; // number of patch merges **per-side** diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 11f9820edeb8..5f0d00b66023 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -253,7 +253,7 @@ clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) : n_embd(hparams.n_embd), n_head(hparams.n_head), n_head_kv(hparams.n_head_kv), - d_head(n_head > 0 ? n_embd / n_head : 0), + d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)), n_layer(hparams.n_layer), n_mmproj_embd(clip_n_mmproj_embd(ctx)), eps(hparams.eps), @@ -372,13 +372,13 @@ ggml_tensor * clip_graph::build_vit( /* nb1 */ ggml_row_size(cur->type, d_head), /* nb2 */ cur->nb[1], /* nb3 */ cur->nb[1] * n_pos, - /* offset */ ggml_row_size(cur->type, n_embd)); + /* offset */ ggml_row_size(cur->type, n_head * d_head)); Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, /* nb1 */ ggml_row_size(cur->type, d_head), /* nb2 */ cur->nb[1], /* nb3 */ cur->nb[1] * n_pos, - /* offset */ ggml_row_size(cur->type, 2 * n_embd)); + /* offset */ ggml_row_size(cur->type, 2 * n_head * d_head)); if (layer.q_norm) { GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]); @@ -1190,6 +1190,7 @@ struct clip_model_loader { const char * prefix = is_vision ? "vision" : "audio"; get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd); get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head); + get_u32(string_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false); get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff); get_u32(string_format(KEY_N_BLOCK, prefix), hparams.n_layer); get_u32(string_format(KEY_PROJ_DIM, prefix), hparams.projection_dim); From eb41d503ba9496b816ff4180a34780bc13a4f4b8 Mon Sep 17 00:00:00 2001 From: Masato Nakasaka Date: Fri, 31 Jul 2026 23:26:37 +0900 Subject: [PATCH 11/98] vulkan: Introduce driver version check for Windows Intel GPU to mitigate crashing (#25192) * Removed crash guard for Intel Crash fixed from driver 32.0.101.8860 * Added driver version check for windows * Change to convert from driverVersion rather than string * No need to use signed * Refactor * allow GPU other than Xe2+ * adjusted function body position --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index e32dd7a56a2f..9c01dabe6908 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1934,6 +1934,7 @@ struct ggml_vk_garbage_collector { static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx); static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr); static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx); +static bool ggml_vk_intel_windows_driver_equals_or_newer_than(uint32_t driver_version, uint32_t threshold_major, uint32_t threshold_minor); static bool vk_memory_logger_enabled = false; @@ -5552,8 +5553,11 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); ggml_vk_create_pipeline(device, device->pipeline_sum_rows_f32, "sum_rows_f32", sum_rows_f32_len, sum_rows_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); - // Intel Arc B390 was observed segfaulting with this shader. - if (device->subgroup_basic && device->subgroup_shuffle && device->vendor_id != VK_VENDOR_ID_INTEL) { + // Intel Windows driver older than 32.0.101.8860 will crash when using fwht kernels on Xe2+ GPUS so we gate that here + const bool can_use_fwht = device->driver_id != vk::DriverId::eIntelProprietaryWindows || + device->architecture != vk_device_architecture::INTEL_XE2 || + (device->architecture == vk_device_architecture::INTEL_XE2 && ggml_vk_intel_windows_driver_equals_or_newer_than(device->properties.driverVersion, 101, 8860)); + if (can_use_fwht && device->subgroup_basic && device->subgroup_shuffle) { int idx = 0; for (uint32_t n : {64, 128, 256, 512}) { if (device->subgroup_size <= n) { @@ -5561,8 +5565,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } ++idx; } - } else if (device->driver_id != vk::DriverId::eIntelProprietaryWindows) { - // Disabled on Intel Windows due to a driver bug: https://github.com/ggml-org/llama.cpp/pull/23964#issuecomment-4598226147 + } else if (can_use_fwht) { int idx = 0; for (uint32_t n : {64, 128, 256, 512}) { const uint32_t block_size = std::min(device->subgroup_size, n); @@ -18466,6 +18469,22 @@ static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) } } +static bool ggml_vk_intel_windows_driver_equals_or_newer_than(uint32_t driver_version, uint32_t threshold_major, uint32_t threshold_minor) { +#if defined(_WIN32) + // Intel Windows encodes xxx.yyyy as [31:14].[13:0]. + const uint32_t major = driver_version >> 14; + const uint32_t minor = driver_version & 0x3fff; + + return major > threshold_major || (major == threshold_major && minor >= threshold_minor); +#else + GGML_UNUSED(driver_version); + GGML_UNUSED(threshold_major); + GGML_UNUSED(threshold_minor); + return true; +#endif +} + + // checks #ifdef GGML_VULKAN_CHECK_RESULTS From 876a4321163249c43ca4e986818fab5ab081f282 Mon Sep 17 00:00:00 2001 From: Anand Patil <126432639+AnandPatil1@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:48:58 -0500 Subject: [PATCH 12/98] vulkan: add POOL_1D op (#25431) * vulkan : add pool1d push constants and pipeline field Declared data structures needed for POOL1D OP, which are the vk_op_pool1d_push_constants struct and pipeline_pool1d_f32 field. * vulkan : add pool1d compute shader Added pool1d.comp for Vulkan backend mirroring the existing pool2d shader. * vulkan : add full GGML_OP_POOL_1D support Added pipeline creation and op dispatch for 1D pooling in the Vulkan backend. * vulkan : fix pool1d shader logic Registered pool1d_f32 in vulkan-shaders-gen.cpp and fixed tensor dimension indices and avg pool scale. * vulkan : fix pool1d end boundary crash and expand test coverage Fixed an issue where the shader crashed when the end boundary was negative when k0 < p0. Also, added more test cases related to this fix. --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 61 +++++++++++++++++ .../ggml-vulkan/vulkan-shaders/pool1d.comp | 65 +++++++++++++++++++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 1 + tests/test-backend-ops.cpp | 6 +- 4 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 9c01dabe6908..ff72fa60a8a9 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -998,6 +998,7 @@ struct vk_device_struct { vk_pipeline pipeline_snake_f32; vk_pipeline pipeline_snake_f16; vk_pipeline pipeline_snake_bf16; + vk_pipeline pipeline_pool1d_f32; vk_pipeline pipeline_pool2d_f32; vk_pipeline pipeline_rwkv_wkv6_f32; vk_pipeline pipeline_rwkv_wkv7_f32; @@ -1687,6 +1688,17 @@ struct vk_op_snake_push_constants { uint32_t ne1; }; +struct vk_op_pool1d_push_constants { + uint32_t IL; + uint32_t OL; + uint32_t OC; + uint32_t pelements; + uint32_t op; + int32_t k0; + int32_t s0; + int32_t p0; +}; + struct vk_op_pool2d_push_constants { uint32_t IW; uint32_t IH; uint32_t OW; uint32_t OH; @@ -5622,6 +5634,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_snake_f16, "snake_f16", snake_f16_len, snake_f16_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_snake_bf16, "snake_bf16", snake_bf16_len, snake_bf16_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_pool1d_f32, "pool1d_f32", pool1d_f32_len, pool1d_f32_data, "main", 2, sizeof(vk_op_pool1d_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_pool2d_f32, "pool2d_f32", pool2d_f32_len, pool2d_f32_data, "main", 2, sizeof(vk_op_pool2d_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv6_f32, "rwkv_wkv6_f32", rwkv_wkv6_f32_len, rwkv_wkv6_f32_data, "main", 7, sizeof(vk_op_rwkv_wkv6_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); @@ -11335,6 +11348,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const case GGML_TYPE_BF16: return ctx->device->pipeline_col2im_1d_bf16; default: return nullptr; } + case GGML_OP_POOL_1D: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_pool1d_f32; + } + return nullptr; case GGML_OP_POOL_2D: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_pool2d_f32; @@ -11857,6 +11875,13 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co { elements = { uint32_t(dst->ne[0]), uint32_t(dst->ne[1]), 1 }; } break; + case GGML_OP_POOL_1D: + { + const uint32_t N = dst->ne[3] * dst->ne[2]; + const uint32_t OC = dst->ne[1]; + const uint32_t OL = dst->ne[0]; + elements = { N * OC * OL, 1, 1}; + } break; case GGML_OP_POOL_2D: { const uint32_t N = dst->ne[3]; @@ -13776,6 +13801,29 @@ static void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_conte ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, a_buf, inv_b_buf, dst_buf }, pc, elements); } +static void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + uint32_t op = static_cast(dst->op_params[0]); + const int32_t k0 = dst->op_params[1]; + const int32_t s0 = dst->op_params[2]; + const int32_t p0 = dst->op_params[3]; + + const uint32_t IL = src0->ne[0]; + + const uint32_t N = dst->ne[3] * dst->ne[2]; + + const uint32_t OC = dst->ne[1]; + const uint32_t OL = dst->ne[0]; + + const uint32_t parallel_elements = N * OC * OL; + + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_POOL_1D, { + IL, OL, OC, + parallel_elements, + op, + k0, s0, p0, + }); +} + static void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { uint32_t op = static_cast(dst->op_params[0]); const int32_t k1 = dst->op_params[1]; @@ -15287,6 +15335,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr case GGML_OP_CONV_TRANSPOSE_1D: ggml_vk_conv_transpose_1d(ctx, compute_ctx, src0, src1, node); + break; + case GGML_OP_POOL_1D: + ggml_vk_pool_1d(ctx, compute_ctx, src0, node); + break; case GGML_OP_POOL_2D: ggml_vk_pool_2d(ctx, compute_ctx, src0, node); @@ -18004,6 +18056,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_CONV_2D_DW: return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && op->src[1]->type == GGML_TYPE_F32; + case GGML_OP_POOL_1D: + return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_POOL_2D: return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_RWKV_WKV6: @@ -18939,6 +18993,13 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * const int32_t oc = tensor->op_params[1]; const int32_t p0 = tensor->op_params[2]; tensor_clone = ggml_col2im_1d(ggml_ctx, src_clone[0], stride, oc, p0); + } else if (tensor->op == GGML_OP_POOL_1D) { + enum ggml_op_pool op = static_cast(tensor->op_params[0]); + const int32_t k0 = tensor->op_params[1]; + const int32_t s0 = tensor->op_params[2]; + const int32_t p0 = tensor->op_params[3]; + + tensor_clone = ggml_pool_1d(ggml_ctx, src_clone[0], op, k0, s0, p0); } else if (tensor->op == GGML_OP_POOL_2D) { enum ggml_op_pool op = static_cast(tensor->op_params[0]); const int32_t k0 = tensor->op_params[1]; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp b/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp new file mode 100644 index 000000000000..bb87631ce363 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp @@ -0,0 +1,65 @@ +#version 450 + +#include "types.glsl" + +#extension GL_EXT_shader_16bit_storage : require + +layout(push_constant) uniform parameter { + uint IL; + uint OL; + uint OC; + uint pelements; + uint op; + int k0; + int s0; + int p0; +} p; + +#define BLOCK_SIZE 512 +#define FLT_MAX 3.402823466e+38F +#define OP_POOL_MAX 0u +#define OP_POOL_AVG 1u + +layout (local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; + +layout(binding = 0) readonly buffer X {A_TYPE data_a[];}; +layout(binding = 1) writeonly buffer D {D_TYPE data_d[];}; + +void main() { + const uint idx = gl_GlobalInvocationID.x; + if (idx >= p.pelements) { + return; + } + + const uint nc = idx / p.OL; + const uint cur_ol = idx % p.OL; + + const int start = int(cur_ol) * p.s0 - p.p0; + const int bl = max(start, 0); + const int el = min(max(start + p.k0, 0), int(p.IL)); + + const int window_size = el - bl; + const float scale = window_size > 0 ? 1.0 / float(window_size) : 0.0; + float res; + + if (p.op == OP_POOL_AVG) { + res = 0.0; + } else if (p.op == OP_POOL_MAX) { + res = -FLT_MAX; + } else { + return; + } + + #pragma unroll + for (uint i = bl; i < el; i++) { + const float cur = D_TYPE(data_a[nc * p.IL + i]); + + if (p.op == OP_POOL_AVG) { + res += cur * scale; + } else if (p.op == OP_POOL_MAX) { + res = max(res, cur); + } + } + + data_d[nc * p.OL + cur_ol] = res; +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 3e6b39576e6d..0223d2e01e91 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -1052,6 +1052,7 @@ void process_shaders() { string_to_spv("snake_f16", "snake.comp", {{"DATA_A_F16", "1"}, {"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); string_to_spv("snake_bf16", "snake.comp", {{"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}, {"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}}); + string_to_spv("pool1d_f32", "pool1d.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("pool2d_f32", "pool2d.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rwkv_wkv6_f32", "wkv6.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index e101ca390526..4098acaaf91a 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8192,9 +8192,9 @@ static std::vector> make_test_cases_eval() { for (ggml_type type_input : {GGML_TYPE_F32}) { for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { - for (int k0 : {1, 3}) { - for (int s0 : {1, 2}) { - for (int p0 : {0, 1}) { + for (int k0 : {1, 2, 3}) { + for (int s0 : {1, 2, 3}) { + for (int p0 : {0, 1, 2, 3}) { test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 10, 3, 2, 1 }, k0, s0, p0)); test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 11, 1, 3, 2 }, k0, s0, p0)); test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 128, 2, 1, 3 }, k0, s0, p0)); From ddd4ec1428a6201e18975ea52b07c71e0f9aef26 Mon Sep 17 00:00:00 2001 From: "Piotr Wilkin (ilintar)" Date: Sat, 1 Aug 2026 07:13:07 +0200 Subject: [PATCH 13/98] chat : enable tool call in thinking for DS4 (#26269) --- common/chat.cpp | 202 +++++++++++++------------ tests/test-chat.cpp | 14 ++ tools/parser/debug-template-parser.cpp | 5 +- 3 files changed, 122 insertions(+), 99 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 7740f35c0edc..0e06fa591f9d 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1943,18 +1943,6 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages); } - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.thinking_start_tag = ""; - data.thinking_end_tags = {""}; - data.preserved_tokens = { - "｜DSML｜", - "", - "", - }; - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; @@ -1972,6 +1960,18 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha const std::string PARAM_END = ""; const std::string GEN_PROMPT = "<｜Assistant｜>"; + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = {THINK_END, FC_START}; + data.preserved_tokens = { + DSML, + THINK_START, + THINK_END, + }; + if (inputs.has_continuation()) { const auto & msg = inputs.continue_msg; @@ -1983,102 +1983,77 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha data.prompt += data.generation_prompt; } + bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.literal(GEN_PROMPT); - auto end = p.end(); - - auto reasoning = p.eps(); - if (extract_reasoning && inputs.enable_thinking) { - reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); - } else if (extract_reasoning) { - // Thinking disabled but reasoning extraction requested: the generation prompt - // contains an empty pair (V3.2) or a bare (V4) that - // must still be consumed. - reasoning = is_v4 - ? p.optional(p.literal(THINK_END)) - : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END)); - } - - if (has_response_format) { - auto response_format = p.rule("response-format", - p.literal("```json") + p.space() + - p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + - p.space() + p.literal("```")); - return generation_prompt + reasoning + response_format + end; - } - - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { - return generation_prompt + reasoning + p.content(p.rest()) + end; - } + auto end = p.end(); + // build tool call section first since we might need it in reasoning auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - const auto & props = params.contains("properties") ? params.at("properties") : json::object(); - - std::set required; - if (params.contains("required")) { - params.at("required").get_to(required); - } - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); + if (has_tool_calls) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + const auto & props = params.contains("properties") ? params.at("properties") : json::object(); - std::vector required_parsers; - std::vector optional_parsers; - for (const auto & [param_name, param_schema] : props.items()) { - bool is_required = required.find(param_name) != required.end(); - bool is_string = schema_info.resolves_to_string(param_schema); - - auto arg = p.tool_arg( - p.tool_arg_open( - p.literal(PARAM_START + " name=\"") + - p.tool_arg_name(p.literal(param_name)) + - p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + - (is_string - ? p.tool_arg_string_value(p.until(PARAM_END)) - : p.tool_arg_json_value(p.schema(p.json(), - "tool-" + name + "-arg-" + param_name + "-schema", - param_schema, false))) + - p.tool_arg_close(p.literal(PARAM_END))); - - auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); - if (is_required) { - required_parsers.push_back(named_arg); - } else { - optional_parsers.push_back(named_arg); + std::set required; + if (params.contains("required")) { + params.at("required").get_to(required); } - } - common_peg_parser args_seq = p.eps(); - for (size_t i = 0; i < required_parsers.size(); i++) { - if (i > 0) { - args_seq = args_seq + p.space(); + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + std::vector required_parsers; + std::vector optional_parsers; + for (const auto & [param_name, param_schema] : props.items()) { + bool is_required = required.find(param_name) != required.end(); + bool is_string = schema_info.resolves_to_string(param_schema); + + auto arg = p.tool_arg( + p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) + + p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + + (is_string ? + p.tool_arg_string_value(p.until(PARAM_END)) : + p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema", + param_schema, false))) + + p.tool_arg_close(p.literal(PARAM_END))); + + auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); + if (is_required) { + required_parsers.push_back(named_arg); + } else { + optional_parsers.push_back(named_arg); + } } - args_seq = args_seq + required_parsers[i]; - } - if (!optional_parsers.empty()) { - common_peg_parser any_opt = p.choice(); - for (const auto & opt : optional_parsers) { - any_opt |= opt; + common_peg_parser args_seq = p.eps(); + for (size_t i = 0; i < required_parsers.size(); i++) { + if (i > 0) { + args_seq = args_seq + p.space(); + } + args_seq = args_seq + required_parsers[i]; } - args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1); - } - common_peg_parser invoke_body = args_seq; - auto func_parser = p.tool( - p.tool_open(p.literal(INVOKE_START + " name=\"") + - p.tool_name(p.literal(name)) + p.literal("\">\n")) + - invoke_body + p.space() + - p.tool_close(p.literal(INVOKE_END))); + if (!optional_parsers.empty()) { + common_peg_parser any_opt = p.choice(); + for (const auto & opt : optional_parsers) { + any_opt |= opt; + } + args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1); + } - tool_choice |= p.rule("tool-" + name, func_parser); - }); + common_peg_parser invoke_body = args_seq; + auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") + + p.tool_name(p.literal(name)) + p.literal("\">\n")) + + invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END))); - auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + tool_choice |= p.rule("tool-" + name, func_parser); + }); + } common_peg_parser tool_calls = p.eps(); if (inputs.parallel_tool_calls) { @@ -2090,18 +2065,49 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); } + auto reasoning = p.eps(); + auto reasoning_with_tc = p.eps(); + auto obligatory_tool_calls = tool_calls; + bool allow_reasoning_with_tc = false; + if (!require_tools) { tool_calls = p.optional(tool_calls); } - auto content_before_tools = p.content(p.until(FC_START)); - return generation_prompt + reasoning + content_before_tools + tool_calls + end; + if (extract_reasoning && inputs.enable_thinking) { + reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); + reasoning_with_tc = THINK_START + p.reasoning(p.until_one_of({ FC_START, THINK_END })) + obligatory_tool_calls; + allow_reasoning_with_tc = true; + } else if (extract_reasoning) { + // Thinking disabled but reasoning extraction requested: the generation prompt + // contains an empty pair (V3.2) or a bare (V4) that + // must still be consumed. + reasoning = is_v4 + ? p.optional(p.literal(THINK_END)) + : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END)); + } + + if (has_response_format) { + auto response_format = p.rule("response-format", + p.literal("```json") + p.space() + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + + p.space() + p.literal("```")); + return generation_prompt + reasoning + response_format + end; + } + + if (!has_tool_calls) { + return generation_prompt + reasoning + p.content(p.rest()) + end; + } + + auto content_before_tools = p.negate(p.literal(THINK_START)) + p.content(p.until(FC_START)); + return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end : + generation_prompt + reasoning + content_before_tools + tool_calls + end; }); data.parser = parser.save(); if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar_lazy = has_tools && !require_tools; data.grammar = build_grammar([&](const common_grammar_builder & builder) { foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 01b07953a627..ef02fdde57ef 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -4227,6 +4227,20 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_reasoning("I'm thinking") .expect_content("Hello, world!\nWhat's up?") .run(); + + tst.test( + "Let me check the time\n\n" + "<｜DSML｜tool_calls>\n" + "<｜DSML｜invoke name=\"get_time\">\n" + "<｜DSML｜parameter name=\"city\" string=\"true\">Tokyo\n" + "\n" + "") // no after the TC close because the grammar will immediately constrain it to end + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ get_time_tool }) + .expect_reasoning("Let me check the time") + .expect_tool_calls({ { "get_time", R"({"city": "Tokyo"})", {} } }) + .run(); } // GLM-4.6 tests - format: function_name\n...\n...\n diff --git a/tools/parser/debug-template-parser.cpp b/tools/parser/debug-template-parser.cpp index 50e8f1efb7d6..8a916f79c78e 100644 --- a/tools/parser/debug-template-parser.cpp +++ b/tools/parser/debug-template-parser.cpp @@ -9,6 +9,7 @@ #include "peg-parser.h" #include +#include #include #include #include @@ -398,7 +399,7 @@ int main(int argc, char ** argv) { if (std::optional spec_tmpl = common_chat_try_specialized_template(chat_template, template_source, params)) { LOG_ERR("\n"); - LOG_ERR("This template uses a specialized parser, analysis results will not be available."); + LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n"); parser_data = *spec_tmpl; } else { // Render template scenarios if requested @@ -426,7 +427,9 @@ int main(int argc, char ** argv) { // Generate Parser parser_data = autoparser::peg_generator::generate_parser(chat_template, params, analysis); } + } + if (!std::empty(parser_data.parser)) { LOG_ERR("\n=== Generated Parser ===\n"); common_peg_arena arena; arena.load(parser_data.parser); From de699957b92f490efebad149665b0dccf127eaff Mon Sep 17 00:00:00 2001 From: tc-mb <157115220+tc-mb@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:38:36 +0800 Subject: [PATCH 14/98] mtmd: add minicpmv46 downsample (#25993) * add minicpmv46 downsample Signed-off-by: tc-mb * put downsample mode inside gguf. Signed-off-by: tc-mb * build mtmd_image_preprocessor_llava_uhd Signed-off-by: tc-mb * fix code Signed-off-by: tc-mb * add convert Signed-off-by: tc-mb * add 4x ignore vit merger Signed-off-by: tc-mb --------- Signed-off-by: tc-mb --- conversion/minicpm.py | 13 +- tools/mtmd/clip.cpp | 145 ++++++++------- tools/mtmd/models/minicpmv.cpp | 313 ++++++++++++++------------------- tools/mtmd/mtmd-image.cpp | 20 +++ tools/mtmd/mtmd-image.h | 10 +- tools/mtmd/mtmd.cpp | 2 +- 6 files changed, 251 insertions(+), 252 deletions(-) diff --git a/conversion/minicpm.py b/conversion/minicpm.py index e31b26a00808..bf3fa81421bd 100644 --- a/conversion/minicpm.py +++ b/conversion/minicpm.py @@ -137,6 +137,15 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca class MiniCPMV4_6VisionModel(MmprojModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.downsample_mode = self.preprocessor_config.get("downsample_mode", "16x") + if self.downsample_mode not in {"4x", "16x"}: + raise ValueError(f"Unsupported downsample mode: {self.downsample_mode}") + if self.downsample_mode == "4x": + self.model_tensors = { + name: tensor for name, tensor in self.model_tensors.items() + if ".vit_merger." not in name + } + if self.hparams_vision is not None: # In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP # positional embedding bucket grid (70 x 70), while the per-slice processing @@ -156,8 +165,8 @@ def set_gguf_parameters(self): # (mapped to PROJECTOR_TYPE_MINICPMV4_6). self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6) - # ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension; used for slice alignment - self.gguf_writer.add_vision_projector_scale_factor(4) + self.gguf_writer.add_vision_projector_scale_factor( + 2 if self.downsample_mode == "4x" else 4) # borrow wa_layer_indexes for vit_merger insertion point insert_layer_id = int(self.global_config.get( diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 5f0d00b66023..c1870813fb93 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1337,6 +1337,7 @@ struct clip_model_loader { // ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension hparams.n_merge = 4; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + GGML_ASSERT(hparams.n_merge == 2 || hparams.n_merge == 4); // borrow wa_layer_indexes for vit_merger insertion point std::vector wa_layer_indexes_vec; @@ -2143,24 +2144,29 @@ struct clip_model_loader { } break; case PROJECTOR_TYPE_MINICPMV4_6: { + const bool merger_required = hparams.n_merge == 4; + auto get_merger_tensor = [&](const std::string & name, bool required = true) { + return get_tensor(name, merger_required && required); + }; + // ViT merger: window self-attention - model.vit_merger_ln1_w = get_tensor(string_format(TN_VIT_MERGER_LN1, "weight")); - model.vit_merger_ln1_b = get_tensor(string_format(TN_VIT_MERGER_LN1, "bias")); - model.vit_merger_attn_q_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "weight")); - model.vit_merger_attn_q_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "bias"), false); - model.vit_merger_attn_k_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_K, "weight")); - model.vit_merger_attn_k_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_K, "bias"), false); - model.vit_merger_attn_v_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_V, "weight")); - model.vit_merger_attn_v_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_V, "bias"), false); - model.vit_merger_attn_o_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_O, "weight")); - model.vit_merger_attn_o_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_O, "bias"), false); + model.vit_merger_ln1_w = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "weight")); + model.vit_merger_ln1_b = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "bias")); + model.vit_merger_attn_q_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "weight")); + model.vit_merger_attn_q_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "bias"), false); + model.vit_merger_attn_k_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "weight")); + model.vit_merger_attn_k_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "bias"), false); + model.vit_merger_attn_v_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "weight")); + model.vit_merger_attn_v_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "bias"), false); + model.vit_merger_attn_o_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "weight")); + model.vit_merger_attn_o_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "bias"), false); // ViT merger: MLP downsample - model.vit_merger_ds_ln_w = get_tensor(string_format(TN_VIT_MERGER_DS_LN, "weight")); - model.vit_merger_ds_ln_b = get_tensor(string_format(TN_VIT_MERGER_DS_LN, "bias")); - model.vit_merger_ds_up_w = get_tensor(string_format(TN_VIT_MERGER_DS_UP, "weight")); - model.vit_merger_ds_up_b = get_tensor(string_format(TN_VIT_MERGER_DS_UP, "bias"), false); - model.vit_merger_ds_down_w = get_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "weight")); - model.vit_merger_ds_down_b = get_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "bias"), false); + model.vit_merger_ds_ln_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "weight")); + model.vit_merger_ds_ln_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "bias")); + model.vit_merger_ds_up_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "weight")); + model.vit_merger_ds_up_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "bias"), false); + model.vit_merger_ds_down_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "weight")); + model.vit_merger_ds_down_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "bias"), false); // Final Merger (DownsampleMLP) model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false); @@ -3591,8 +3597,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } break; case PROJECTOR_TYPE_MINICPMV4_6: { - // ViT merger 4x + final merger 4x = 16x total spatial downsample - n_patches = n_patches / 16; + n_patches /= params.n_merge * params.n_merge; } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: @@ -3974,6 +3979,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 } break; case PROJECTOR_TYPE_MINICPMV4_6: { + const bool is_4x = hparams.n_merge == 2; + // SigLIP position buckets (same as resampler path) std::vector positions(pos_h * pos_w); int bucket_coords_h[1024]; @@ -3994,40 +4001,6 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 const int half_h = pos_h / 2; const int half_w = pos_w / 2; - // window reorder indices for 2x2 windows - std::vector window_idx(n_pos); - std::vector inv_window_idx(n_pos); - { - int k = 0; - for (int wi = 0; wi < half_h; wi++) { - for (int wj = 0; wj < half_w; wj++) { - window_idx[k++] = (2*wi ) * pos_w + (2*wj ); - window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1); - window_idx[k++] = (2*wi + 1) * pos_w + (2*wj ); - window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1); - } - } - for (int i = 0; i < n_pos; i++) { - inv_window_idx[window_idx[i]] = i; - } - } - set_input_i32("vit_merger_window_idx", window_idx); - set_input_i32("vit_merger_inv_window_idx", inv_window_idx); - - // block-diagonal attention mask: tokens in the same 4-token - // window attend to each other (mask = 0), all other positions - // are masked out (-inf). matches the window-major reorder above. - std::vector window_mask_data(n_pos * n_pos, std::numeric_limits::lowest()); - for (int wi = 0; wi < n_pos / 4; wi++) { - for (int i = 0; i < 4; i++) { - for (int j = 0; j < 4; j++) { - window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f; - } - } - } - set_input_f32("vit_merger_window_mask", window_mask_data); - - // ViT merger 2x2 downsample indices auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) { std::vector idx(ds_h * ds_w); for (int i = 0; i < ds_h; i++) { @@ -4037,22 +4010,58 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 } return idx; }; - auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w); - auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w); - auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w); - auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w); - set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0); - set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1); - set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2); - set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3); - - // final merger 2x2 downsample indices (operates on half_h x half_w grid) - const int qh = half_h / 2; - const int qw = half_w / 2; - auto m_ds_0 = make_ds_idx(0, 0, qh, qw, half_w); - auto m_ds_1 = make_ds_idx(0, 1, qh, qw, half_w); - auto m_ds_2 = make_ds_idx(1, 0, qh, qw, half_w); - auto m_ds_3 = make_ds_idx(1, 1, qh, qw, half_w); + + if (!is_4x) { + // window reorder indices for 2x2 windows + std::vector window_idx(n_pos); + std::vector inv_window_idx(n_pos); + { + int k = 0; + for (int wi = 0; wi < half_h; wi++) { + for (int wj = 0; wj < half_w; wj++) { + window_idx[k++] = (2*wi ) * pos_w + (2*wj ); + window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1); + window_idx[k++] = (2*wi + 1) * pos_w + (2*wj ); + window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1); + } + } + for (int i = 0; i < n_pos; i++) { + inv_window_idx[window_idx[i]] = i; + } + } + set_input_i32("vit_merger_window_idx", window_idx); + set_input_i32("vit_merger_inv_window_idx", inv_window_idx); + + // block-diagonal attention mask: tokens in the same 4-token + // window attend to each other (mask = 0), all other positions + // are masked out (-inf). matches the window-major reorder above. + std::vector window_mask_data(n_pos * n_pos, std::numeric_limits::lowest()); + for (int wi = 0; wi < n_pos / 4; wi++) { + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f; + } + } + } + set_input_f32("vit_merger_window_mask", window_mask_data); + + // ViT merger 2x2 downsample indices + auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w); + auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w); + auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w); + auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w); + set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0); + set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1); + set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2); + set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3); + } + + const int merger_h = is_4x ? pos_h : half_h; + const int merger_w = is_4x ? pos_w : half_w; + auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w); set_input_i32("merger_ds_idx_0", m_ds_0); set_input_i32("merger_ds_idx_1", m_ds_1); set_input_i32("merger_ds_idx_2", m_ds_2); diff --git a/tools/mtmd/models/minicpmv.cpp b/tools/mtmd/models/minicpmv.cpp index bac087ffdfce..3e9c4c2a1114 100644 --- a/tools/mtmd/models/minicpmv.cpp +++ b/tools/mtmd/models/minicpmv.cpp @@ -114,14 +114,12 @@ ggml_cgraph * clip_graph_minicpmv::build() { } ggml_cgraph * clip_graph_minicpmv4_6::build() { - const int insert_lid = hparams.insert_layer_id; - const int n_pos = n_patches; - const int half_h = n_patches_y / 2; - const int half_w = n_patches_x / 2; - const int n_ds = half_h * half_w; // after ViT merger 2x2 downsample - const int qh = half_h / 2; - const int qw = half_w / 2; - const int n_ds2 = qh * qw; // after final merger 2x2 downsample + const bool is_4x = hparams.n_merge == 2; + const int n_pos = n_patches; + const int half_h = n_patches_y / 2; + const int half_w = n_patches_x / 2; + const int n_ds = half_h * half_w; + const int n_out = is_4x ? n_ds : (half_h / 2) * (half_w / 2); auto add_i32_input = [&](const char * name, int n) { ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n); @@ -134,29 +132,39 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() { ggml_tensor * positions = add_i32_input("positions", n_pos); ggml_tensor * learned_pos_embd = ggml_get_rows(ctx0, model.position_embeddings, positions); - // ViT merger window reorder indices + block-diagonal mask - // (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal, - // so each window-major group of 4 tokens only attends to itself) - ggml_tensor * vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos); - ggml_tensor * vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos); - ggml_tensor * vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos); - ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask"); - ggml_set_input(vit_merger_window_mask); - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { - vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16); - } + ggml_tensor * vit_merger_window_idx = nullptr; + ggml_tensor * vit_merger_inv_window_idx = nullptr; + ggml_tensor * vit_merger_window_mask = nullptr; + ggml_tensor * vit_merger_ds_idx_0 = nullptr; + ggml_tensor * vit_merger_ds_idx_1 = nullptr; + ggml_tensor * vit_merger_ds_idx_2 = nullptr; + ggml_tensor * vit_merger_ds_idx_3 = nullptr; + + if (!is_4x) { + // ViT merger window reorder indices + block-diagonal mask + // (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal, + // so each window-major group of 4 tokens only attends to itself) + vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos); + vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos); + vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos); + ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask"); + ggml_set_input(vit_merger_window_mask); + if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { + vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16); + } - // ViT merger 2x2 downsample gather indices - ggml_tensor * vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds); - ggml_tensor * vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds); - ggml_tensor * vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds); - ggml_tensor * vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds); + // ViT merger 2x2 downsample gather indices + vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds); + vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds); + vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds); + vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds); + } // final merger 2x2 downsample gather indices - ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_ds2); - ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_ds2); - ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_ds2); - ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_ds2); + ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_out); + ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_out); + ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_out); + ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_out); // patch embedding + positional embedding ggml_tensor * inp = build_inp(); @@ -169,150 +177,10 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() { cb(inpL, "pre_ln", -1); } - // ViT layers 0..insert_layer_id (inclusive) - // Mirrors the separate-qkv path of clip_graph::build_vit so the two manually - // unrolled segments around the ViT merger read like build_vit() expansions. - for (int il = 0; il <= insert_lid; il++) { - auto & layer = model.layers[il]; - ggml_tensor * cur = inpL; - - cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il); - cb(cur, "layer_inp_normed", il); - - { - ggml_tensor * Qcur = build_mm(layer.q_w, cur); - if (layer.q_b) { - Qcur = ggml_add(ctx0, Qcur, layer.q_b); - } - ggml_tensor * Kcur = build_mm(layer.k_w, cur); - if (layer.k_b) { - Kcur = ggml_add(ctx0, Kcur, layer.k_b); - } - ggml_tensor * Vcur = build_mm(layer.v_w, cur); - if (layer.v_b) { - Vcur = ggml_add(ctx0, Vcur, layer.v_b); - } - - Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos); - Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos); - Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos); - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, nullptr, kq_scale, il); - cb(cur, "attn_out", il); - } - - if (layer.ls_1_w) { - cur = ggml_mul(ctx0, cur, layer.ls_1_w); - cb(cur, "attn_out_scaled", il); - } - cur = ggml_add(ctx0, cur, inpL); - inpL = cur; - cb(cur, "ffn_inp", il); - - cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il); - cb(cur, "ffn_inp_normed", il); - - cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, layer.ff_gate_w, layer.ff_gate_b, - layer.ff_down_w, layer.ff_down_b, hparams.ffn_op, il); - cb(cur, "ffn_out", il); - - if (layer.ls_2_w) { - cur = ggml_mul(ctx0, cur, layer.ls_2_w); - cb(cur, "ffn_out_scaled", il); - } - cur = ggml_add(ctx0, inpL, cur); - cb(cur, "layer_out", il); - - inpL = cur; - } - - // ViT merger: window self-attention - // Tokens are reordered to window-major (4 tokens per window are contiguous), - // and a block-diagonal mask restricts attention to within each window. This - // mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the - // flash-attention path when available. - { - ggml_tensor * residual = inpL; - ggml_tensor * cur = build_norm(inpL, - model.vit_merger_ln1_w, model.vit_merger_ln1_b, - NORM_TYPE_NORMAL, eps, -1); - cb(cur, "vit_merger_attn_inp_normed", -1); - - cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx); - cb(cur, "vit_merger_window_reorder", -1); - - ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur); - if (model.vit_merger_attn_q_b) { - Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b); - } - ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur); - if (model.vit_merger_attn_k_b) { - Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b); - } - ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur); - if (model.vit_merger_attn_v_b) { - Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b); - } - - Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos); - Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos); - Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos); - cb(Qcur, "vit_merger_Qcur", -1); - cb(Kcur, "vit_merger_Kcur", -1); - cb(Vcur, "vit_merger_Vcur", -1); - - cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b, - Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1); - cb(cur, "vit_merger_attn_out", -1); - - cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx); - inpL = ggml_add(ctx0, cur, residual); - cb(inpL, "vit_merger_attn_residual", -1); - } - - // ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1) - { - ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0); - ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1); - ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2); - ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3); - - ggml_tensor * mean_res = ggml_add(ctx0, p0, p1); - mean_res = ggml_add(ctx0, mean_res, p2); - mean_res = ggml_add(ctx0, mean_res, p3); - mean_res = ggml_scale(ctx0, mean_res, 0.25f); - cb(mean_res, "vit_merger_ds_mean_res", -1); - - ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0); - cat = ggml_concat(ctx0, cat, p2, 0); - cat = ggml_concat(ctx0, cat, p3, 0); - - ggml_tensor * cur = build_norm(cat, - model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b, - NORM_TYPE_NORMAL, eps, -1); - cb(cur, "vit_merger_ds_normed", -1); - - // ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU) - cur = build_ffn(cur, - model.vit_merger_ds_up_w, model.vit_merger_ds_up_b, - nullptr, nullptr, - model.vit_merger_ds_down_w, model.vit_merger_ds_down_b, - FFN_GELU, -1); - cb(cur, "vit_merger_ds_mlp_out", -1); - - inpL = ggml_add(ctx0, cur, mean_res); - cb(inpL, "vit_merger_ds_out", -1); - } - - // ViT layers (insert_layer_id+1)..n_layer-1, operating on the downsampled tokens - { - const int64_t n_pos_ds = n_ds; - for (int il = insert_lid + 1; il < n_layer; il++) { + auto build_vit_layers = [&](ggml_tensor * input, int il_begin, int il_end, int64_t n_pos_layer) { + for (int il = il_begin; il < il_end; il++) { auto & layer = model.layers[il]; - ggml_tensor * cur = inpL; + ggml_tensor * cur = input; cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il); cb(cur, "layer_inp_normed", il); @@ -331,9 +199,9 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() { Vcur = ggml_add(ctx0, Vcur, layer.v_b); } - Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_ds); - Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_ds); - Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_ds); + Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_layer); + Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_layer); + Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_layer); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); @@ -346,8 +214,8 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() { cur = ggml_mul(ctx0, cur, layer.ls_1_w); cb(cur, "attn_out_scaled", il); } - cur = ggml_add(ctx0, cur, inpL); - inpL = cur; + cur = ggml_add(ctx0, cur, input); + input = cur; cb(cur, "ffn_inp", il); cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il); @@ -361,11 +229,98 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() { cur = ggml_mul(ctx0, cur, layer.ls_2_w); cb(cur, "ffn_out_scaled", il); } - cur = ggml_add(ctx0, inpL, cur); - cb(cur, "layer_out", il); + input = ggml_add(ctx0, input, cur); + cb(input, "layer_out", il); + } + return input; + }; + + if (!is_4x) { + const int insert_lid = hparams.insert_layer_id; + + inpL = build_vit_layers(inpL, 0, insert_lid + 1, n_pos); + + // ViT merger: window self-attention + // Tokens are reordered to window-major (4 tokens per window are contiguous), + // and a block-diagonal mask restricts attention to within each window. This + // mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the + // flash-attention path when available. + { + ggml_tensor * residual = inpL; + ggml_tensor * cur = build_norm(inpL, + model.vit_merger_ln1_w, model.vit_merger_ln1_b, + NORM_TYPE_NORMAL, eps, -1); + cb(cur, "vit_merger_attn_inp_normed", -1); + + cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx); + cb(cur, "vit_merger_window_reorder", -1); + + ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur); + if (model.vit_merger_attn_q_b) { + Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b); + } + ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur); + if (model.vit_merger_attn_k_b) { + Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b); + } + ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur); + if (model.vit_merger_attn_v_b) { + Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b); + } - inpL = cur; + Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos); + Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos); + Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos); + cb(Qcur, "vit_merger_Qcur", -1); + cb(Kcur, "vit_merger_Kcur", -1); + cb(Vcur, "vit_merger_Vcur", -1); + + cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b, + Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1); + cb(cur, "vit_merger_attn_out", -1); + + cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx); + inpL = ggml_add(ctx0, cur, residual); + cb(inpL, "vit_merger_attn_residual", -1); } + + // ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1) + { + ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0); + ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1); + ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2); + ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3); + + ggml_tensor * mean_res = ggml_add(ctx0, p0, p1); + mean_res = ggml_add(ctx0, mean_res, p2); + mean_res = ggml_add(ctx0, mean_res, p3); + mean_res = ggml_scale(ctx0, mean_res, 0.25f); + cb(mean_res, "vit_merger_ds_mean_res", -1); + + ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0); + cat = ggml_concat(ctx0, cat, p2, 0); + cat = ggml_concat(ctx0, cat, p3, 0); + + ggml_tensor * cur = build_norm(cat, + model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b, + NORM_TYPE_NORMAL, eps, -1); + cb(cur, "vit_merger_ds_normed", -1); + + // ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU) + cur = build_ffn(cur, + model.vit_merger_ds_up_w, model.vit_merger_ds_up_b, + nullptr, nullptr, + model.vit_merger_ds_down_w, model.vit_merger_ds_down_b, + FFN_GELU, -1); + cb(cur, "vit_merger_ds_mlp_out", -1); + + inpL = ggml_add(ctx0, cur, mean_res); + cb(inpL, "vit_merger_ds_out", -1); + } + + inpL = build_vit_layers(inpL, insert_lid + 1, n_layer, n_ds); + } else { + inpL = build_vit_layers(inpL, 0, n_layer, n_pos); } if (model.post_ln_w) { diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 72d35fce6918..10cfe52f56f7 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -972,6 +972,26 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const cl return output; } +// +// mtmd_image_preprocessor_minicpmv +// + +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const clip_image_size & original_size) { + if (hparams.n_merge == 2) { + const int slice_size = hparams.image_size; + const float ratio = (float)original_size.width * original_size.height / (slice_size * slice_size); + if (ratio <= 1.0f) { + mtmd_image_preprocessor_llava_uhd::slice_instructions inst; + const int patch_size = hparams.patch_size * hparams.n_merge; + inst.overview_size = get_best_resize(original_size, slice_size, patch_size, true); + inst.refined_size = clip_image_size{0, 0}; + inst.grid_size = clip_image_size{0, 0}; + return inst; + } + } + return mtmd_image_preprocessor_llava_uhd::get_slice_instructions(original_size); +} + // // mtmd_image_preprocessor_lfm2 // diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 115cba51e8f4..ecb203f76791 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -74,7 +74,6 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { std::vector slices; }; - // LFM2 override this function to implement its custom slicing logic virtual slice_instructions get_slice_instructions(const clip_image_size & original_size); struct slice_output { @@ -83,9 +82,10 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { }; slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst); -private: +protected: clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false); +private: clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max); /** @@ -129,6 +129,12 @@ struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor { mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; +// custom llava-uhd slicing logic for MiniCPM-V +struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd { + using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; + slice_instructions get_slice_instructions(const clip_image_size & original_size) override; +}; + // custom llava-uhd slicing logic for LFM2 // ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd { diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 93ca8cbcf8ae..d3899f5c853d 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -451,7 +451,7 @@ struct mtmd_context { tok_row_end = {lookup_token("\n")}; tok_row_end_trail = false; // no trailing end-of-row token ov_img_first = true; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(ctx_v); } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: From c629da565c80b0b17fac6262acdca4d772e745d8 Mon Sep 17 00:00:00 2001 From: Nico Date: Sat, 1 Aug 2026 12:03:32 -0400 Subject: [PATCH 15/98] cli : persist reasoning_content in chat history (#26362) * cli : persist reasoning_content in chat history llama-cli collected reasoning from the stream for display but only stored assistant content in messages, so --reasoning-preserve could not re-inject prior thoughts on later turns. --- tools/cli/cli-context.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/cli/cli-context.cpp b/tools/cli/cli-context.cpp index 0de8f69025ce..3d801b73d4c1 100644 --- a/tools/cli/cli-context.cpp +++ b/tools/cli/cli-context.cpp @@ -624,10 +624,14 @@ int cli_context::run() { generated_content content; generate_completion(content, timings); - impl->messages.push_back({ + json assistant_msg = { {"role", "assistant"}, {"content", content.content} - }); + }; + if (!content.reasoning.empty()) { + assistant_msg["reasoning_content"] = content.reasoning; + } + impl->messages.push_back(std::move(assistant_msg)); if (output_file) { std::string out_content = "Assistant:\n"; From 89482bd665673362b59c57d63295bd7c8259af62 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sat, 1 Aug 2026 18:45:46 +0200 Subject: [PATCH 16/98] agents: clarify comment style and jinja knowledge (#26405) * agents: clarify comment style and jinja knowledge * improve Security review a bit --- AGENTS.md | 29 +++++++++++++++++++++++------ skills/code-review/SKILL.md | 3 +++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1bf2a5781e69..48833d3cfcef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,11 +71,20 @@ For first-time contributors, confirm they have reviewed [CONTRIBUTING.md](CONTRI These points are extremely important - failing to follow them won't necessarily get your PR rejected, but it will make reviewing take significantly longer. Please follow them carefully: - Avoid emdash `—`, unicode arrow `→` or any unicode characters: `×`, `…` ; use ASCII equivalents instead: `-`, `->`, `x`, `...` -- Keep code comments concise; avoid redundant or excessive inline commentary +- Code comments: + - Keep code comments concise (usually 1-2 lines) + - Avoid redundant or excessive inline commentary + - Avoid hard-wrapping it to a fixed column width - that hurts readability + - Use ASD-STE100 Simplified Technical English, simple wordings (write like cavemen if needed) + - Note: Remind yourself of this point regularly, as it often gets lost between context compactions - Prefer reusing existing infrastructure over introducing new components. Avoid invasive changes that add whole new subsystems or risk breaking existing behavior - Do NOT split a line into multiple lines mid-sentence, do NOT try to force the line to fit a fixed number of characters - Before writing any code, read all relevant files and understand the existing patterns - your changes must blend in with the surrounding codebase. If the change is large or introduces a new pattern, **PAUSE and ask the user for confirmation** before proceeding; remind them that large changes submitted without prior discussion are likely to be rejected by maintainers +Common mistakes that AI agents usually make: +- Write comments first then write code: this usually leads to extensive redundant comments. Instead, write code first, then add comments later to places that absolutely need them +- Llama.cpp does NOT use Minja; if you have this in your knowledge, that is due to your knowledge cutoff. Llama.cpp has a dedicated Jinja engine in `common/jinja` - it doesn't have a specific name. + ### Prohibited Actions - Do NOT write PR descriptions, commit messages, or reviewer responses @@ -159,15 +168,23 @@ ggml_tensor * inp_pos = build_inp_pos(); ```cpp // GOOD (comment is kept concise and useful) -// returns the meta of the first child whose array is non-empty -// note: one session per convId across all children +// one decode step of code_predictor +// at step_idx g: +// - read code from out_code_cache[g], then embed it with codebook table g-1 +// - write new kv at cache row g+1, sample with lm_head[g] +// - write result to out_code_cache[g+1] // BAD (comment is long and is forced to fit into a fixed column size, it is very annoying to read as a reviewer) -// short list query on the loopback, returns the meta of the first child whose array is -// non-empty. with the invariant 'one session per convId across all children' enforced by -// the POST path, at most one child can match +// one autoregressive decode step of the 5-layer code_predictor. See the +// comment in models.h for the cache/tensor conventions this relies on. +// +// index mapping (derived from the reference pipeline-tts.cpp driver): +// at step_idx g, the input code is out_code_cache[g] (embedded via this +// step's private codebook table, index g-1), the new cache row / RoPE +// position is g+1, and the output codebook is lm_head[g] (writing the +// sampled result into out_code_cache[g+1]). ``` Commit message: diff --git a/skills/code-review/SKILL.md b/skills/code-review/SKILL.md index ba76c481150e..726edbb0ccb1 100644 --- a/skills/code-review/SKILL.md +++ b/skills/code-review/SKILL.md @@ -47,6 +47,7 @@ Mandatory on every review; any finding here is **blocking**. Rule of thumb: GGUF - **Sizes/counts from tensor dims:** validate before allocating. Products like `ne[i]*nb[i]`/nbytes can overflow on crafted dims into an undersized alloc then heap overflow. Overflow checks must run BEFORE the arithmetic they guard - padding/alignment macros wrap to 0 near `SIZE_MAX`, so a guard after the pad passes. - **GGUF strings/arrays:** cap declared lengths and element counts before using them to size a loop or buffer; validate element type and length before casting an array to a pointer or reading fixed indices (`[i+1]`, `[0..2]`). - **File-supplied counts indexing fixed arrays:** bound any count (e.g. layer/block count into a `LLAMA_MAX_*` array) before indexing; watch checks that only fire when an optional key is present. +- **Declared vs actual array length:** check the declared length of a GGUF array against the count actually read, not just against a buffer size. - **Bounds comparisons:** flag narrowing casts (`size_t`->`int32_t`) and signed/unsigned mixing that can bypass a length check and copy past a buffer. - **Parsed/derived indices:** range-check `stoi`/`atoi` results and catch parse throws; never use a default or derived token id (EOS/BOS/...) as an index without a bounds check. - **Reused/reserved buffers:** recheck bounds after a buffer is shrunk or reused; watch `reserve()` then index-by-assumed-size, and header fields read before their length is checked. @@ -131,6 +132,8 @@ Enforce the `AGENTS.md` / `CONTRIBUTING.md` coding and naming guidelines on ever - Reuse existing infrastructure over introducing new components; no new third-party dependencies, extra headers, or files unless clearly justified. - Keep it simple: a simpler change doing 90% is often preferable to a complex one doing 100%. Flag unnecessary templates/fancy STL; basic `for` loops are fine here. - Every added line should be something the contributor can explain and defend to a reviewer without AI help - flag anything that looks copied-in without understanding. +- `Co-authored-by:` must be reserved for human co-authors; AI contributions (claude, cursor, codex, etc.) must use `Assisted-by:`; if this point is violated, it's a blocking finding. +- Any mentions of Minja must be treated as blocking; see `AGENTS.md` for why. ## Reporting From 815a2a5915f22ce6a760c676389c5dfe8535c08f Mon Sep 17 00:00:00 2001 From: "Alessandro de Oliveira Faria (A.K.A.CABELO)" Date: Sat, 1 Aug 2026 15:53:00 -0300 Subject: [PATCH 17/98] vendor : update BoringSSL to 0.20260730.0 (#26353) --- vendor/cpp-httplib/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/cpp-httplib/CMakeLists.txt b/vendor/cpp-httplib/CMakeLists.txt index c6eb372b5f2c..bf674583ff9c 100644 --- a/vendor/cpp-httplib/CMakeLists.txt +++ b/vendor/cpp-httplib/CMakeLists.txt @@ -41,7 +41,7 @@ if (LLAMA_BUILD_BORINGSSL) set(FIPS OFF CACHE BOOL "Enable FIPS (BoringSSL)") set(BORINGSSL_GIT "https://boringssl.googlesource.com/boringssl" CACHE STRING "BoringSSL git repository") - set(BORINGSSL_VERSION "0.20260728.0" CACHE STRING "BoringSSL version") + set(BORINGSSL_VERSION "0.20260730.0" CACHE STRING "BoringSSL version") message(STATUS "Fetching BoringSSL version ${BORINGSSL_VERSION}") From a7a6d0d269c896218b6c78e0933bd6a17519d3f6 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Sat, 1 Aug 2026 14:18:07 -0500 Subject: [PATCH 18/98] vulkan: extend topk_moe fusion to support sqrt(softplus) (#26124) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 89 ++++++++++++++++--- .../ggml-vulkan/vulkan-shaders/topk_moe.comp | 8 ++ 2 files changed, 85 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index ff72fa60a8a9..510fb6892b37 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -610,6 +610,13 @@ static constexpr std::initializer_list topk_moe_sigmoid_norm_bias{ GGML GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE }; +static constexpr std::initializer_list topk_moe_sqrt_softplus_norm_bias{ GGML_OP_UNARY, GGML_OP_SQRT, + GGML_OP_RESHAPE, GGML_OP_ADD, + GGML_OP_ARGSORT, GGML_OP_VIEW, + GGML_OP_GET_ROWS, GGML_OP_RESHAPE, + GGML_OP_SUM_ROWS, GGML_OP_CLAMP, + GGML_OP_DIV, GGML_OP_RESHAPE }; + static constexpr std::initializer_list topk_moe_early_softmax { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS }; @@ -673,6 +680,22 @@ static constexpr std::initializer_list> topk_moe_sigmoid_norm {10, 0, 9 }, // reshape->src[0] == div }; +static constexpr std::initializer_list> topk_moe_sqrt_softplus_norm_bias_edges { + { 1, 0, 0 }, // sqrt->src[0] == softplus + { 2, 0, 1 }, // reshape->src[0] == sqrt + { 3, 0, 1 }, // add->src[0] == sqrt + { 4, 0, 3 }, // argsort->src[0] == add + { 5, 0, 4 }, // view->src[0] == argsort + { 6, 0, 2 }, // get_rows->src[0] == reshape + { 6, 1, 5 }, // get_rows->src[1] == view + { 7, 0, 6 }, // reshape->src[0] == get_rows + { 8, 0, 7 }, // sum_rows->src[0] == reshape + { 9, 0, 8 }, // clamp->src[0] == sum_rows + {10, 0, 7 }, // div->src[0] == reshape + {10, 1, 9 }, // div->src[1] == clamp + {11, 0,10 }, // reshape->src[0] == div +}; + // same as early_softmax_norm but ending after the get_rows static constexpr std::initializer_list> topk_moe_early_softmax_edges { { 1, 0, 0 }, // reshape->src[0] == softmax @@ -701,6 +724,7 @@ enum topk_moe_mode { TOPK_MOE_EARLY_SOFTMAX_NORM, TOPK_MOE_LATE_SOFTMAX, TOPK_MOE_SIGMOID_NORM_BIAS, + TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS, TOPK_MOE_COUNT, }; @@ -13203,12 +13227,16 @@ static void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& sub static void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) { topk_moe_mode mode = ctx->fused_topk_moe_mode; + const bool has_bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS || mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS; ggml_tensor * logits = cgraph->nodes[node_idx + 0]->src[0]; - ggml_tensor * bias = (mode == TOPK_MOE_SIGMOID_NORM_BIAS) ? cgraph->nodes[node_idx + 2]->src[1] : logits; + ggml_tensor * bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? cgraph->nodes[node_idx + 2]->src[1] : + mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? cgraph->nodes[node_idx + 3]->src[1] : + logits; ggml_tensor * weights = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; - ggml_tensor * ids = (mode == TOPK_MOE_SIGMOID_NORM_BIAS) ? cgraph->nodes[node_idx + 4] : - (mode == TOPK_MOE_LATE_SOFTMAX) ? cgraph->nodes[node_idx + 1] : - cgraph->nodes[node_idx + 3]; + ggml_tensor * ids = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? cgraph->nodes[node_idx + 4] : + mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? cgraph->nodes[node_idx + 5] : + mode == TOPK_MOE_LATE_SOFTMAX ? cgraph->nodes[node_idx + 1] : + cgraph->nodes[node_idx + 3]; GGML_ASSERT(logits->type == GGML_TYPE_F32); GGML_ASSERT(bias->type == GGML_TYPE_F32); @@ -13248,16 +13276,24 @@ static void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, pc.clamp_min = ggml_get_op_params_f32(clamp, 0); pc.clamp_max = ggml_get_op_params_f32(clamp, 1); } + if (mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS) { + ggml_tensor * clamp = cgraph->nodes[node_idx + 9]; + GGML_ASSERT(clamp->op == GGML_OP_CLAMP); + pc.clamp_min = ggml_get_op_params_f32(clamp, 0); + pc.clamp_max = ggml_get_op_params_f32(clamp, 1); + } #define GATING_FUNC_SOFTMAX 0 #define GATING_FUNC_SIGMOID 1 #define GATING_FUNC_SOFTMAX_WEIGHT 2 - - pc.gating_func = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? GATING_FUNC_SIGMOID : - mode == TOPK_MOE_LATE_SOFTMAX ? GATING_FUNC_SOFTMAX_WEIGHT : - GATING_FUNC_SOFTMAX; - pc.has_bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS; - pc.with_norm = mode == TOPK_MOE_EARLY_SOFTMAX_NORM || mode == TOPK_MOE_SIGMOID_NORM_BIAS; +#define GATING_FUNC_SQRT_SOFTPLUS 3 + + pc.gating_func = mode == TOPK_MOE_SIGMOID_NORM_BIAS ? GATING_FUNC_SIGMOID : + mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS ? GATING_FUNC_SQRT_SOFTPLUS : + mode == TOPK_MOE_LATE_SOFTMAX ? GATING_FUNC_SOFTMAX_WEIGHT : + GATING_FUNC_SOFTMAX; + pc.has_bias = has_bias; + pc.with_norm = mode == TOPK_MOE_EARLY_SOFTMAX_NORM || has_bias; if (ctx->fused_topk_moe_scale) { GGML_ASSERT(weights->op == GGML_OP_SCALE); pc.output_scale = ggml_get_op_params_f32(weights, 0); @@ -16366,6 +16402,20 @@ static bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struc return false; } break; + case TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS: + softmax = cgraph->nodes[node_idx + 0]; // really softplus + weights = cgraph->nodes[node_idx + 11]; + get_rows = cgraph->nodes[node_idx + 6]; + argsort = cgraph->nodes[node_idx + 4]; + if (ggml_get_unary_op(softmax) != GGML_UNARY_OP_SOFTPLUS) { + return false; + } + // bias is expected to be 1D + if (ggml_nrows(cgraph->nodes[node_idx + 3]->src[1]) != 1 || + !ggml_is_contiguous(cgraph->nodes[node_idx + 3]->src[1])) { + return false; + } + break; case TOPK_MOE_EARLY_SOFTMAX: softmax = cgraph->nodes[node_idx + 0]; weights = cgraph->nodes[node_idx + 4]; @@ -16389,7 +16439,9 @@ static bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struc probs = probs->src[0]; ggml_tensor * selection_probs = argsort->src[0]; - if (probs != selection_probs && mode != TOPK_MOE_SIGMOID_NORM_BIAS) { + if (probs != selection_probs && + mode != TOPK_MOE_SIGMOID_NORM_BIAS && + mode != TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS) { return false; } @@ -16757,7 +16809,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // the fused result in an elementwise-way. This affects whether the memory for // the src is allowed to overlap the memory for the destination. // The array is sized to handle the largest fusion (asserted later). - bool op_srcs_fused_elementwise[12]; + bool op_srcs_fused_elementwise[13]; ctx->fused_topk_moe_mode = TOPK_MOE_COUNT; ctx->fused_topk_moe_scale = false; @@ -16868,6 +16920,15 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->fused_topk_moe_mode = TOPK_MOE_SIGMOID_NORM_BIAS; fusion_string = "TOPK_MOE_SIGMOID_NORM_BIAS"; std::fill_n(op_srcs_fused_elementwise, ctx->num_additional_fused_ops + 1, false); + } else if (ggml_can_fuse_subgraph(cgraph, i, topk_moe_sqrt_softplus_norm_bias, { i + 5, i + 11 }) && + ggml_check_edges(cgraph, i, topk_moe_sqrt_softplus_norm_bias_edges) && + ggml_vk_can_fuse_topk_moe(ctx, cgraph, i, TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS)) { + ctx->num_additional_fused_ops = topk_moe_sqrt_softplus_norm_bias.size() - 1; + // view of argsort writes to memory + ctx->fused_ops_write_mask |= 1 << 5; + ctx->fused_topk_moe_mode = TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS; + fusion_string = "TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS"; + std::fill_n(op_srcs_fused_elementwise, ctx->num_additional_fused_ops + 1, false); } else if (ggml_can_fuse_subgraph(cgraph, i, topk_moe_early_softmax, { i + 3, i + 4 }) && ggml_check_edges(cgraph, i, topk_moe_early_softmax_edges) && ggml_vk_can_fuse_topk_moe(ctx, cgraph, i, TOPK_MOE_EARLY_SOFTMAX)) { @@ -17134,6 +17195,9 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * if (keep_pattern(topk_moe_sigmoid_norm_bias)) { continue; } + if (keep_pattern(topk_moe_sqrt_softplus_norm_bias)) { + continue; + } if (keep_pattern(topk_moe_early_softmax)) { continue; } @@ -17164,6 +17228,7 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * // Don't pull forward nodes from fusion patterns if (match_pattern(topk_moe_early_softmax_norm, j) || match_pattern(topk_moe_sigmoid_norm_bias, j) || + match_pattern(topk_moe_sqrt_softplus_norm_bias, j) || match_pattern(topk_moe_early_softmax, j) || match_pattern(topk_moe_late_softmax, j) || match_pattern(snake_pattern, j)) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp b/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp index ef2f202ec9b6..d219201fda0a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp @@ -10,6 +10,7 @@ #define GATING_FUNC_SOFTMAX 0 #define GATING_FUNC_SIGMOID 1 #define GATING_FUNC_SOFTMAX_WEIGHT 2 +#define GATING_FUNC_SQRT_SOFTPLUS 3 layout (push_constant) uniform parameter { @@ -120,6 +121,13 @@ void main() { const uint expert = i + lane; probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? 1.f / (1.f + exp(-probs[i / WARP_SIZE])) : -INFINITY; } + } else if (gating_func == GATING_FUNC_SQRT_SOFTPLUS) { + [[unroll]] + for (uint i = 0; i < n_experts; i += WARP_SIZE) { + const uint expert = i + lane; + const float val = probs[i / WARP_SIZE]; + probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? sqrt(val > 20.0f ? val : log(1.0f + exp(val))) : -INFINITY; + } } float selection_probs[experts_per_thread]; From 11924d4c17abc27383376a1ac6a24fa3e36c1c0c Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sun, 2 Aug 2026 00:16:29 +0200 Subject: [PATCH 19/98] test: fix some CI errors (#26415) --- tests/test-llama-archs.cpp | 2 +- tools/ui/CMakeLists.txt | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index a1ed2a76f879..4336e4e13d4f 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -430,7 +430,7 @@ static bool arch_supported(const llm_arch arch) { // FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI. #ifdef GGML_USE_WEBGPU - if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA) { + if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_MINIMAX_M3) { return false; } #endif // GGML_USE_WEBGPU diff --git a/tools/ui/CMakeLists.txt b/tools/ui/CMakeLists.txt index 74bca417e372..208b46a5c15a 100644 --- a/tools/ui/CMakeLists.txt +++ b/tools/ui/CMakeLists.txt @@ -61,12 +61,30 @@ if(CMAKE_CROSSCOMPILING) # phony target to tie it into the dependency graph add_custom_target(llama-ui-embed DEPENDS "${LLAMA_UI_EMBED_EXE}") else() + # exclude llama-ui-embed from sanitizer flags, + # it's a build-time-only tool, no need to instrument it + # this is to fix TSan "memory layout is incompatible" error on CI + get_directory_property(_llama_ui_dir_co COMPILE_OPTIONS) + get_directory_property(_llama_ui_dir_ll LINK_LIBRARIES) + set(_llama_ui_embed_co ${_llama_ui_dir_co}) + set(_llama_ui_embed_ll ${_llama_ui_dir_ll}) + list(FILTER _llama_ui_embed_co EXCLUDE REGEX ".*-fsanitize=.*") + list(FILTER _llama_ui_embed_ll EXCLUDE REGEX ".*-fsanitize=.*") + set_directory_properties(PROPERTIES + COMPILE_OPTIONS "${_llama_ui_embed_co}" + LINK_LIBRARIES "${_llama_ui_embed_ll}") + add_executable(llama-ui-embed embed.cpp) target_compile_features(llama-ui-embed PRIVATE cxx_std_17) set_target_properties(llama-ui-embed PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) set(LLAMA_UI_EMBED_EXE "$") + + # restore so the llama-ui library below keeps sanitizer instrumentation + set_directory_properties(PROPERTIES + COMPILE_OPTIONS "${_llama_ui_dir_co}" + LINK_LIBRARIES "${_llama_ui_dir_ll}") endif() # Run the provisioning script every build so source changes in tools/ui/ are From 7a2db1a0cf38e5f4eb278a58c3903d295258d0ea Mon Sep 17 00:00:00 2001 From: Masashi Yoshimura Date: Sun, 2 Aug 2026 15:28:31 +0900 Subject: [PATCH 20/98] ggml-webgpu: add support for f16 repeat (#26307) --- ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp | 4 ++++ ggml/src/ggml-webgpu/ggml-webgpu.cpp | 3 ++- ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl | 3 +++ tests/test-backend-ops.cpp | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp index babaddb6542d..66c1c3c8977e 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp @@ -2774,6 +2774,10 @@ class ggml_webgpu_shader_lib { defines.push_back("TYPE_F32"); variant += "_f32"; break; + case GGML_TYPE_F16: + defines.push_back("TYPE_F16"); + variant += "_f16"; + break; case GGML_TYPE_I32: defines.push_back("TYPE_I32"); variant += "_i32"; diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 370f05dfe67a..c001cda7d116 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -4290,7 +4290,8 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const supports_op = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32); break; case GGML_OP_REPEAT: - supports_op = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32 || src0->type == GGML_TYPE_I16); + supports_op = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_I32 || + src0->type == GGML_TYPE_I16); break; case GGML_OP_CPY: case GGML_OP_CONT: diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl index 6e2a1a8b614d..43b883e67945 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl @@ -27,6 +27,9 @@ struct Params { #ifdef TYPE_I32 #define DataType i32 #endif +#ifdef TYPE_F16 +#define DataType f16 +#endif #ifdef TYPE_I16 // same size (16-bit) is sufficient for repeat #define DataType f16 diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 4098acaaf91a..8f3fb45dab65 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8511,6 +8511,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 2, 1, 1})); test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 2, 1})); test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 1, 2})); + test_cases.emplace_back(new test_repeat(GGML_TYPE_F16, {10, 5, 4, ne3}, {2, 1, 1, 1})); test_cases.emplace_back(new test_repeat(GGML_TYPE_I32, {10, 5, 4, ne3}, {2, 1, 1, 1})); test_cases.emplace_back(new test_repeat(GGML_TYPE_I16, {10, 5, 4, ne3}, {1, 1, 1, 2})); test_cases.emplace_back(new test_repeat(GGML_TYPE_BF16, {10, 5, 4, ne3}, {2, 1, 1, 1})); From 75587a05b39e5d5866a2e120178c75c12dfb75cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sun, 2 Aug 2026 09:03:05 +0200 Subject: [PATCH 21/98] model : load MiMo V2 MTP tensors only if used (#26412) --- src/models/mimo2.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cpp index 4080a934cb98..d50e186cce92 100644 --- a/src/models/mimo2.cpp +++ b/src/models/mimo2.cpp @@ -30,7 +30,11 @@ void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) { const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); - const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); From 272700b360944e40816a7ea13da8cd723119000a Mon Sep 17 00:00:00 2001 From: KyleHagy <59183061+KyleHagy@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:10:32 -0700 Subject: [PATCH 22/98] sycl: fix classification of iGPUs (#26105) --- ggml/src/ggml-sycl/common.hpp | 1 + ggml/src/ggml-sycl/ggml-sycl.cpp | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 160331191d10..619933e0fda7 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -235,6 +235,7 @@ struct sycl_device_info { int max_wg_per_cu; // max work groups per compute unit - refer to // cudaOccupancyMaxActiveBlocksPerMultiprocessor bool vmm; // virtual memory support + bool l0_device_type_valid; bool l0_discrete_gpu; // Level Zero backend and not an integrated GPU size_t vmm_granularity; // granularity of virtual memory size_t total_vram; diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 07ddfda88764..d91e41f9575a 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -167,7 +167,10 @@ static ggml_sycl_device_info ggml_sycl_init() { ze_device_properties_t props = {}; props.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; ze_result_t r = zeDeviceGetProperties(ze_dev, &props); - info.devices[i].l0_discrete_gpu = r == ZE_RESULT_SUCCESS && !(props.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED); + if (r == ZE_RESULT_SUCCESS) { + info.devices[i].l0_device_type_valid = true; + info.devices[i].l0_discrete_gpu = !(props.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED); + } } #endif } @@ -5606,7 +5609,11 @@ static void ggml_backend_sycl_device_get_memory(ggml_backend_dev_t dev, size_t * } static enum ggml_backend_dev_type ggml_backend_sycl_device_get_type(ggml_backend_dev_t dev) { - GGML_UNUSED(dev); + ggml_backend_sycl_device_context * ctx = (ggml_backend_sycl_device_context *)dev->context; + const sycl_device_info & info = ggml_sycl_info().devices[ctx->device]; + if (info.l0_device_type_valid && !info.l0_discrete_gpu) { + return GGML_BACKEND_DEVICE_TYPE_IGPU; + } return GGML_BACKEND_DEVICE_TYPE_GPU; } From f5919bf458ef190468b5c329bb293f8a54a1e69c Mon Sep 17 00:00:00 2001 From: Aldehir Rojas Date: Sun, 2 Aug 2026 04:13:20 -0500 Subject: [PATCH 23/98] chat : add qwen3 specialized parser (#26252) * Add tagged thinking tool parser * chat : refactor and add permute helper * cont : add support for omission * cont : update tool delimiters * cont : add comment for qwen3-coder * cont : fix trigger pattern for --- common/chat-peg-parser.cpp | 40 +++++++ common/chat-peg-parser.h | 5 + common/chat.cpp | 174 +++++++++++++++++++++++++++++ tests/test-chat-peg-parser.cpp | 103 ++++++++++++++++++ tests/test-chat.cpp | 193 +++++++++++++++++++-------------- 5 files changed, 431 insertions(+), 84 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index f786f5ff2314..1910b4f1e135 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -6,6 +6,9 @@ #include +#include +#include + using ordered_json = nlohmann::ordered_json; static std::string_view trim_trailing_space(std::string_view sv, int max = -1) { @@ -235,6 +238,43 @@ common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::stri return zero_or_more(choice({ p, content_chunk })); } +common_peg_parser common_chat_peg_builder::permute(const std::string & rule_prefix, + const std::vector & parsers) { + if (parsers.empty()) { + return eps(); + } + + if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) { + return sequence(parsers); + } + + std::map rules; + std::function remaining_of; + + remaining_of = [&](uint32_t remaining) -> common_peg_parser { + if (remaining == 0) { + return eps(); + } + + auto cached = rules.find(remaining); + if (cached != rules.end()) { + return cached->second; + } + + auto alternatives = choice(); + for (size_t i = 0; i < parsers.size(); i++) { + const uint32_t bit = 1u << i; + if (remaining & bit) { + alternatives |= parsers[i] + remaining_of(remaining & ~bit); + } + } + + return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second; + }; + + return remaining_of((1u << parsers.size()) - 1); +} + std::string & common_chat_peg_mapper::args_target() { return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer; } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index cd14f2c11750..5d764dbaa0ec 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -55,6 +55,8 @@ class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper { struct content_structure; struct tool_call_structure; +constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6; + class common_chat_peg_builder : public common_peg_parser_builder { public: // Tag constants (from former common_chat_peg_base_builder) @@ -105,6 +107,9 @@ class common_chat_peg_builder : public common_peg_parser_builder { common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); } + // Matches every parser exactly once, in any order. + common_peg_parser permute(const std::string & rule_prefix, const std::vector & parsers); + // Return a parser that parses the prefix of a string, up to a given delimiter. common_peg_parser prefix(const std::string & s, const std::string & delimiter = {}); diff --git a/common/chat.cpp b/common/chat.cpp index 0e06fa591f9d..f7431a6f13f0 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1110,6 +1110,172 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_ return data; } +static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + const std::string GEN_PREFIX = "<|im_start|>assistant\n"; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + + auto supports_reasoning = tmpl.source().find("") != std::string::npos; + + data.supports_thinking = supports_reasoning; + data.preserved_tokens = { + "", + "", + }; + + if (supports_reasoning) { + data.thinking_start_tag = ""; + // Support both and as reasoning end sequences. + // ", "" }; + data.preserved_tokens.insert(data.preserved_tokens.end(), { "", "" }); + } + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3 + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash + { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PREFIX; + if (supports_reasoning) { + data.generation_prompt += "\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "\n\n\n"; + } + } + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PREFIX); + + auto reasoning = p.eps(); + if (supports_reasoning && extract_reasoning) { + reasoning = p.optional("" + p.space() + + p.reasoning(p.until_one_of({ "", "" })) + + (p.literal("") | p.peek(p.literal("")))); + } + + // Response format parser + if (has_response_format) { + return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); + } + + // Tool call parser + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto arg_close = p.tool_arg_close(p.literal("\n\n")); + auto arg_string = p.rule("xml-arg-string", + p.ac(p.tool_arg_string_value(p.until("\n\n")) + arg_close, "\n\n")); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto parameters = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto schema_info = common_schema_info(); + schema_info.resolve_refs(parameters); + + std::vector required_args; + std::vector optional_args; + + foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { + auto rule_name = "tool-" + name + "-arg-" + param_name; + + auto arg_open = p.tool_arg_open("\n"); + + auto arg_value = schema_info.resolves_to_string(param_schema) ? + arg_string : + p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close; + + auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); + + (is_required ? required_args : optional_args).push_back(arg_rule); + }); + + // Accept required arguments in any order, as Qwen does not always adhere to the + // order provided. + auto args = p.permute("tool-" + name + "-args", required_args); + if (!optional_args.empty()) { + args = args + p.zero_or_more(p.choice(optional_args)); + } + + auto func = p.tool(p.tool_open("\n") + + p.tool_args(args) + + p.tool_close(p.literal("\n"))); + + tool_choice |= p.rule("tool-" + name, func); + }); + + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + + // Qwen3-Coder models may occasionally omit the token. + auto tool_call_body = tool_choice + "" + p.space(); + auto tool_call_first = p.rule("tool-call-first", p.optional(p.literal("\n")) + tool_call_body); + auto tool_call = p.rule("tool-call", "\n" + tool_call_body); + + auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; + auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); + + return generation_prompt + + (reasoning << p.content(p.until_one_of({ "", "" }, + // Trigger on " common_chat_try_specialized_template( return common_chat_params_init_minicpm5(tmpl, params); } + // Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash + if (src.find("") != std::string::npos && + src.find(" #include +#include #include #include "nlohmann/json.hpp" @@ -21,6 +22,7 @@ static void test_example_qwen3_non_coder(testing & t); static void test_command7_parser_compare(testing & t); static void test_prefix_tool_names(testing & t); static void test_tagged_peg_parser(testing & t); +static void test_permute(testing & t); int main(int argc, char * argv[]) { testing t(std::cout); @@ -39,6 +41,7 @@ int main(int argc, char * argv[]) { t.test("comparison", test_command7_parser_compare); t.test("prefix tool names", test_prefix_tool_names); t.test("tagged peg parser", test_tagged_peg_parser); + t.test("permute", test_permute); return t.summary(); } @@ -981,3 +984,103 @@ static void test_tagged_peg_parser(testing & t) { t.assert_equal("fun_post should be '>'", ">", result.tags["fun_post"]); }); } + +static void test_permute(testing & t) { + auto accepts = [](const common_peg_arena & parser, const std::string & input) { + common_peg_parse_context ctx(input); + return parser.parse(ctx).success(); + }; + + auto gbnf_of = [](const common_peg_arena & parser) { + return build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + }; + + auto assert_gbnf_equal = [](testing & t, const std::string & expected, const std::string & actual) { + static const std::regex leading_ws_re = std::regex(R"((^|\n)\s+)"); + t.assert_equal("gbnf are equal", std::regex_replace(expected, leading_ws_re, "$1"), actual); + }; + + auto count_rules = [](const std::string & gbnf, const std::string & prefix) { + size_t count = 0; + for (const auto & line : string_split(gbnf, '\n')) { + if (line.rfind(prefix, 0) == 0) { + count++; + } + } + return count; + }; + + t.test("accepts every ordering", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end(); + }); + + for (const std::string input : { "abc", "acb", "bac", "bca", "cab", "cba" }) { + t.assert_true("accepts " + input, accepts(parser, input)); + } + }); + + t.test("single element", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("a", { p.literal("a") }) + p.end(); + }); + + t.assert_true("accepts a", accepts(parser, "a")); + t.assert_true("rejects aa", !accepts(parser, "aa")); + }); + + t.test("grammar left-factorizes shared tails", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end(); + }); + + // Every rule is one remaining subset, keyed by bitmask: abc-3 is {a,b}, abc-7 is {a,b,c}. + // Each subset is emitted once and shared by every branch that leads into it. + assert_gbnf_equal(t, R"""( + abc-1 ::= "a" + abc-2 ::= "b" + abc-3 ::= "a" abc-2 | "b" abc-1 + abc-4 ::= "c" + abc-5 ::= "a" abc-4 | "c" abc-1 + abc-6 ::= "b" abc-4 | "c" abc-2 + abc-7 ::= "a" abc-6 | "b" abc-5 | "c" abc-3 + root ::= abc-7 + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf_of(parser)); + }); + + t.test("grammar emits one rule per remaining subset", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("abcd", { p.literal("a"), p.literal("b"), p.literal("c"), p.literal("d") }) + p.end(); + }); + + // 2^4 - 1 non-empty subsets, one rule each - not the 4! = 24 orderings. + t.assert_equal("permute rule count", 15u, count_rules(gbnf_of(parser), "abcd-")); + }); + + t.test("grammar emits no rules for a single element", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("a", { p.literal("a") }) + p.end(); + }); + + assert_gbnf_equal(t, R"""( + root ::= "a" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf_of(parser)); + }); + + t.test("grammar falls back to the given order when too large", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + std::vector parsers; + for (size_t i = 0; i <= COMMON_CHAT_MAX_PERMUTE; i++) { + parsers.push_back(p.literal(std::string(1, (char) ('a' + i)))); + } + return p.permute("big", parsers) + p.end(); + }); + + assert_gbnf_equal(t, R"""( + root ::= "a" "b" "c" "d" "e" "f" "g" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf_of(parser)); + }); +} diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index ef02fdde57ef..8d571d369e55 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -2278,46 +2278,39 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") .run(); - // tool call segment in reasoning + // a tool call ends the prefilled thinking block, with or without a closing tst.test( - "Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" + "\n" + "\n" + "\n" + "pwd\n" "\n" "\n" - "\n\n\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + // ...including after the model has thought about it + tst.test( + "Need to inspect the current directory.\n" "\n" - "\n" - "\n" - "def hello():\n" - " print(\"Hello, world!\")\n" - "\n" - "hello()\n" + "\n" + "\n" + "pwd\n" "\n" "\n" "") .enable_thinking(true) .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ - python_tool - }) - .expect_reasoning( - "Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "") + .tools({ run_in_terminal_tool }) + .expect_reasoning("Need to inspect the current directory.") .expect_tool_calls({ - { "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} }, + { "run_in_terminal", R"({"command": "pwd"})", {} }, }) .run(); @@ -2461,17 +2454,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); - tst.test( - "I might call later, but I am still thinking.\n" - "\n\n" - "Final answer without tools.") - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .enable_thinking(true) - .tools({ run_in_terminal_tool }) - .expect_reasoning("I might call later, but I am still thinking.") - .expect_content("Final answer without tools.") - .run(); - // Continuation tests tst.test("world!\nWhat's up?") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) @@ -2776,49 +2758,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") .run(); - // tool call segment in reasoning - tst.test( - "Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n\n" - "\n" - "\n" - "\n" - "def hello():\n" - " print(\"Hello, world!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n" - ) - .enable_thinking(true) - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ - python_tool - }) - .expect_reasoning("Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n") - .expect_tool_calls({ - { "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} }, - }) - .run(); - // Continuation tests tst.test("world!\nWhat's up?") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) @@ -3572,6 +3511,61 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_reconstruction() .run(); + // Some models skip the opening and go straight to + tst.test( + "\n" + "\n" + "1\n" + "\n" + "\n" + "") + .tools({ special_function_tool }) + .expect(message_assist_call) + .run(); + + tst.test( + "Let me call it.\n" + "\n" + "\n" + "1\n" + "\n" + "\n" + "") + .tools({ special_function_tool }) + .expect_content("Let me call it.\n") + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + }) + .run(); + + // Only the first call may omit it, the rest keep the \n separator + tst.test( + "\n" + "\n" + "1\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "1\n" + "\n" + "\n" + "2\n" + "\n" + "\n" + "") + .parallel_tool_calls(true) + .tools({ + special_function_tool, special_function_tool_with_optional_param + }) + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} }, + }) + .run(); + tst.test( "\n" "\n" @@ -3680,6 +3674,37 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_reconstruction() .run(); + // Test flexible required argument ordering (required args still come first, in any order) + tst.test( + "\n" + "\n" + "\n#include\n\n" + "\nfoo.c\n\n" + "\n#iclunde\n\n" + "\n" + "") + .tools({ edit_tool }) + .expect_tool_calls({ + { "edit", R"({"newString": "#include", "filename": "foo.c", "oldString": "#iclunde"})", {} }, + }) + .expect_reconstruction() + .run(); + + tst.test( + "\n" + "\n" + "\n42\n\n" + "\nhello\n\n" + "\n200\n\n" + "\n" + "") + .tools({ tool_2req_4opt }) + .expect_tool_calls({ + { "tool_2req_4opt", R"({"req2": 42, "req1": "hello", "opt2": 200})", {} }, + }) + .expect_reconstruction() + .run(); + // Test flexible optional argument ordering (2 required + 4 optional, reversed optional order) tst.test( "\n" From 596a5795bdd6da317ea103fc06c0a71c296e3669 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Sun, 2 Aug 2026 20:55:34 +0800 Subject: [PATCH 24/98] DeepseekV4 MTP + DSpark (#25784) --- common/speculative.cpp | 2 +- conversion/deepseek.py | 116 +++++- gguf-py/gguf/constants.py | 6 + src/llama-arch.cpp | 2 + src/llama-context.cpp | 24 +- src/llama-graph.cpp | 172 ++++++++- src/llama-graph.h | 63 ++- src/llama-kv-cache-dsv4.cpp | 345 +++++++++++++---- src/llama-kv-cache-dsv4.h | 35 +- src/llama-model.cpp | 93 ++++- src/models/deepseek4.cpp | 489 ++++++++++++++++++++---- src/models/dflash.cpp | 270 +++++++++++++ src/models/models.h | 24 ++ tests/test-recurrent-state-rollback.cpp | 114 ++++-- 14 files changed, 1532 insertions(+), 223 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index 5653a90b889c..514fe852180f 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1291,7 +1291,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { GGML_ASSERT(ctx_tgt && ctx_dft && "MTP requires ctx_tgt and ctx_dft to be set"); n_embd = llama_model_n_embd_out(llama_get_model(ctx_dft)); - GGML_ASSERT(n_embd == llama_model_n_embd(llama_get_model(ctx_tgt)) && + GGML_ASSERT(n_embd == llama_model_n_embd_out(llama_get_model(ctx_tgt)) && "MTP input row width must match the target h_nextn width"); n_mtp_layers = std::max(1, (int) llama_model_n_layer_nextn(llama_get_model(ctx_dft))); diff --git a/conversion/deepseek.py b/conversion/deepseek.py index ea6ae23d58e7..0bf69be3be98 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -475,7 +475,10 @@ def set_gguf_parameters(self): @ModelBase.register("DeepseekV4ForCausalLM") class DeepseekV4Model(TextModel): model_arch = gguf.MODEL_ARCH.DEEPSEEK4 + supports_mtp_export = True _skipped_mtp_tensors = 0 + _dsv4_main_layers: int | None = None + _dsv4_nextn_layers: int = 0 def __init__(self, *args, **kwargs): type(self)._skipped_mtp_tensors = 0 @@ -487,6 +490,8 @@ def __init__(self, *args, **kwargs): self.hparams.setdefault(key, value) self.block_count = self.hparams["num_hidden_layers"] + if self.mtp_only: + self.block_count += self.hparams.get("num_nextn_predict_layers", 0) self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) self._dsv4_fp8_dequantized: set[str] = set() @@ -504,13 +509,63 @@ def __init__(self, *args, **kwargs): with open(template_path, "r", encoding="utf-8") as f: self.gguf_writer.add_chat_template(f.read()) + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + type(self)._dsv4_main_layers = self.hparams["num_hidden_layers"] + type(self)._dsv4_nextn_layers = self.hparams.get("num_nextn_predict_layers", 0) + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, _ = item + name, gen = item if name.startswith("mtp."): - cls._skipped_mtp_tensors += 1 - return None - return super().filter_tensors(item) + if not cls.mtp_only: + cls._skipped_mtp_tensors += 1 + return None + + assert cls._dsv4_main_layers is not None + parts = name.split(".", 2) + if len(parts) < 3 or not parts[1].isdecimal(): + raise ValueError(f"Unexpected DeepSeek-V4 MTP tensor {name!r}") + + mtp_idx = int(parts[1]) + if mtp_idx >= cls._dsv4_nextn_layers: + raise ValueError(f"Unexpected DeepSeek-V4 MTP layer {mtp_idx}") + + bid = cls._dsv4_main_layers + mtp_idx + suffix = parts[2] + root_hc_head = { + "hc_head_fn", + "hc_head_base", + "hc_head_scale", + } + if suffix in root_hc_head: + name = suffix + elif suffix in ( + "e_proj.weight", "e_proj.scale", + "h_proj.weight", "h_proj.scale", + ): + name = f"layers.{bid}.nextn.{suffix}" + elif suffix == "enorm.weight": + name = f"layers.{bid}.nextn.enorm.weight" + elif suffix == "hnorm.weight": + name = f"layers.{bid}.nextn.hnorm.weight" + elif suffix == "norm.weight": + name = f"layers.{bid}.nextn.shared_head_norm.weight" + else: + name = f"layers.{bid}.{suffix}" + return name, gen + + if cls.mtp_only: + keep = name in ( + "embed.weight", + "norm.weight", + "head.weight", + "head.scale", + ) + if not keep: + return None + + return super().filter_tensors((name, gen)) @staticmethod def _float8_dtypes() -> tuple[torch.dtype, ...]: @@ -565,6 +620,9 @@ def set_gguf_parameters(self): self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"]) self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"]) self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"]) + self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"]) + if self.mtp_only and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers", 0)) > 0: + self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers) def dequant_model(self): fp8_dtypes = self._float8_dtypes() @@ -669,12 +727,37 @@ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: if self._dsv4_mxfp4_generated: return () - consumed: list[str] = self._write_hash_routing_tensors() + consumed: list[str] = [] + main_layers = self.hparams["num_hidden_layers"] + if not self.mtp_only: + consumed.extend(self._write_hash_routing_tensors()) + elif self.hparams["num_hash_layers"] > 0: + for bid in range(self.hparams["num_hash_layers"]): + name = f"layers.{bid}.ffn.gate.tid2eid" + if name in self.model_tensors: + consumed.extend(self._write_hash_routing_tensors()) + break + for bid in range(self.block_count): + if self.mtp_only and bid < main_layers: + continue consumed.extend(self._write_mxfp4_expert_tensor(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP)) consumed.extend(self._write_mxfp4_expert_tensor(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP)) consumed.extend(self._write_mxfp4_expert_tensor(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP)) + for bid in range(main_layers, self.block_count): + e_name = f"layers.{bid}.nextn.e_proj.weight" + h_name = f"layers.{bid}.nextn.h_proj.weight" + if e_name not in self.model_tensors and h_name not in self.model_tensors: + continue + if e_name not in self.model_tensors or h_name not in self.model_tensors: + raise KeyError(f"Missing DeepSeek-V4 MTP e/h projection pair for block {bid}") + + e_proj = LazyTorchTensor.to_eager(self.model_tensors[e_name]()) + h_proj = LazyTorchTensor.to_eager(self.model_tensors[h_name]()) + yield (f"layers.{bid}.nextn.eh_proj.weight", torch.cat((e_proj, h_proj), dim=1).contiguous()) + consumed.extend((e_name, h_name)) + for name in consumed: del self.model_tensors[name] @@ -737,6 +820,12 @@ def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_ "ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"), "ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"), "ffn.shared_experts.w3.weight": (gguf.MODEL_TENSOR.FFN_UP_SHEXP, ".weight"), + "nextn.eh_proj.weight": (gguf.MODEL_TENSOR.NEXTN_EH_PROJ, ".weight"), + "nextn.enorm.weight": (gguf.MODEL_TENSOR.NEXTN_ENORM, ".weight"), + "nextn.hnorm.weight": (gguf.MODEL_TENSOR.NEXTN_HNORM, ".weight"), + "nextn.shared_head_norm.weight": (gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ".weight"), + "nextn.embed_tokens.weight": (gguf.MODEL_TENSOR.NEXTN_EMBED_TOKENS, ".weight"), + "nextn.shared_head_head.weight": (gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, ".weight"), } tensor_name = match.group(2) @@ -759,10 +848,12 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)] def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: - del new_name, bid # unused + del bid # unused if name in self._dsv4_fp8_dequantized and n_dims >= 2: return gguf.GGMLQuantizationType.Q8_0 + if new_name.endswith(".nextn.eh_proj.weight"): + return gguf.GGMLQuantizationType.Q8_0 if name in self._dsv4_f32_tensors: return gguf.GGMLQuantizationType.F32 if name in self._dsv4_bf16_tensors and n_dims >= 2: @@ -770,6 +861,19 @@ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: return False + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) + + if not self.mtp_only or not from_dir: + return + + output_type: str = self.ftype.name.partition("_")[2] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, + self.metadata.version, size_label=None, output_type=output_type, model_type=None) + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + def prepare_tensors(self): super().prepare_tensors() self._is_mxfp4 = True diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index d86e614d8fd4..978231091c87 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -3331,6 +3331,12 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_GATE_SHEXP, MODEL_TENSOR.FFN_DOWN_SHEXP, MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.NEXTN_EH_PROJ, + MODEL_TENSOR.NEXTN_EMBED_TOKENS, + MODEL_TENSOR.NEXTN_ENORM, + MODEL_TENSOR.NEXTN_HNORM, + MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, + MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], MODEL_ARCH.ERNIE4_5_MOE: [ MODEL_TENSOR.TOKEN_EMBD, diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index e81ff647eee4..ea0ddd114c0c 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -968,6 +968,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) { case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_QWEN35: case LLM_ARCH_QWEN35MOE: + case LLM_ARCH_DEEPSEEK4: return true; default: return false; @@ -990,6 +991,7 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) { switch (arch) { case LLM_ARCH_QWEN35: case LLM_ARCH_QWEN35MOE: + case LLM_ARCH_DEEPSEEK4: return true; default: return false; diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 5ef7becf6f29..19cca7df1e9d 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -120,8 +120,9 @@ llama_context::llama_context( cparams.no_perf = params.no_perf; cparams.warmup = false; - cparams.embeddings_layer_inp.resize(hparams.n_layer(), false); - embd_layer_inp.resize(hparams.n_layer()); + // +1: id n_layer() taps the output of the last layer ("input" of the head) + cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false); + embd_layer_inp.resize(hparams.n_layer() + 1); cparams.ctx_type = params.ctx_type; cparams.pooling_type = params.pooling_type; @@ -1164,7 +1165,7 @@ void llama_context::set_embeddings_nextn(bool value, bool masked) { void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) { LLAMA_LOG_DEBUG("%s: lid = %d, enable = %d\n", __func__, lid, enable); - GGML_ASSERT(lid < model.hparams.n_layer()); + GGML_ASSERT(lid <= model.hparams.n_layer()); cparams.embeddings_layer_inp[lid] = enable; @@ -1716,7 +1717,8 @@ int llama_context::decode(const llama_batch & batch_inp) { const auto & hparams = model.hparams; const int64_t n_vocab = vocab.n_tokens(); - const int64_t n_embd = hparams.n_embd_inp(); + const bool mtp_embd = cparams.ctx_type == LLAMA_CONTEXT_TYPE_MTP && batch_inp.embd; + const int64_t n_embd = mtp_embd ? hparams.n_embd_out() : hparams.n_embd_inp(); // when computing embeddings, all tokens are output const bool output_all = cparams.embeddings; @@ -2275,8 +2277,9 @@ void llama_context::extract_layer_inputs(const llm_graph_result * res, size_t to } void llama_context::output_reorder() { - const uint64_t n_vocab = model.vocab.n_tokens(); - const uint64_t n_embd = model.hparams.n_embd; + const uint64_t n_vocab = model.vocab.n_tokens(); + const uint64_t n_embd = model.hparams.n_embd; + const uint64_t n_embd_out = model.hparams.n_embd_out(); for (size_t s = 0; s < output_swaps.size(); ++s) { const uint64_t i0 = output_swaps[s].i0; @@ -2289,14 +2292,14 @@ void llama_context::output_reorder() { } if (embd.size > 0) { - for (uint64_t k = 0; k < n_embd; k++) { - std::swap(embd.data[i0*n_embd + k], embd.data[i1*n_embd + k]); + for (uint64_t k = 0; k < n_embd_out; k++) { + std::swap(embd.data[i0*n_embd_out + k], embd.data[i1*n_embd_out + k]); } } if (embd_nextn.size > 0) { - for (uint64_t k = 0; k < n_embd; k++) { - std::swap(embd_nextn.data[i0*n_embd + k], embd_nextn.data[i1*n_embd + k]); + for (uint64_t k = 0; k < n_embd_out; k++) { + std::swap(embd_nextn.data[i0*n_embd_out + k], embd_nextn.data[i1*n_embd_out + k]); } } @@ -2351,6 +2354,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE || model.arch == LLM_ARCH_DEEPSEEK4 || + (model.arch == LLM_ARCH_DFLASH && model.hparams.dsv4_hc_mult > 0) || model.arch == LLM_ARCH_NANBEIGE || model.arch == LLM_ARCH_MINIMAX_M3) { return std::max(n_tokens * 40, 32u * model.n_tensors()); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 6d1c8f4e42a8..e12a8cdc2aac 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -619,6 +619,63 @@ bool llm_graph_input_attn_kv_iswa::can_reuse(const llm_graph_params & params) { return res; } +void llm_graph_input_attn_k_iswa::set_input(const llama_ubatch * ubatch) { + // base tensors may not be allocated if there are no non-SWA attention layers + if (self_k_idxs && self_k_idxs->buffer) { + mctx->get_base()->set_input_k_idxs(self_k_idxs, ubatch); + } + + // the kq mask guards on its own buffer: shared cells leave idxs unbacked while the mask stays live + if (self_kq_mask && self_kq_mask->buffer) { + mctx->get_base()->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn); + } + + // swa tensors may not be allocated if there are no SWA attention layers + if (self_k_idxs_swa && self_k_idxs_swa->buffer) { + mctx->get_swa()->set_input_k_idxs(self_k_idxs_swa, ubatch); + } + + if (self_kq_mask_swa && self_kq_mask_swa->buffer) { + mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn); + } + + if (self_k_rot && self_k_rot->buffer) { + mctx->get_base()->set_input_k_rot(self_k_rot); + } + + if (self_k_rot_swa && self_k_rot_swa->buffer) { + mctx->get_swa()->set_input_k_rot(self_k_rot_swa); + } +} + +bool llm_graph_input_attn_k_iswa::can_reuse(const llm_graph_params & params) { + const auto * mctx = static_cast(params.mctx); + + this->mctx = mctx; + + bool res = true; + + // base tensors may not be allocated if there are no non-SWA attention layers + if (self_k_idxs && self_k_idxs->buffer) { + res &= self_k_idxs->ne[0] == params.ubatch.n_tokens; + } + + if (self_kq_mask && self_kq_mask->buffer) { + res &= can_reuse_kq_mask(self_kq_mask, mctx->get_base(), params.ubatch, params.cparams); + } + + // swa tensors may not be allocated if there are no SWA attention layers + if (self_k_idxs_swa && self_k_idxs_swa->buffer) { + res &= self_k_idxs_swa->ne[0] == params.ubatch.n_tokens; + } + + if (self_kq_mask_swa && self_kq_mask_swa->buffer) { + res &= can_reuse_kq_mask(self_kq_mask_swa, mctx->get_swa(), params.ubatch, params.cparams); + } + + return res; +} + static void dsv4_set_i64(ggml_tensor * dst, const std::vector & src) { if (!dst || !dst->buffer) { return; @@ -754,6 +811,10 @@ static void dsv4_set_comp_inputs( dsv4_set_i32(inp.state_pos, plan.state_pos); dsv4_set_i32(inp.state_persist_src_idxs, plan.state_persist_src_idxs); dsv4_set_i32(inp.state_persist_dst_idxs, plan.state_persist_dst_idxs); + dsv4_set_i32(inp.state_restore_src_idxs, plan.state_restore_src_idxs); + dsv4_set_i32(inp.state_restore_dst_idxs, plan.state_restore_dst_idxs); + dsv4_set_i32(inp.state_snapshot_src_idxs, plan.state_snapshot_src_idxs); + dsv4_set_i32(inp.state_snapshot_dst_idxs, plan.state_snapshot_dst_idxs); dsv4_set_i32(inp.state_read_idxs, plan.state_read_idxs); dsv4_set_i64(inp.state_write_idxs, plan.state_write_idxs); dsv4_set_i32(inp.state_write_pos, plan.state_write_pos); @@ -798,6 +859,10 @@ static bool dsv4_can_reuse_comp_input( res &= dsv4_can_reuse_tensor_1d(inp.state_pos, plan.state_pos.size()); res &= dsv4_can_reuse_tensor_1d(inp.state_persist_src_idxs, plan.state_persist_src_idxs.size()); res &= dsv4_can_reuse_tensor_1d(inp.state_persist_dst_idxs, plan.state_persist_dst_idxs.size()); + res &= dsv4_can_reuse_tensor_1d(inp.state_restore_src_idxs, plan.state_restore_src_idxs.size()); + res &= dsv4_can_reuse_tensor_1d(inp.state_restore_dst_idxs, plan.state_restore_dst_idxs.size()); + res &= dsv4_can_reuse_tensor_1d(inp.state_snapshot_src_idxs, plan.state_snapshot_src_idxs.size()); + res &= dsv4_can_reuse_tensor_1d(inp.state_snapshot_dst_idxs, plan.state_snapshot_dst_idxs.size()); res &= dsv4_can_reuse_tensor_1d(inp.state_read_idxs, plan.state_read_idxs.size()); res &= dsv4_can_reuse_tensor_1d(inp.state_write_idxs, plan.state_write_idxs.size()); res &= dsv4_can_reuse_tensor_1d(inp.state_write_pos, plan.state_write_pos.size()); @@ -832,6 +897,10 @@ static void dsv4_build_comp_inputs( inp.state_pos = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_pos.size(), std::string("dsv4_") + name + "_state_pos"); inp.state_persist_src_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_persist_src_idxs.size(), std::string("dsv4_") + name + "_state_persist_src_idxs"); inp.state_persist_dst_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_persist_dst_idxs.size(), std::string("dsv4_") + name + "_state_persist_dst_idxs"); + inp.state_restore_src_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_restore_src_idxs.size(), std::string("dsv4_") + name + "_state_restore_src_idxs"); + inp.state_restore_dst_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_restore_dst_idxs.size(), std::string("dsv4_") + name + "_state_restore_dst_idxs"); + inp.state_snapshot_src_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_snapshot_src_idxs.size(), std::string("dsv4_") + name + "_state_snapshot_src_idxs"); + inp.state_snapshot_dst_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_snapshot_dst_idxs.size(), std::string("dsv4_") + name + "_state_snapshot_dst_idxs"); inp.state_read_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_read_idxs.size(), std::string("dsv4_") + name + "_state_read_idxs"); inp.state_write_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I64, plan.state_write_idxs.size(), std::string("dsv4_") + name + "_state_write_idxs"); inp.state_write_pos = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_write_pos.size(), std::string("dsv4_") + name + "_state_write_pos"); @@ -1195,7 +1264,7 @@ void llm_graph_result::reset() { t_embd_pooled = nullptr; t_h_nextn = nullptr; - t_layer_inp.resize(LLAMA_MAX_LAYERS); + t_layer_inp.resize(LLAMA_MAX_LAYERS + 1); std::fill(t_layer_inp.begin(), t_layer_inp.end(), nullptr); t_sampled.clear(); @@ -1650,7 +1719,7 @@ ggml_tensor * llm_graph_context::build_ffn( tmp = ggml_clamp(ctx0, tmp, -limit, limit); cb(tmp, "ffn_up_clamped", il); - if (arch == LLM_ARCH_DEEPSEEK4) { + if (arch == LLM_ARCH_DEEPSEEK4 || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) { cur = ggml_clamp(ctx0, cur, -INFINITY, limit); cb(cur, "ffn_gate_clamped", il); cur = ggml_swiglu_split(ctx0, cur, tmp); @@ -2045,7 +2114,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( up = ggml_clamp(ctx0, up, -limit, limit); cb(up, "ffn_moe_up_clamped", il); - if (arch == LLM_ARCH_DEEPSEEK4) { + if (arch == LLM_ARCH_DEEPSEEK4 || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) { cur = ggml_clamp(ctx0, cur, -INFINITY, limit); cb(cur, "ffn_moe_gate_clamped", il); cur = ggml_swiglu_split(ctx0, cur, up); @@ -2962,6 +3031,75 @@ ggml_tensor * llm_graph_context::build_attn( return cur; } +ggml_tensor * llm_graph_context::build_attn( + llm_graph_input_attn_k_iswa * inp, + ggml_tensor * wo, + ggml_tensor * wo_b, + ggml_tensor * wo_s, + ggml_tensor * q_cur, + ggml_tensor * k_cur, + ggml_tensor * v_cur, + ggml_tensor * kq_b, + ggml_tensor * sinks, + ggml_tensor * v_mla, + float kq_scale, + int il) const { + const bool is_swa = hparams.is_swa(il); + + GGML_UNUSED(v_cur); + + auto * k_rot = is_swa ? inp->self_k_rot_swa : inp->self_k_rot; + + if (k_rot) { + q_cur = llama_mul_mat_hadamard(ctx0, q_cur, k_rot); + if (k_cur) { + k_cur = llama_mul_mat_hadamard(ctx0, k_cur, k_rot); + } + } + + // these nodes are added to the graph together so that they are not reordered + // by doing so, the number of splits in the graph is reduced + ggml_build_forward_expand(gf, q_cur); + + if (k_cur) { + ggml_build_forward_expand(gf, k_cur); + } + + const auto * mctx_iswa = inp->mctx; + const auto * mctx_cur = is_swa ? mctx_iswa->get_swa() : mctx_iswa->get_base(); + + // optionally store to KV cache + if (k_cur) { + const auto & k_idxs = is_swa ? inp->get_k_idxs_swa() : inp->get_k_idxs(); + + ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il)); + } + + const auto & kq_mask = is_swa ? inp->get_kq_mask_swa() : inp->get_kq_mask(); + + // MLA-style attention: the cached K is used as V + ggml_tensor * q = q_cur; + ggml_tensor * k = mctx_cur->get_k(ctx0, il); + ggml_tensor * v = k; + + ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il); + cb(cur, "kqv_out", il); + + if (k_rot) { + cur = llama_mul_mat_hadamard(ctx0, cur, k_rot); + } + + if (wo) { + cur = build_lora_mm(wo, cur, wo_s); + } + + if (wo_b) { + cur = ggml_add(ctx0, cur, wo_b); + } + + return cur; +} + llm_graph_input_attn_cross * llm_graph_context::build_attn_inp_cross() const { auto inp = std::make_unique(cross); @@ -3084,6 +3222,34 @@ llm_graph_input_attn_kv_iswa * llm_graph_context::build_attn_inp_kv_iswa() const return (llm_graph_input_attn_kv_iswa *) res->add_input(std::move(inp)); } +llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const { + const auto * mctx_cur = static_cast(mctx); + + auto inp = std::make_unique(hparams, cparams, mctx_cur); + + { + inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch); + + inp->self_kq_mask = build_attn_inp_kq_mask(ctx0, mctx_cur->get_base(), ubatch, cparams); + inp->self_kq_mask_cnv = inp->self_kq_mask; + } + + { + GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE && "Use llama_kv_cache for non-SWA"); + + inp->self_k_idxs_swa = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch); + + inp->self_kq_mask_swa = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams); + inp->self_kq_mask_swa_cnv = inp->self_kq_mask_swa; + } + + inp->self_k_rot = mctx_cur->get_base()->build_input_k_rot(ctx0); + + inp->self_k_rot_swa = mctx_cur->get_swa()->build_input_k_rot(ctx0); + + return (llm_graph_input_attn_k_iswa *) res->add_input(std::move(inp)); +} + llm_graph_input_dsv4 * llm_graph_context::build_inp_dsv4() const { const auto * mctx_cur = static_cast(mctx); const auto * raw_ctx = mctx_cur->get_raw(); diff --git a/src/llama-graph.h b/src/llama-graph.h index 7ed490ce6728..160e29413552 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -471,6 +471,45 @@ class llm_graph_input_attn_kv_iswa : public llm_graph_input_i { const llama_kv_cache_iswa_context * mctx; }; +class llm_graph_input_attn_k_iswa : public llm_graph_input_i { +public: + llm_graph_input_attn_k_iswa( + const llama_hparams & hparams, + const llama_cparams & cparams, + const llama_kv_cache_iswa_context * mctx) : + hparams(hparams), + cparams(cparams), + mctx(mctx) { + } + ~llm_graph_input_attn_k_iswa() = default; + + void set_input(const llama_ubatch * ubatch) override; + + bool can_reuse(const llm_graph_params & params) override; + + ggml_tensor * get_k_idxs() const { return self_k_idxs; } + ggml_tensor * get_k_idxs_swa() const { return self_k_idxs_swa; } + + ggml_tensor * get_kq_mask() const { return self_kq_mask_cnv; } + ggml_tensor * get_kq_mask_swa() const { return self_kq_mask_swa_cnv; } + + ggml_tensor * self_k_idxs = nullptr; // I64 [n_batch] + ggml_tensor * self_k_idxs_swa = nullptr; // I64 [n_batch] + + ggml_tensor * self_kq_mask = nullptr; // F32/F16 [n_kv, n_batch/n_stream, 1, n_stream] + ggml_tensor * self_kq_mask_cnv = nullptr; // [n_kv, n_batch/n_stream, 1, n_stream] + ggml_tensor * self_kq_mask_swa = nullptr; // F32/F16 [n_kv, n_batch/n_stream, 1, n_stream] + ggml_tensor * self_kq_mask_swa_cnv = nullptr; // [n_kv, n_batch/n_stream, 1, n_stream] + + ggml_tensor * self_k_rot = nullptr; + ggml_tensor * self_k_rot_swa = nullptr; + + const llama_hparams hparams; + const llama_cparams cparams; + + const llama_kv_cache_iswa_context * mctx; +}; + // DSV4 raw graph inputs are SWA-only, but their mask may be stream-shaped // so raw K can be concatenated with DSV4 compressed K in one attention op. class llm_graph_input_dsv4_raw { @@ -505,6 +544,10 @@ class llm_graph_input_dsv4 : public llm_graph_input_i { ggml_tensor * state_pos = nullptr; // I32 [n_state] ggml_tensor * state_persist_src_idxs = nullptr; // I32 [n_state_persist] ggml_tensor * state_persist_dst_idxs = nullptr; // I32 [n_state_persist] + ggml_tensor * state_restore_src_idxs = nullptr; // I32 [n_state_restore] + ggml_tensor * state_restore_dst_idxs = nullptr; // I32 [n_state_restore] + ggml_tensor * state_snapshot_src_idxs = nullptr; // I32 [n_state_snapshot] + ggml_tensor * state_snapshot_dst_idxs = nullptr; // I32 [n_state_snapshot] ggml_tensor * state_read_idxs = nullptr; // I32 [ratio*n_state_write] ggml_tensor * state_write_idxs = nullptr; // I64 [n_state_write] ggml_tensor * state_write_pos = nullptr; // I32 [n_state_write] @@ -1068,7 +1111,7 @@ struct llm_graph_context { ggml_tensor * build_attn_mha( ggml_tensor * q, // [n_embd_head_q, n_head_q, n_tokens] ggml_tensor * k, // [n_embd_head_k, n_head_k, n_tokens] - ggml_tensor * v, // [n_embd_head_v, n_head_v, n_tokens] (v_trans == false) + ggml_tensor * v, // [n_embd_head_v, n_head_v, n_tokens] (v_trans = false) ggml_tensor * kq_b, ggml_tensor * kq_mask, ggml_tensor * sinks, // [n_head_q] @@ -1160,6 +1203,24 @@ struct llm_graph_context { float kq_scale, int il) const; + llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa() const; + + // note: if k_cur is not provided, it will not be stored in the memory + // note: the K cache is used as V (MLA-style attention) + ggml_tensor * build_attn( + llm_graph_input_attn_k_iswa * inp, + ggml_tensor * wo, + ggml_tensor * wo_b, + ggml_tensor * wo_s, + ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens] + ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] optional + ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] optional + ggml_tensor * kq_b, + ggml_tensor * sinks, // [n_head_q] + ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v] + float kq_scale, + int il) const; + llm_graph_input_attn_cross * build_attn_inp_cross() const; ggml_tensor * build_attn( diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index 069da45f4ea3..5caa05e8b07d 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -252,7 +252,8 @@ static void dsv4_state_write_tensor_streams( uint32_t tensor_rows, uint32_t n_rows, uint32_t s0, - uint32_t ns) { + uint32_t ns, + const std::vector * stream_ids = nullptr) { const int32_t type_i = (int32_t) tensor->type; const uint64_t ne0 = tensor->ne[0]; const uint64_t rows = n_rows; @@ -273,8 +274,16 @@ static void dsv4_state_write_tensor_streams( return; } + if (stream_ids && stream_ids->size() != ns) { + throw std::runtime_error("DSV4 state tensor stream map size mismatch"); + } + for (uint32_t s = 0; s < ns; ++s) { - const size_t offset = (size_t) (s0 + s)*stream_stride; + const uint32_t stream = stream_ids ? (*stream_ids)[s] : s0 + s; + if ((int64_t) stream >= tensor->ne[2]) { + throw std::runtime_error("DSV4 state tensor stream out of range"); + } + const size_t offset = (size_t) stream*stream_stride; io.write_tensor(tensor, offset, size); } } @@ -421,7 +430,9 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan( bool overlap, uint32_t state_size, uint32_t kv_size, - uint32_t n_stream) { + uint32_t n_stream, + uint32_t n_rs_seq, + const std::vector & rs_idx) { llama_kv_cache_dsv4_context::comp_plan plan; plan.n_visible.resize(ubatch.n_tokens); plan.n_stream = dsv4_comp_graph_n_stream(ubatch, n_stream); @@ -451,6 +462,7 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan( std::vector overlap_cur_reads; std::map, int64_t> curr_token_idx_map; + std::map state_write_counts; for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { for (int32_t s = 0; s < ubatch.n_seq_id[i]; ++s) { @@ -513,6 +525,7 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan( plan.state_write_idxs.push_back(cache_off + pos/ratio); plan.state_write_pos.push_back((int32_t) source_start); + ++state_write_counts[seq_id]; if (overlap) { const llama_pos prev_start = source_start - ratio; @@ -531,33 +544,57 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan( } } - if (ratio == DSV4_CSA_RATIO && plan.state_write_idxs.empty() && !plan.state_pos.empty()) { - // Non-boundary CSA steps still need a write op so their graph matches - // boundary steps. Use a padded scratch row that is masked from attention. + if (ratio == DSV4_CSA_RATIO && !plan.state_pos.empty()) { assert(kv_size > 0); - uint32_t i = 0; - while (i < ubatch.n_tokens && ubatch.pos[i] < 0) { - ++i; - } - assert(i < ubatch.n_tokens); + // Pad each stream to the reserve plan's block count. + const auto append_dummy_block = [&](llama_seq_id seq_id, uint32_t i) { + const int64_t cache_off = dsv4_stream_offset(n_stream, seq_id, kv_size); + const int32_t source_idx = state_source_idx(seq_id, ubatch.pos[i]); - const llama_pos pos = ubatch.pos[i]; - const llama_seq_id seq_id = ubatch.seq_id[i][0]; - const int64_t cache_off = dsv4_stream_offset(n_stream, seq_id, kv_size); - const int32_t source_idx = state_source_idx(seq_id, pos); + plan.state_write_idxs.push_back(cache_off + kv_size - 1); + plan.state_write_pos .push_back(0); - plan.state_write_idxs.push_back(cache_off + kv_size - 1); - plan.state_write_pos .push_back(0); + if (overlap) { + for (uint32_t j = 0; j < ratio; ++j) { + overlap_prev_reads.push_back(source_idx); + overlap_cur_reads .push_back(source_idx); + } + } else { + for (uint32_t j = 0; j < ratio; ++j) { + plan.state_read_idxs.push_back(source_idx); + } + } + }; - if (overlap) { - for (uint32_t j = 0; j < ratio; ++j) { - overlap_prev_reads.push_back(source_idx); - overlap_cur_reads .push_back(source_idx); + if (dsv4_ubatch_has_coupled(ubatch)) { + if (plan.state_write_idxs.empty()) { + uint32_t i = 0; + while (i < ubatch.n_tokens && ubatch.pos[i] < 0) { + ++i; + } + assert(i < ubatch.n_tokens); + append_dummy_block(ubatch.seq_id[i][0], i); } } else { - for (uint32_t j = 0; j < ratio; ++j) { - plan.state_read_idxs.push_back(source_idx); + const uint32_t n_blocks = (std::max(1, ubatch.n_seq_tokens) + ratio - 1)/ratio; + + for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) { + const llama_seq_id seq_id = ubatch.seq_id_unq[s]; + const uint32_t n_writes = state_write_counts[seq_id]; + if (n_writes >= n_blocks) { + continue; + } + if (n_writes + 1 != n_blocks) { + throw std::runtime_error("DSV4 CSA sequence positions are not contiguous"); + } + + uint32_t i = 0; + while (i < ubatch.n_tokens && (ubatch.pos[i] < 0 || !dsv4_token_has_seq(ubatch, i, seq_id))) { + ++i; + } + assert(i < ubatch.n_tokens); + append_dummy_block(seq_id, i); } } } @@ -583,6 +620,63 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan( plan.state_persist_dst_idxs.push_back(row.dst); } + + if (n_rs_seq > 0) { + for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) { + const llama_seq_id seq_id = ubatch.seq_id_unq[s]; + if (seq_id < 0 || (uint32_t) seq_id >= n_stream) { + continue; + } + + const int64_t stream_off = dsv4_stream_offset(n_stream, seq_id, state_size); + const uint32_t rollback = (uint32_t) seq_id < rs_idx.size() ? rs_idx[seq_id] : 0; + // Keep the restore graph fixed-width when no rollback is pending. + const int64_t src_plane = rollback > 0 && rollback <= n_rs_seq ? (int64_t) rollback*state_rows : 0; + for (uint32_t r = 0; r < state_size; ++r) { + plan.state_restore_src_idxs.push_back((int32_t) (src_plane + stream_off + r)); + plan.state_restore_dst_idxs.push_back((int32_t) (stream_off + r)); + } + + std::vector token_idxs; + token_idxs.reserve(ubatch.n_tokens); + for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { + if (dsv4_token_has_seq(ubatch, i, seq_id)) { + token_idxs.push_back(i); + } + } + if (token_idxs.empty()) { + continue; + } + + const uint32_t n_seq_tokens = (uint32_t) token_idxs.size(); + const int64_t scratch_off = (int64_t) state_rows*(1 + n_rs_seq); + for (uint32_t d = 1; d <= n_rs_seq; ++d) { + const int64_t dst_plane = (int64_t) d*state_rows; + + for (uint32_t r = 0; r < state_size; ++r) { + int32_t src; + if (d <= n_seq_tokens) { + const uint32_t prefix = n_seq_tokens - d; + src = (int32_t) (stream_off + r); + + for (uint32_t j = 0; j < prefix; ++j) { + const uint32_t i_tok = token_idxs[j]; + if (ubatch.pos[i_tok] >= 0 && (uint32_t) (ubatch.pos[i_tok]%state_size) == r) { + src = (int32_t) (scratch_off + i_tok); + } + } + } else { + const int64_t src_plane = (int64_t) (d - n_seq_tokens)*state_rows; + src = (int32_t) (src_plane + stream_off + r); + } + + plan.state_snapshot_src_idxs.push_back(src); + plan.state_snapshot_dst_idxs.push_back((int32_t) (dst_plane + stream_off + r)); + } + } + } + } + static const bool debug = []() { const char * env = getenv("LLAMA_DSV4_COMPRESS_DEBUG"); return env && atoi(env) > 0; @@ -604,12 +698,14 @@ static std::vector dsv4_build_comp_plans bool overlap, uint32_t state_size, uint32_t kv_size, - uint32_t n_stream) { + uint32_t n_stream, + uint32_t n_rs_seq, + const std::vector & rs_idx) { std::vector plans; plans.reserve(ubatches.size()); for (const llama_ubatch & ubatch : ubatches) { - plans.push_back(dsv4_build_comp_plan(ubatch, ratio, overlap, state_size, kv_size, n_stream)); + plans.push_back(dsv4_build_comp_plan(ubatch, ratio, overlap, state_size, kv_size, n_stream, n_rs_seq, rs_idx)); } return plans; @@ -696,7 +792,8 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_reserve_comp_plan( bool overlap, uint32_t state_size, uint32_t kv_size, - uint32_t n_stream) { + uint32_t n_stream, + uint32_t n_rs_seq) { llama_kv_cache_dsv4_context::comp_plan plan; plan.n_visible.resize(ubatch.n_tokens); plan.n_stream = dsv4_comp_graph_n_stream(ubatch, n_stream); @@ -714,10 +811,16 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_reserve_comp_plan( const uint64_t state_rows = (uint64_t) state_size*n_stream; const size_t n_persist = (size_t) std::min(ubatch.n_tokens, state_rows); + const size_t n_restore = n_rs_seq > 0 ? (size_t) state_size*std::max(1, ubatch.n_seqs_unq) : 0; + const size_t n_snapshot = (size_t) n_rs_seq*state_size*std::max(1, ubatch.n_seqs_unq); plan.state_pos .resize(ubatch.n_tokens); plan.state_persist_src_idxs.resize(n_persist); plan.state_persist_dst_idxs.resize(n_persist); + plan.state_restore_src_idxs.resize(n_restore); + plan.state_restore_dst_idxs.resize(n_restore); + plan.state_snapshot_src_idxs.resize(n_snapshot); + plan.state_snapshot_dst_idxs.resize(n_snapshot); plan.state_read_idxs .resize((overlap ? 2u : 1u)*ratio*n_blocks); plan.state_write_idxs.resize(n_blocks); plan.state_write_pos .resize(n_blocks); @@ -743,12 +846,14 @@ llama_dsv4_comp_state::llama_dsv4_comp_state( uint32_t ratio, uint32_t state_size, uint32_t n_embd_state, + uint32_t n_rs_seq, const char * name, const llama_memory_i::layer_filter_cb & filter) : ratio(ratio), state_size(state_size), n_embd_state(n_embd_state), - n_stream(unified ? 1 : n_seq_max) { + n_stream(unified ? 1 : n_seq_max), + n_rs_seq(n_rs_seq) { const llama_hparams & hparams = model.hparams; struct ggml_backend_buft_comparator { @@ -804,8 +909,9 @@ llama_dsv4_comp_state::llama_dsv4_comp_state( throw std::runtime_error("failed to create ggml context for DSV4 compressor state"); } - ggml_tensor * kv = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_state, state_size, n_stream); - ggml_tensor * score = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_state, state_size, n_stream); + const uint32_t n_planes = n_stream*(1 + n_rs_seq); + ggml_tensor * kv = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_state, state_size, n_planes); + ggml_tensor * score = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_state, state_size, n_planes); ggml_format_name(kv, "dsv4_%s_state_kv_l%d", name, il); ggml_format_name(score, "dsv4_%s_state_score_l%d", name, il); @@ -837,8 +943,8 @@ llama_dsv4_comp_state::llama_dsv4_comp_state( ctxs_bufs.emplace_back(std::move(ctx), buf); } - LLAMA_LOG_INFO("%s: %s ratio = %u, state = %u x %u, streams = %u, layers = %zu, size = %7.2f MiB\n", - __func__, name, ratio, state_size, n_embd_state, n_stream, layers.size(), total_size()/1024.0/1024.0); + LLAMA_LOG_INFO("%s: %s ratio = %u, state = %u x %u, streams = %u, rs_seq = %u, layers = %zu, size = %7.2f MiB\n", + __func__, name, ratio, state_size, n_embd_state, n_stream, n_rs_seq, layers.size(), total_size()/1024.0/1024.0); } void llama_dsv4_comp_state::clear(llama_seq_id seq_id, bool data) { @@ -848,9 +954,13 @@ void llama_dsv4_comp_state::clear(llama_seq_id seq_id, bool data) { if (seq_id >= 0) { GGML_ASSERT((uint32_t) seq_id < n_stream); + for (const auto & layer : layers) { - dsv4_clear_tensor_stream(layer.kv, (uint32_t) seq_id); - dsv4_clear_tensor_stream(layer.score, (uint32_t) seq_id); + for (uint32_t d = 0; d <= n_rs_seq; ++d) { + const uint32_t stream = d*n_stream + (uint32_t) seq_id; + dsv4_clear_tensor_stream(layer.kv, stream); + dsv4_clear_tensor_stream(layer.score, stream); + } } return; } @@ -868,6 +978,8 @@ void llama_dsv4_comp_state::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_ return; } + clear(seq_id_dst, true); + sc_info.ssrc.push_back((uint32_t) seq_id_src); sc_info.sdst.push_back((uint32_t) seq_id_dst); } @@ -896,6 +1008,14 @@ uint32_t llama_dsv4_comp_state::get_n_stream() const { return n_stream; } +uint32_t llama_dsv4_comp_state::get_n_rs_seq() const { + return n_rs_seq; +} + +uint32_t llama_dsv4_comp_state::get_n_rows() const { + return state_size*n_stream; +} + std::map llama_dsv4_comp_state::memory_breakdown() const { std::map ret; for (const auto & [_, buf] : ctxs_bufs) { @@ -905,13 +1025,26 @@ std::map llama_dsv4_comp_state::memory_break return ret; } -void llama_dsv4_comp_state::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const { +void llama_dsv4_comp_state::state_write( + llama_io_write_i & io, + llama_seq_id seq_id, + llama_state_seq_flags flags, + const std::vector & rs_idx) const { GGML_UNUSED(flags); uint32_t s0; uint32_t ns; dsv4_state_src_stream_range(n_stream, seq_id, s0, ns); + std::vector stream_ids(ns); + for (uint32_t s = 0; s < ns; ++s) { + const uint32_t seq = seq_id >= 0 ? (uint32_t) seq_id : s0 + s; + if (seq >= rs_idx.size() || rs_idx[seq] > n_rs_seq) { + throw std::runtime_error("DSV4 recurrent state rollback index out of range"); + } + stream_ids[s] = rs_idx[seq]*n_stream + s0 + s; + } + const uint32_t version = DSV4_COMP_STATE_VER; const uint32_t n_layer = layers.size(); @@ -925,8 +1058,8 @@ void llama_dsv4_comp_state::state_write(llama_io_write_i & io, llama_seq_id seq_ for (const auto & layer : layers) { io.write(&layer.il, sizeof(layer.il)); - dsv4_state_write_tensor_streams(io, layer.kv, state_size, state_size, s0, ns); - dsv4_state_write_tensor_streams(io, layer.score, state_size, state_size, s0, ns); + dsv4_state_write_tensor_streams(io, layer.kv, state_size, state_size, s0, ns, &stream_ids); + dsv4_state_write_tensor_streams(io, layer.score, state_size, state_size, s0, ns, &stream_ids); } } @@ -972,28 +1105,40 @@ void llama_dsv4_comp_state::state_read(llama_io_read_i & io, llama_seq_id seq_id } } -ggml_tensor * llama_dsv4_comp_state::get_kv(ggml_context * ctx, int32_t il) const { +ggml_tensor * llama_dsv4_comp_state::get_kv_all(ggml_context * ctx, int32_t il) const { const int32_t ids = map_layer_ids.at(il); - ggml_tensor * state = layers[ids].kv; - return ggml_reshape_2d(ctx, state, state->ne[0], state->ne[1]*state->ne[2]); + return ggml_view_2d(ctx, state, state->ne[0], get_n_rows()*(1 + n_rs_seq), state->nb[1], 0); } -ggml_tensor * llama_dsv4_comp_state::get_score(ggml_context * ctx, int32_t il) const { +ggml_tensor * llama_dsv4_comp_state::get_score_all(ggml_context * ctx, int32_t il) const { const int32_t ids = map_layer_ids.at(il); - ggml_tensor * state = layers[ids].score; - return ggml_reshape_2d(ctx, state, state->ne[0], state->ne[1]*state->ne[2]); + return ggml_view_2d(ctx, state, state->ne[0], get_n_rows()*(1 + n_rs_seq), state->nb[1], 0); +} + +ggml_tensor * llama_dsv4_comp_state::get_kv(ggml_context * ctx, int32_t il) const { + ggml_tensor * state = get_kv_all(ctx, il); + const size_t row_size = ggml_row_size(state->type, state->ne[0]); + + return ggml_view_2d(ctx, state, state->ne[0], get_n_rows(), state->nb[1], 0*row_size); +} + +ggml_tensor * llama_dsv4_comp_state::get_score(ggml_context * ctx, int32_t il) const { + ggml_tensor * state = get_score_all(ctx, il); + const size_t row_size = ggml_row_size(state->type, state->ne[0]); + + return ggml_view_2d(ctx, state, state->ne[0], get_n_rows(), state->nb[1], 0*row_size); } ggml_tensor * llama_dsv4_comp_state::cpy_kv(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const { - return ggml_set_rows(ctx, get_kv(ctx, il), cur, idxs); + return ggml_set_rows(ctx, get_kv_all(ctx, il), cur, idxs); } ggml_tensor * llama_dsv4_comp_state::cpy_score(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const { - return ggml_set_rows(ctx, get_score(ctx, il), cur, idxs); + return ggml_set_rows(ctx, get_score_all(ctx, il), cur, idxs); } size_t llama_dsv4_comp_state::total_size() const { @@ -1022,13 +1167,16 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( uint32_t n_seq_max, uint32_t n_ubatch, uint32_t n_pad, + uint32_t n_rs_seq, const layer_filter_cb & filter, const layer_reuse_cb & reuse) : hparams_raw(model.hparams), hparams_csa(model.hparams), hparams_hca(model.hparams), hparams_lid(model.hparams), - n_seq_max(n_seq_max) { + n_seq_max(n_seq_max), + n_rs_seq(n_rs_seq), + rs_idx(n_seq_max, 0) { const layer_filter_cb filter_raw = [&](int32_t il) { if (filter && !filter(il)) { @@ -1043,6 +1191,11 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( // Keep DSV4 KV/state streams per sequence even when public KV mode is unified. const bool unified_raw = false; + hparams_raw.n_layer_nextn = 0; + hparams_csa.n_layer_nextn = 0; + hparams_hca.n_layer_nextn = 0; + hparams_lid.n_layer_nextn = 0; + LLAMA_LOG_INFO("%s: creating DSV4 raw KV cache\n", __func__); dsv4_make_k_only(hparams_raw); @@ -1109,19 +1262,19 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4( csa_state = std::make_unique( model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO, - 2*model.hparams.n_embd_head_k(), "csa", filter_csa); + 2*model.hparams.n_embd_head_k(), n_rs_seq, "csa", filter_csa); LLAMA_LOG_INFO("%s: creating DSV4 HCA compressor state\n", __func__); hca_state = std::make_unique( model, offload, unified_compressed, n_seq_max, DSV4_HCA_RATIO, DSV4_HCA_RATIO, - model.hparams.n_embd_head_k(), "hca", filter_hca); + model.hparams.n_embd_head_k(), n_rs_seq, "hca", filter_hca); LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer compressor state\n", __func__); lid_state = std::make_unique( model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO, - 2*model.hparams.indexer_head_size, "lid", filter_csa); + 2*model.hparams.indexer_head_size, n_rs_seq, "lid", filter_csa); // DSV4 attention reads compressed-K / compressor-state rows that the current // graph does not necessarily overwrite; uninitialized buffer contents would @@ -1255,17 +1408,35 @@ bool llama_kv_cache_dsv4::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1 } if (p0 > 0) { - if (seq_id < 0 || (uint32_t) seq_id >= n_seq_max || - p0 <= kv_raw->seq_pos_max(seq_id)) { + if (seq_id < 0 || (uint32_t) seq_id >= n_seq_max) { + return false; + } + + const llama_pos pos_max = kv_raw->seq_pos_max(seq_id); + if (p0 > pos_max) { + bool res = true; + + res = res & kv_raw->seq_rm(seq_id, p0, -1); + res = res & kv_csa->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1); + res = res & kv_hca->seq_rm(seq_id, p0/DSV4_HCA_RATIO, -1); + res = res & kv_lid->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1); + + return res; + } + + if (n_rs_seq == 0) { return false; } - bool res = true; + const llama_pos rollback = pos_max - (p0 - 1); + if (rollback < 1 || rollback > (llama_pos) n_rs_seq) { + return false; + } - res = res & kv_raw->seq_rm(seq_id, p0, -1); - res = res & kv_csa->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1); - res = res & kv_hca->seq_rm(seq_id, p0/DSV4_HCA_RATIO, -1); - res = res & kv_lid->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1); + const bool res = kv_raw->seq_rm(seq_id, p0, p1); + if (res) { + rs_idx[seq_id] = (uint32_t) rollback; + } return res; } @@ -1290,6 +1461,10 @@ void llama_kv_cache_dsv4::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_ds csa_state->seq_cp(seq_id_src, seq_id_dst); hca_state->seq_cp(seq_id_src, seq_id_dst); lid_state->seq_cp(seq_id_src, seq_id_dst); + + if (seq_id_src != seq_id_dst) { + rs_idx[seq_id_dst] = 0; + } } void llama_kv_cache_dsv4::seq_keep(llama_seq_id seq_id) { @@ -1386,9 +1561,9 @@ void llama_kv_cache_dsv4::state_write(llama_io_write_i & io, llama_seq_id seq_id dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags, n_rows_lid); } - csa_state->state_write(io, seq_id, flags); - hca_state->state_write(io, seq_id, flags); - lid_state->state_write(io, seq_id, flags); + csa_state->state_write(io, seq_id, flags, rs_idx); + hca_state->state_write(io, seq_id, flags, rs_idx); + lid_state->state_write(io, seq_id, flags, rs_idx); } void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { @@ -1432,6 +1607,12 @@ void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id, hca_state->state_read(io, seq_id, flags); lid_state->state_read(io, seq_id, flags); + if (seq_id >= 0) { + GGML_ASSERT((uint32_t) seq_id < n_seq_max); + rs_idx[seq_id] = 0; + } else { + std::fill(rs_idx.begin(), rs_idx.end(), 0); + } } llama_kv_cache_iswa * llama_kv_cache_dsv4::get_raw() const { @@ -1462,6 +1643,31 @@ llama_dsv4_comp_state * llama_kv_cache_dsv4::get_lid_state() const { return lid_state.get(); } +uint32_t llama_kv_cache_dsv4::get_n_rs_seq() const { + return n_rs_seq; +} + +const std::vector & llama_kv_cache_dsv4::get_rs_idx() const { + return rs_idx; +} + +void llama_kv_cache_dsv4::reset_rs_idx_for_ubatches(const std::vector & ubatches) { + if (n_rs_seq == 0) { + return; + } + + for (const llama_ubatch & ubatch : ubatches) { + for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { + for (int32_t s = 0; s < ubatch.n_seq_id[i]; ++s) { + const llama_seq_id seq_id = ubatch.seq_id[i][s]; + if (seq_id >= 0 && (uint32_t) seq_id < n_seq_max) { + rs_idx[seq_id] = 0; + } + } + } + } +} + void llama_kv_cache_dsv4::clear_compressed(llama_seq_id seq_id, bool data) { if (seq_id < 0) { kv_csa->clear(data); @@ -1488,6 +1694,12 @@ void llama_kv_cache_dsv4::clear_compressed(llama_seq_id seq_id, bool data) { csa_state->clear(seq_id, data); hca_state->clear(seq_id, data); lid_state->clear(seq_id, data); + + if (seq_id >= 0) { + rs_idx[seq_id] = 0; + } else { + std::fill(rs_idx.begin(), rs_idx.end(), 0); + } } // @@ -1779,10 +1991,14 @@ llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context( std::vector ubatches_raw) : ubatches(std::move(ubatches)), plans_csa(dsv4_build_comp_plans(this->ubatches, DSV4_CSA_RATIO, true, - kv->get_csa_state()->get_state_size(), kv->get_csa()->get_size(), kv->get_csa_state()->get_n_stream())), + kv->get_csa_state()->get_state_size(), kv->get_csa()->get_size(), kv->get_csa_state()->get_n_stream(), + kv->get_n_rs_seq(), kv->get_rs_idx())), plans_hca(dsv4_build_comp_plans(this->ubatches, DSV4_HCA_RATIO, false, - kv->get_hca_state()->get_state_size(), kv->get_hca()->get_size(), kv->get_hca_state()->get_n_stream())), - plans_lid(plans_csa), + kv->get_hca_state()->get_state_size(), kv->get_hca()->get_size(), kv->get_hca_state()->get_n_stream(), + kv->get_n_rs_seq(), kv->get_rs_idx())), + plans_lid(dsv4_build_comp_plans(this->ubatches, DSV4_CSA_RATIO, true, + kv->get_lid_state()->get_state_size(), kv->get_lid()->get_size(), kv->get_lid_state()->get_n_stream(), + kv->get_n_rs_seq(), kv->get_rs_idx())), ctx_raw(std::make_unique( kv->get_raw(), std::move(sinfos_raw_base_write), @@ -1809,6 +2025,7 @@ llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context( hca_state(kv->get_hca_state()), lid_state(kv->get_lid_state()), status(ctx_raw->get_status()) { + kv->reset_rs_idx_for_ubatches(this->ubatches); } llama_kv_cache_dsv4_context::~llama_kv_cache_dsv4_context() = default; @@ -1944,7 +2161,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_ reserve_plan_csa = dsv4_build_reserve_comp_plan( ubatch, DSV4_CSA_RATIO, true, - csa_state->get_state_size(), get_csa()->get_n_kv(), csa_state->get_n_stream()); + csa_state->get_state_size(), get_csa()->get_n_kv(), csa_state->get_n_stream(), csa_state->get_n_rs_seq()); return reserve_plan_csa; } @@ -1958,7 +2175,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_ reserve_plan_hca = dsv4_build_reserve_comp_plan( ubatch, DSV4_HCA_RATIO, false, - hca_state->get_state_size(), get_hca()->get_n_kv(), hca_state->get_n_stream()); + hca_state->get_state_size(), get_hca()->get_n_kv(), hca_state->get_n_stream(), hca_state->get_n_rs_seq()); return reserve_plan_hca; } @@ -1972,7 +2189,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_ reserve_plan_lid = dsv4_build_reserve_comp_plan( ubatch, DSV4_CSA_RATIO, true, - lid_state->get_state_size(), get_lid()->get_n_kv(), lid_state->get_n_stream()); + lid_state->get_state_size(), get_lid()->get_n_kv(), lid_state->get_n_stream(), lid_state->get_n_rs_seq()); return reserve_plan_lid; } diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h index 76b1daf57871..ce39867c0342 100644 --- a/src/llama-kv-cache-dsv4.h +++ b/src/llama-kv-cache-dsv4.h @@ -22,6 +22,7 @@ class llama_dsv4_comp_state { uint32_t ratio, uint32_t state_size, uint32_t n_embd_state, + uint32_t n_rs_seq, const char * name, const llama_memory_i::layer_filter_cb & filter); @@ -29,17 +30,21 @@ class llama_dsv4_comp_state { void seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst); void apply_copies(const stream_copy_info & sc_info) const; - uint32_t get_ratio() const; + uint32_t get_ratio() const; uint32_t get_state_size() const; - uint32_t get_n_stream() const; + uint32_t get_n_stream() const; + uint32_t get_n_rs_seq() const; + uint32_t get_n_rows() const; std::map memory_breakdown() const; - void state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const; + void state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags, const std::vector & rs_idx) const; void state_read (llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags); - ggml_tensor * get_kv (ggml_context * ctx, int32_t il) const; - ggml_tensor * get_score(ggml_context * ctx, int32_t il) const; + ggml_tensor * get_kv (ggml_context * ctx, int32_t il) const; + ggml_tensor * get_score (ggml_context * ctx, int32_t il) const; + ggml_tensor * get_kv_all (ggml_context * ctx, int32_t il) const; + ggml_tensor * get_score_all(ggml_context * ctx, int32_t il) const; ggml_tensor * cpy_kv (ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const; ggml_tensor * cpy_score(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const; @@ -59,6 +64,7 @@ class llama_dsv4_comp_state { const uint32_t state_size; const uint32_t n_embd_state; const uint32_t n_stream; + const uint32_t n_rs_seq; std::vector> ctxs_bufs; @@ -93,6 +99,7 @@ class llama_kv_cache_dsv4 : public llama_memory_i { uint32_t n_seq_max, uint32_t n_ubatch, uint32_t n_pad, + uint32_t n_rs_seq, const layer_filter_cb & filter, const layer_reuse_cb & reuse); @@ -141,6 +148,10 @@ class llama_kv_cache_dsv4 : public llama_memory_i { llama_dsv4_comp_state * get_hca_state() const; llama_dsv4_comp_state * get_lid_state() const; + uint32_t get_n_rs_seq() const; + const std::vector & get_rs_idx() const; + void reset_rs_idx_for_ubatches(const std::vector & ubatches); + private: llama_hparams hparams_raw; llama_hparams hparams_csa; @@ -148,6 +159,9 @@ class llama_kv_cache_dsv4 : public llama_memory_i { llama_hparams hparams_lid; const uint32_t n_seq_max; + const uint32_t n_rs_seq; + + std::vector rs_idx; std::unique_ptr kv_raw; std::unique_ptr kv_csa; @@ -268,6 +282,17 @@ class llama_kv_cache_dsv4_context : public llama_memory_context_i { std::vector state_persist_src_idxs; std::vector state_persist_dst_idxs; + // Device-side rollback restore copies snapshot planes back to the + // current compressor-state plane before the graph reads it. + std::vector state_restore_src_idxs; + std::vector state_restore_dst_idxs; + + // Device-side rollback snapshots copy rows from the graph-local + // [persistent_state | current_ubatch_scratch] tensor into rollback + // planes after the graph has computed current-token compressor state. + std::vector state_snapshot_src_idxs; + std::vector state_snapshot_dst_idxs; + // Flattened source row ids used for state-backed commits. Source rows // index the graph-local [persistent_state | current_ubatch_scratch] // tensor. For overlapped compression the first half is previous rows diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 91c2fc9c008d..e93641b63ddc 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2138,6 +2138,75 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, nullptr); } } break; + case LLM_ARCH_DEEPSEEK4: + { + GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); + + if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { + const llama_memory_i::layer_filter_cb filter_mtp = [&](int32_t il) { + return il >= (int32_t) hparams.n_layer(); + }; + + res = new llama_kv_cache_iswa( + *this, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + params.swa_full, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + cparams.n_ubatch, + 1, + nullptr, + filter_mtp, + nullptr, + nullptr); + } else { + res = new llama_kv_cache_dsv4( + *this, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + params.swa_full, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + cparams.n_ubatch, + 1, + cparams.n_rs_seq, + nullptr, + nullptr); + } + } break; + case LLM_ARCH_DFLASH: + { + // DSV4 DSpark stages store a single MLA-style K per position (window = the draft ring) + if (hparams.dsv4_hc_mult > 0) { + GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); + + res = new llama_kv_cache_iswa( + *this, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + params.swa_full, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + cparams.n_ubatch, + 1, + nullptr, + nullptr, + nullptr, + nullptr); + break; + } + } + [[fallthrough]]; // Models that need standard caching should rely on recurrent/hybrid // checks default: @@ -2253,24 +2322,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, } } - if (arch == LLM_ARCH_DEEPSEEK4) { - GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); - - res = new llama_kv_cache_dsv4( - *this, - params.type_k, - params.type_v, - !cparams.flash_attn, - cparams.offload_kqv, - params.swa_full, - cparams.kv_unified, - cparams.n_ctx_seq, - cparams.n_seq_max, - cparams.n_ubatch, - 1, - filter, - reuse); - } else if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { + if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { GGML_ASSERT(hparams.is_swa_any()); if (arch == LLM_ARCH_GEMMA4_ASSISTANT) { @@ -2619,9 +2671,12 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_STEP35: case LLM_ARCH_TALKIE: case LLM_ARCH_MELLUM: - case LLM_ARCH_DFLASH: return LLAMA_ROPE_TYPE_NEOX; + case LLM_ARCH_DFLASH: + // DSV4 DSpark drafters use DeepSeek-V4's normal RoPE; legacy DFlash backbones are NeoX + return model->hparams.dsv4_hc_mult > 0 ? LLAMA_ROPE_TYPE_NORM : LLAMA_ROPE_TYPE_NEOX; + case LLM_ARCH_QWEN2VL: case LLM_ARCH_PADDLEOCR: return LLAMA_ROPE_TYPE_MROPE; diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 2d41dace0b28..e68dc49b6dff 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -16,6 +16,16 @@ static float dsv4_rope_attn_factor(float freq_scale, float ext_factor) { } void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false); + if (hparams.n_layer_nextn > 0 && hparams.n_layer_nextn < hparams.n_layer_all) { + const uint32_t n_layer_main = hparams.n_layer_all - hparams.n_layer_nextn; + const std::string mtp_probe = "blk." + std::to_string(n_layer_main) + ".nextn.eh_proj.weight"; + if (ml.get_weight(mtp_probe.c_str()) == nullptr) { + hparams.n_layer_nextn = 0; + } + } + GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < block_count"); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); @@ -24,8 +34,8 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm); - ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer()); - if (!ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer(), 0)) { + ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all); + if (!ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, 0)) { hparams.swiglu_clamp_shexp = hparams.swiglu_clamp_exp; } @@ -41,9 +51,11 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps); ml.get_key(LLM_KV_HASH_LAYER_COUNT, hparams.dsv4_hash_layer_count); + hparams.n_embd_out_impl = hparams.dsv4_hc_mult * hparams.n_embd; + uint32_t n_compress_ratios = 0; ml.get_arr_n(LLM_KV_ATTENTION_COMPRESS_RATIOS, n_compress_ratios); - if (n_compress_ratios < hparams.n_layer()) { + if (n_compress_ratios < hparams.n_layer_all) { throw std::runtime_error("DeepSeek-V4 compress_ratios is shorter than block_count"); } ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios); @@ -54,6 +66,9 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) { } hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.set_swa_pattern(0); + for (uint32_t il = hparams.n_layer(); il < hparams.n_layer_all; ++il) { + hparams.is_swa_impl[il] = true; + } switch (hparams.n_layer()) { case 43: type = LLM_TYPE_UNKNOWN; break; @@ -61,7 +76,7 @@ void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) { } } -void llama_model_deepseek4::load_arch_tensors(llama_model_loader &) { +void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) { LLAMA_LOAD_LOCALS; const int64_t q_lora_rank = hparams.n_lora_q; @@ -75,6 +90,10 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader &) { const int64_t hc_dim = hc_mult * n_embd; const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult; + const bool mtp_only = (n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); + const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + const int mtp_flags = ml.load_mtp ? 0 : TENSOR_SKIP; + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); @@ -84,69 +103,82 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader &) { hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0); hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0); - for (int i = 0; i < n_layer; ++i) { + for (int i = 0; i < n_layer_all; ++i) { auto & layer = layers[i]; - - layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); - layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0); - layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0); - layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0); - layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, 0); - layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, 0); - layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, 0); - layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, 0); - layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, 0); - - layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0); - layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, 0); - layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, 0); - layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0); - layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, 0); - layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, 0); + const int flags = i < n_layer ? trunk_flags : mtp_flags; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags); + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, flags); + layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, flags); + layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, flags); + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, flags); + layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, flags); + layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, flags); + layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, flags); + layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, flags); + + layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, flags); + layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, flags); + layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, flags); + layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc_dim, hc_mix_dim}, flags); + layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, flags); + layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, flags); const int64_t ratio = hparams.dsv4_compress_ratios[i]; if (ratio != 0) { const int64_t coff = ratio == 4 ? 2 : 1; - layer.attn_comp_wkv = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WKV, "weight", i), {n_embd, coff * n_embd_head}, 0); - layer.attn_comp_wgate = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WGATE, "weight", i), {n_embd, coff * n_embd_head}, 0); - layer.attn_comp_ape = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_APE, "weight", i), {coff * n_embd_head, ratio}, 0); - layer.attn_comp_norm = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_NORM, "weight", i), {n_embd_head}, 0); + layer.attn_comp_wkv = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WKV, "weight", i), {n_embd, coff * n_embd_head}, flags); + layer.attn_comp_wgate = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WGATE, "weight", i), {n_embd, coff * n_embd_head}, flags); + layer.attn_comp_ape = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_APE, "weight", i), {coff * n_embd_head, ratio}, flags); + layer.attn_comp_norm = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_NORM, "weight", i), {n_embd_head}, flags); if (ratio == 4) { const int64_t n_embd_indexer = hparams.indexer_head_size; - layer.indexer_proj = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ, "weight", i), {n_embd, hparams.indexer_n_head}, 0); - layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, hparams.indexer_n_head * n_embd_indexer}, 0); + layer.indexer_proj = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ, "weight", i), {n_embd, hparams.indexer_n_head}, flags); + layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, hparams.indexer_n_head * n_embd_indexer}, flags); - layer.indexer_comp_wkv = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_WKV, "weight", i), {n_embd, 2 * n_embd_indexer}, 0); - layer.indexer_comp_wgate = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_WGATE, "weight", i), {n_embd, 2 * n_embd_indexer}, 0); - layer.indexer_comp_ape = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_APE, "weight", i), {2 * n_embd_indexer, ratio}, 0); - layer.indexer_comp_norm = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_NORM, "weight", i), {n_embd_indexer}, 0); + layer.indexer_comp_wkv = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_WKV, "weight", i), {n_embd, 2 * n_embd_indexer}, flags); + layer.indexer_comp_wgate = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_WGATE, "weight", i), {n_embd, 2 * n_embd_indexer}, flags); + layer.indexer_comp_ape = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_APE, "weight", i), {2 * n_embd_indexer, ratio}, flags); + layer.indexer_comp_norm = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_NORM, "weight", i), {n_embd_indexer}, flags); } else if (ratio != 128) { throw std::runtime_error("DeepSeek-V4 loader only supports compression ratios 0, 4, and 128"); } } - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, flags); if ((uint32_t) i < hparams.dsv4_hash_layer_count) { - layer.ffn_gate_tid2eid = create_tensor(tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i), {n_expert_used, n_vocab}, 0); + layer.ffn_gate_tid2eid = create_tensor(tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i), {n_expert_used, n_vocab}, flags); } else { - layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, flags); + } + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags); + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, flags); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, flags); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd }, flags); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); + + if (i >= n_layer) { + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED | flags); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED | flags); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED | flags); } - layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); - - layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); - - layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); - layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd }, 0); - layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); } } std::unique_ptr llama_model_deepseek4::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique(*this, params); + } return std::make_unique(*this, params); } @@ -175,18 +207,69 @@ static ggml_tensor * dsv4_append_zero_row(ggml_context * ctx, ggml_tensor * t, b return ggml_concat(ctx, t, row, 1); } -static ggml_tensor * dsv4_with_zero_dep(ggml_context * ctx, ggml_tensor * t, ggml_tensor * dep) { - if (dep == nullptr) { - return t; +struct dsv4_state_tensors { + ggml_tensor * kv; + ggml_tensor * score; +}; + +static dsv4_state_tensors dsv4_build_state_restore( + ggml_context * ctx, + const llm_graph_input_dsv4::comp_input & inp, + const llama_dsv4_comp_state * state, + int32_t il) { + dsv4_state_tensors restored = { + state->get_kv_all(ctx, il), + state->get_score_all(ctx, il), + }; + + if (inp.state_restore_src_idxs == nullptr || inp.state_restore_dst_idxs == nullptr) { + return restored; + } + + ggml_tensor * kv_rows = ggml_get_rows(ctx, restored.kv, inp.state_restore_src_idxs); + restored.kv = state->cpy_kv(ctx, kv_rows, inp.state_restore_dst_idxs, il); + + ggml_tensor * score_rows = ggml_get_rows(ctx, restored.score, inp.state_restore_src_idxs); + restored.score = state->cpy_score(ctx, score_rows, inp.state_restore_dst_idxs, il); + + return restored; +} + +static dsv4_state_tensors dsv4_build_state_snapshot( + ggml_context * ctx, + const llm_graph_input_dsv4::comp_input & inp, + const llama_dsv4_comp_state * state, + ggml_tensor * source_kv, + ggml_tensor * source_score, + int32_t il) { + if (inp.state_snapshot_src_idxs == nullptr || inp.state_snapshot_dst_idxs == nullptr || + source_kv == nullptr || source_score == nullptr) { + return {}; } - ggml_tensor * zero = ggml_scale(ctx, ggml_sum(ctx, dep), 0.0f); - return ggml_add(ctx, t, zero); + ggml_tensor * kv_rows = ggml_get_rows(ctx, source_kv, inp.state_snapshot_src_idxs); + ggml_tensor * kv = state->cpy_kv(ctx, kv_rows, inp.state_snapshot_dst_idxs, il); + + ggml_tensor * score_rows = ggml_get_rows(ctx, source_score, inp.state_snapshot_src_idxs); + ggml_tensor * score = state->cpy_score(ctx, score_rows, inp.state_snapshot_dst_idxs, il); + + return { kv, score }; } static constexpr int64_t DSV4_CSA_RATIO = 4; static constexpr int64_t DSV4_HCA_RATIO = 128; +// mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens] +static ggml_tensor * dsv4_hc_mean(ggml_context * ctx, ggml_tensor * x) { + const int64_t hc = x->ne[1]; + + ggml_tensor * acc = ggml_view_2d(ctx, x, x->ne[0], x->ne[2], x->nb[2], 0); + for (int64_t s = 1; s < hc; ++s) { + acc = ggml_add(ctx, acc, ggml_view_2d(ctx, x, x->ne[0], x->ne[2], x->nb[2], s*x->nb[1])); + } + return ggml_scale(ctx, acc, 1.0f/hc); +} + static ggml_tensor * dsv4_hc_affine( ggml_context * ctx, ggml_tensor * x, @@ -804,8 +887,29 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention( ggml_tensor * cur, ggml_tensor * inp_pos, int il) const { + return build_attention_impl(model, inp_dsv4, nullptr, cur, inp_pos, il); +} + +ggml_tensor * llama_model_deepseek4::graph::build_attention( + const llama_model & model, + llm_graph_input_attn_k_iswa * inp_mtp, + ggml_tensor * cur, + ggml_tensor * inp_pos, + int il) const { + return build_attention_impl(model, nullptr, inp_mtp, cur, inp_pos, il); +} + +ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( + const llama_model & model, + llm_graph_input_dsv4 * inp_dsv4, + llm_graph_input_attn_k_iswa * inp_mtp, + ggml_tensor * cur, + ggml_tensor * inp_pos, + int il) const { + GGML_ASSERT((inp_dsv4 == nullptr) != (inp_mtp == nullptr)); + const auto & layer = model.layers[il]; - llm_graph_input_dsv4_raw * inp_attn = inp_dsv4->get_raw(); + llm_graph_input_dsv4_raw * inp_attn = inp_dsv4 ? inp_dsv4->get_raw() : nullptr; const int64_t n_embd_head = hparams.n_embd_head_k(); const int64_t n_embd_head_rope = hparams.n_rot(); @@ -873,9 +977,12 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention( cb(kv, "kv", il); const int64_t ratio = hparams.dsv4_compress_ratios[il]; + GGML_ASSERT(inp_dsv4 || ratio == 0); ggml_tensor * hca_state_kv = nullptr; ggml_tensor * hca_state_score = nullptr; + ggml_tensor * hca_source_kv = nullptr; + ggml_tensor * hca_source_score = nullptr; if (ratio == DSV4_HCA_RATIO && inp_dsv4->get_hca().state_pos) { hca_state_kv = build_lora_mm(layer.attn_comp_wkv, cur); cb(hca_state_kv, "hca_state_kv", il); @@ -906,10 +1013,16 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention( GGML_ASSERT(inp_dsv4->get_csa().state_write_idxs); - ggml_tensor * csa_source_kv = ggml_concat(ctx0, - inp_dsv4->mctx->get_csa_state()->get_kv(ctx0, il), csa_state_kv, 1); - ggml_tensor * csa_source_score = ggml_concat(ctx0, - inp_dsv4->mctx->get_csa_state()->get_score(ctx0, il), csa_state_score, 1); + const auto * csa_state = inp_dsv4->mctx->get_csa_state(); + const dsv4_state_tensors csa_restored = dsv4_build_state_restore( + ctx0, inp_dsv4->get_csa(), csa_state, il); + ggml_tensor * csa_base_kv = dsv4_view_2d( + ctx0, csa_restored.kv, csa_restored.kv->ne[0], csa_state->get_n_rows(), 0); + ggml_tensor * csa_base_score = dsv4_view_2d( + ctx0, csa_restored.score, csa_restored.score->ne[0], csa_state->get_n_rows(), 0); + + ggml_tensor * csa_source_kv = ggml_concat(ctx0, csa_base_kv, csa_state_kv, 1); + ggml_tensor * csa_source_score = ggml_concat(ctx0, csa_base_score, csa_state_score, 1); ggml_tensor * kv_comp_csa_state = build_overlap_compressed_kv_from_state( csa_source_kv, @@ -930,8 +1043,19 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention( ggml_build_forward_expand(gf, inp_dsv4->mctx->get_csa()->cpy_k(ctx0, kv_comp_csa_state, inp_dsv4->get_csa().state_write_idxs, il)); - csa_state_kv = dsv4_with_zero_dep(ctx0, csa_state_kv, kv_comp_csa_state); - csa_state_score = dsv4_with_zero_dep(ctx0, csa_state_score, kv_comp_csa_state); + ggml_tensor * csa_snapshot_source_kv = ggml_concat(ctx0, + csa_restored.kv, csa_state_kv, 1); + ggml_tensor * csa_snapshot_source_score = ggml_concat(ctx0, + csa_restored.score, csa_state_score, 1); + + const dsv4_state_tensors csa_snapshot = dsv4_build_state_snapshot( + ctx0, inp_dsv4->get_csa(), csa_state, csa_snapshot_source_kv, csa_snapshot_source_score, il); + if (csa_snapshot.kv != nullptr) { + ggml_build_forward_expand(gf, csa_snapshot.kv); + } + if (csa_snapshot.score != nullptr) { + ggml_build_forward_expand(gf, csa_snapshot.score); + } ggml_tensor * csa_persist_kv = ggml_get_rows(ctx0, csa_state_kv, inp_dsv4->get_csa().state_persist_src_idxs); ggml_tensor * csa_persist_score = ggml_get_rows(ctx0, csa_state_score, inp_dsv4->get_csa().state_persist_src_idxs); @@ -958,10 +1082,16 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention( GGML_ASSERT(inp_dsv4->get_lid().state_write_idxs); - ggml_tensor * lid_source_kv = ggml_concat(ctx0, - inp_dsv4->mctx->get_lid_state()->get_kv(ctx0, il), lid_state_kv, 1); - ggml_tensor * lid_source_score = ggml_concat(ctx0, - inp_dsv4->mctx->get_lid_state()->get_score(ctx0, il), lid_state_score, 1); + const auto * lid_state = inp_dsv4->mctx->get_lid_state(); + const dsv4_state_tensors lid_restored = dsv4_build_state_restore( + ctx0, inp_dsv4->get_lid(), lid_state, il); + ggml_tensor * lid_base_kv = dsv4_view_2d( + ctx0, lid_restored.kv, lid_restored.kv->ne[0], lid_state->get_n_rows(), 0); + ggml_tensor * lid_base_score = dsv4_view_2d( + ctx0, lid_restored.score, lid_restored.score->ne[0], lid_state->get_n_rows(), 0); + + ggml_tensor * lid_source_kv = ggml_concat(ctx0, lid_base_kv, lid_state_kv, 1); + ggml_tensor * lid_source_score = ggml_concat(ctx0, lid_base_score, lid_state_score, 1); ggml_tensor * kv_comp_lid_state = build_overlap_compressed_kv_from_state( lid_source_kv, @@ -982,8 +1112,19 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention( ggml_build_forward_expand(gf, inp_dsv4->mctx->get_lid()->cpy_k(ctx0, kv_comp_lid_state, inp_dsv4->get_lid().state_write_idxs, il)); - lid_state_kv = dsv4_with_zero_dep(ctx0, lid_state_kv, kv_comp_lid_state); - lid_state_score = dsv4_with_zero_dep(ctx0, lid_state_score, kv_comp_lid_state); + ggml_tensor * lid_snapshot_source_kv = ggml_concat(ctx0, + lid_restored.kv, lid_state_kv, 1); + ggml_tensor * lid_snapshot_source_score = ggml_concat(ctx0, + lid_restored.score, lid_state_score, 1); + + const dsv4_state_tensors lid_snapshot = dsv4_build_state_snapshot( + ctx0, inp_dsv4->get_lid(), lid_state, lid_snapshot_source_kv, lid_snapshot_source_score, il); + if (lid_snapshot.kv != nullptr) { + ggml_build_forward_expand(gf, lid_snapshot.kv); + } + if (lid_snapshot.score != nullptr) { + ggml_build_forward_expand(gf, lid_snapshot.score); + } ggml_tensor * lid_persist_kv = ggml_get_rows(ctx0, lid_state_kv, inp_dsv4->get_lid().state_persist_src_idxs); ggml_tensor * lid_persist_score = ggml_get_rows(ctx0, lid_state_score, inp_dsv4->get_lid().state_persist_src_idxs); @@ -997,15 +1138,21 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention( ggml_build_forward_expand(gf, lid_state_score); } - ggml_tensor * hca_state_dep = nullptr; + const llama_dsv4_comp_state * hca_state = nullptr; + dsv4_state_tensors hca_restored = {}; if (ratio == DSV4_HCA_RATIO && inp_dsv4->get_hca().state_write_idxs) { GGML_ASSERT(hca_state_kv); GGML_ASSERT(hca_state_score); - ggml_tensor * hca_source_kv = ggml_concat(ctx0, - inp_dsv4->mctx->get_hca_state()->get_kv(ctx0, il), hca_state_kv, 1); - ggml_tensor * hca_source_score = ggml_concat(ctx0, - inp_dsv4->mctx->get_hca_state()->get_score(ctx0, il), hca_state_score, 1); + hca_state = inp_dsv4->mctx->get_hca_state(); + hca_restored = dsv4_build_state_restore(ctx0, inp_dsv4->get_hca(), hca_state, il); + ggml_tensor * hca_base_kv = dsv4_view_2d( + ctx0, hca_restored.kv, hca_restored.kv->ne[0], hca_state->get_n_rows(), 0); + ggml_tensor * hca_base_score = dsv4_view_2d( + ctx0, hca_restored.score, hca_restored.score->ne[0], hca_state->get_n_rows(), 0); + + hca_source_kv = ggml_concat(ctx0, hca_base_kv, hca_state_kv, 1); + hca_source_score = ggml_concat(ctx0, hca_base_score, hca_state_score, 1); ggml_tensor * kv_comp_hca = build_hca_compressed_kv_from_state( hca_source_kv, @@ -1024,15 +1171,41 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention( ggml_build_forward_expand(gf, inp_dsv4->mctx->get_hca()->cpy_k(ctx0, kv_comp_hca, inp_dsv4->get_hca().state_write_idxs, il)); - hca_state_dep = kv_comp_hca; } if (ratio == DSV4_HCA_RATIO && inp_dsv4->get_hca().state_pos) { GGML_ASSERT(hca_state_kv); GGML_ASSERT(hca_state_score); - hca_state_kv = dsv4_with_zero_dep(ctx0, hca_state_kv, hca_state_dep); - hca_state_score = dsv4_with_zero_dep(ctx0, hca_state_score, hca_state_dep); + if (hca_state == nullptr) { + hca_state = inp_dsv4->mctx->get_hca_state(); + } + if (hca_restored.kv == nullptr) { + hca_restored = dsv4_build_state_restore(ctx0, inp_dsv4->get_hca(), hca_state, il); + } + if (hca_source_kv == nullptr || hca_source_score == nullptr) { + ggml_tensor * hca_base_kv = dsv4_view_2d( + ctx0, hca_restored.kv, hca_restored.kv->ne[0], hca_state->get_n_rows(), 0); + ggml_tensor * hca_base_score = dsv4_view_2d( + ctx0, hca_restored.score, hca_restored.score->ne[0], hca_state->get_n_rows(), 0); + + hca_source_kv = ggml_concat(ctx0, hca_base_kv, hca_state_kv, 1); + hca_source_score = ggml_concat(ctx0, hca_base_score, hca_state_score, 1); + } + + ggml_tensor * hca_snapshot_source_kv = ggml_concat(ctx0, + hca_restored.kv, hca_state_kv, 1); + ggml_tensor * hca_snapshot_source_score = ggml_concat(ctx0, + hca_restored.score, hca_state_score, 1); + + const dsv4_state_tensors hca_snapshot = dsv4_build_state_snapshot( + ctx0, inp_dsv4->get_hca(), hca_state, hca_snapshot_source_kv, hca_snapshot_source_score, il); + if (hca_snapshot.kv != nullptr) { + ggml_build_forward_expand(gf, hca_snapshot.kv); + } + if (hca_snapshot.score != nullptr) { + ggml_build_forward_expand(gf, hca_snapshot.score); + } ggml_tensor * hca_persist_kv = ggml_get_rows(ctx0, hca_state_kv, inp_dsv4->get_hca().state_persist_src_idxs); ggml_tensor * hca_persist_score = ggml_get_rows(ctx0, hca_state_score, inp_dsv4->get_hca().state_persist_src_idxs); @@ -1047,7 +1220,14 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention( } ggml_tensor * out = nullptr; - if (ratio == DSV4_CSA_RATIO && + if (inp_mtp) { + out = build_attn(inp_mtp, + nullptr, nullptr, nullptr, + q, kv, nullptr, + nullptr, layer.attn_sinks, nullptr, + 1.0f/sqrtf(float(n_embd_head)), il); + cb(out, "attn_raw", il); + } else if (ratio == DSV4_CSA_RATIO && inp_dsv4->get_csa().kq_mask && inp_dsv4->get_lid().kq_mask && inp_dsv4->get_lid().k_rot) { @@ -1106,6 +1286,12 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p cb(inpL, "hc_init", -1); for (int il = 0; il < n_layer; ++il) { + if ((size_t) il < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[il]) { + res->t_layer_inp[il] = dsv4_hc_mean(ctx0, inpL); + cb(res->t_layer_inp[il], "layer_inp", il); + ggml_build_forward_expand(gf, res->t_layer_inp[il]); + } + ggml_tensor * residual = inpL; ggml_tensor * post = nullptr; ggml_tensor * comb = nullptr; @@ -1182,10 +1368,23 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p cb(inpL, "l_last", il); } + if ((size_t) n_layer < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[n_layer]) { + res->t_layer_inp[n_layer] = dsv4_hc_mean(ctx0, inpL); + cb(res->t_layer_inp[n_layer], "layer_inp", n_layer); + ggml_build_forward_expand(gf, res->t_layer_inp[n_layer]); + } + + ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens); + ggml_tensor * flat_out = inp_out_ids ? ggml_get_rows(ctx0, flat, inp_out_ids) : flat; + + if (cparams.embeddings_nextn) { + ggml_tensor * h_nextn = cparams.embeddings_nextn_masked ? flat_out : inpL; + cb(h_nextn, "h_nextn", -1); + res->t_h_nextn = h_nextn; + } + if (inp_out_ids) { - ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens); - flat = ggml_get_rows(ctx0, flat, inp_out_ids); - inpL = ggml_reshape_3d(ctx0, flat, n_embd, hc, n_outputs); + inpL = ggml_reshape_3d(ctx0, flat_out, n_embd, hc, n_outputs); } cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base); @@ -1201,3 +1400,145 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p ggml_build_forward_expand(gf, cur); } + + +llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : + graph(params) { + GGML_ASSERT(hparams.n_layer_nextn > 0 && "DEEPSEEK4 MTP requires n_layer_nextn > 0"); + GGML_ASSERT(hparams.n_layer_nextn == 1 && "DEEPSEEK4 MTP currently only supports a single MTP block"); + GGML_ASSERT(cparams.nextn_layer_offset >= 0 && + cparams.nextn_layer_offset < (int) hparams.n_layer_nextn && + "nextn_layer_offset out of range [0, n_layer_nextn)"); + GGML_ASSERT(ubatch.token && "DEEPSEEK4 MTP requires token input"); + + const int64_t hc = hparams.dsv4_hc_mult; + GGML_ASSERT(hparams.n_embd_out() == (uint32_t) (n_embd*hc) && "DEEPSEEK4 MTP hidden width mismatch"); + + const int il = hparams.n_layer() + cparams.nextn_layer_offset; + const auto & layer = model.layers[il]; + + GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj"); + GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm"); + GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); + + auto inp = std::make_unique(hparams.n_embd_out()); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens); + ggml_set_input(inp->embd); + + inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens); + ggml_set_input(inp->h); + ggml_set_name(inp->h, "mtp_h_input"); + + ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; + ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens); + cb(tok_embd, "mtp_tok_embd", il); + + ggml_tensor * h_state = ggml_reshape_3d(ctx0, inp->h, n_embd, hc, n_tokens); + cb(h_state, "mtp_h_state", il); + + res->add_input(std::move(inp)); + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa(); + + ggml_tensor * h_norm = build_norm(h_state, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + + ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il); + e_norm = ggml_reshape_3d(ctx0, e_norm, n_embd, 1, n_tokens); + e_norm = ggml_repeat_4d(ctx0, e_norm, n_embd, hc, n_tokens, 1); + cb(e_norm, "mtp_enorm", il); + + ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, 0); + cb(concat, "mtp_concat", il); + + ggml_tensor * inpL = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s); + cb(inpL, "mtp_eh_proj", il); + + ggml_tensor * residual = inpL; + ggml_tensor * post = nullptr; + ggml_tensor * comb = nullptr; + + ggml_tensor * cur = build_hc_pre(inpL, + layer.hc_attn_fn, + layer.hc_attn_scale, + layer.hc_attn_base, + &post, &comb, il); + cb(cur, "mtp_hc_attn_pre", il); + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_norm", il); + + cur = build_attention(model, inp_attn, cur, inp_pos, il); + + inpL = build_hc_post(cur, residual, post, comb, il); + cb(inpL, "mtp_hc_attn_post", il); + + residual = inpL; + cur = build_hc_pre(inpL, + layer.hc_ffn_fn, + layer.hc_ffn_scale, + layer.hc_ffn_base, + &post, &comb, il); + cb(cur, "mtp_hc_ffn_pre", il); + + cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_ffn_norm", il); + + GGML_ASSERT((uint32_t) il >= hparams.dsv4_hash_layer_count && "DEEPSEEK4 MTP does not support hash-routed MTP blocks"); + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, hparams.n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il); + cb(moe_out, "mtp_ffn_moe_out", il); + + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, nullptr, nullptr, + layer.ffn_gate_shexp, nullptr, nullptr, + layer.ffn_down_shexp, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "mtp_ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "mtp_ffn_out", il); + + inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_cvec(inpL, il); + cb(inpL, "mtp_l_out", il); + + ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens); + ggml_tensor * h_nextn = ggml_get_rows(ctx0, flat, inp_out_ids); + cb(h_nextn, "h_nextn", -1); + res->t_h_nextn = h_nextn; + + inpL = ggml_reshape_3d(ctx0, h_nextn, n_embd, hc, n_outputs); + + cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base); + cb(cur, "mtp_hc_head", -1); + + ggml_tensor * head_norm_w = layer.nextn.shared_head_norm ? layer.nextn.shared_head_norm : model.output_norm; + GGML_ASSERT(head_norm_w && "DEEPSEEK4 MTP missing shared head norm"); + cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1); + cb(cur, "mtp_shared_head_norm", -1); + res->t_embd = cur; + + ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output; + GGML_ASSERT(head_w && "DEEPSEEK4 MTP missing LM head"); + cur = ggml_mul_mat(ctx0, head_w, cur); + cb(cur, "result_output", -1); + + res->t_logits = cur; + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index dcff3aec9fa2..6c82ab3da43a 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -20,6 +20,48 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) { } LLAMA_LOG_INFO("]\n"); + // DeepSeek-V4 DSpark backbone: stages are full DSV4 blocks, uniform sliding window (the draft KV ring) + ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult, false); + if (hparams.dsv4_hc_mult > 0) { + ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); + ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all); + if (!ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, 0)) { + hparams.swiglu_clamp_shexp = hparams.swiglu_clamp_exp; + } + ml.get_key(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count); + ml.get_key(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank); + ml.get_key(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters); + ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps); + ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, false); + + if (hparams.expert_gating_func != LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) { + throw std::runtime_error("DSpark DSV4 draft expects sqrtsoftplus MoE scoring"); + } + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + if (hparams.dsv4_compress_ratios[il] != 0) { + throw std::runtime_error("DSpark DSV4 draft expects uncompressed attention on all stages"); + } + } + + GGML_ASSERT(hparams.n_swa > 0); + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + hparams.set_swa_pattern(0); + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.is_swa_impl[il] = true; + } + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + + type = LLM_TYPE_UNKNOWN; + return; + } + // optional interleaved sliding-window attention with per-layer pattern array. // DFlash has a single rope, so the SWA rope == main rope. if (ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false) && hparams.n_swa > 0) { @@ -58,6 +100,56 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), { n_embd }, 0); // encoder hidden_norm (after fc) output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); // decoder final norm + if (hparams.dsv4_hc_mult > 0) { + const int64_t q_lora_rank = hparams.n_lora_q; + const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_expert_shared = hparams.n_expert_shared; + const int64_t n_embd_head = hparams.n_embd_head_k(); + const int64_t o_groups = hparams.dsv4_o_group_count; + const int64_t o_lora_rank = hparams.dsv4_o_lora_rank; + const int64_t hc_mult = hparams.dsv4_hc_mult; + const int64_t hc_dim = hc_mult * n_embd; + const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult; + + hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0); + hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0); + hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0); + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0); + layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0); + layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0); + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, 0); + layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, 0); + layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, 0); + layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, 0); + layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, 0); + + layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0); + layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, 0); + layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, 0); + layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0); + layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, 0); + layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, 0); + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd }, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); + } + return; + } + for (int i = 0; i < n_layer; ++i) { auto & layer = layers[i]; @@ -84,6 +176,9 @@ std::unique_ptr llama_model_dflash::build_arch_graph(const ll return std::make_unique>(*this, params); case LLM_GRAPH_TYPE_DEFAULT: case LLM_GRAPH_TYPE_DECODER: + if (hparams.dsv4_hc_mult > 0) { + return std::make_unique(*this, params); + } return std::make_unique>(*this, params); default: GGML_ABORT("invalid graph type"); @@ -403,3 +498,178 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_gra build_dspark_markov_head(*this, model, inp_tokens); } } + +// DSV4 DSpark decoder, dual-mode by batch type (see the DFlash decoder above): +// * embd batch -> project main_x through each stage's wkv and inject K into the ring cache +// * token batch -> noise block through 3 full DSV4 stages (hc + MLA + MoE), markov + confidence heads +llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_graph_params & params) : + llama_model_deepseek4::graph(params) { + const int64_t n_embd_head = hparams.n_embd_head_k(); + const int64_t n_embd_head_rope = hparams.n_rot(); + const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope; + + ggml_tensor * inp_pos = build_inp_pos(); + + llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa(); + + // KV cache injection: fused target features from the encoder + if (ubatch.embd) { + auto inp = std::make_unique(n_embd); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens); + ggml_set_input(inp->embd); + + ggml_tensor * inp_g = inp->embd; + cb(inp_g, "inp_g_embeddings", -1); + + res->add_input(std::move(inp)); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + // main-track KV: kv_norm(wkv(main_x)) with rope on the trailing dims, same + // rope parameters as the uncompressed layers in build_attention_impl + ggml_tensor * kv = build_lora_mm(layer.wkv, inp_g); + kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il); + kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens); + + ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, n_tokens, + ggml_row_size(kv->type, n_embd_head), + ggml_row_size(kv->type, n_embd_head), + 0); + ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, n_tokens, + ggml_row_size(kv->type, n_embd_head), + ggml_row_size(kv->type, n_embd_head), + ggml_row_size(kv->type, n_embd_head_nope)); + kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, 0, + freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + kv = ggml_concat(ctx0, kv_nope, kv_pe, 0); + cb(kv, "kv_injected", il); + + if (inp_attn->self_k_rot_swa) { + kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa); + } + ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il)); + } + + res->t_embd = inp_g; + + ggml_build_forward_expand(gf, inp_g); + return; + } + + // tok_embd from the target model (shared via ctx_other) + auto * tok_embd = model.tok_embd; + if (tok_embd == nullptr) { + GGML_ASSERT(cparams.ctx_other != nullptr); + const auto * model_other = llama_get_model(cparams.ctx_other); + + GGML_ASSERT(model_other->tok_embd != nullptr && "DSpark decoder requires the target model's token embeddings"); + tok_embd = model_other->tok_embd; + } + + auto inp = std::make_unique(n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + ggml_tensor * inp_tokens = inp->tokens; + + ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens); + cb(inpL, "inp_noise_embd", -1); + + res->add_input(std::move(inp)); + + const int64_t hc = hparams.dsv4_hc_mult; + inpL = ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens); + inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1); + cb(inpL, "hc_init", -1); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + ggml_tensor * residual = inpL; + ggml_tensor * post = nullptr; + ggml_tensor * comb = nullptr; + + ggml_tensor * cur = build_hc_pre(inpL, + layer.hc_attn_fn, + layer.hc_attn_scale, + layer.hc_attn_base, + &post, &comb, il); + cb(cur, "hc_attn_pre", il); + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + cur = build_attention(model, inp_attn, cur, inp_pos, il); + + inpL = build_hc_post(cur, residual, post, comb, il); + cb(inpL, "hc_attn_post", il); + + residual = inpL; + cur = build_hc_pre(inpL, + layer.hc_ffn_fn, + layer.hc_ffn_scale, + layer.hc_ffn_base, + &post, &comb, il); + cb(cur, "hc_ffn_pre", il); + + cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, hparams.n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il); + cb(moe_out, "ffn_moe_out", il); + + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, nullptr, nullptr, + layer.ffn_gate_shexp, nullptr, nullptr, + layer.ffn_down_shexp, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + + inpL = build_hc_post(cur, residual, post, comb, il); + cb(inpL, "l_out", il); + } + + ggml_tensor * cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base); + cb(cur, "hc_head", -1); + + // confidence head input: the reference scores the pre-norm collapsed hidden state + res->t_embd = cur; + + cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + + // lm_head from the target model (shared via ctx_other) + auto * output = model.output; + if (output == nullptr) { + GGML_ASSERT(cparams.ctx_other != nullptr); + const auto * model_other = llama_get_model(cparams.ctx_other); + GGML_ASSERT(model_other->output != nullptr && "DSpark decoder requires the target model's output projection"); + output = model_other->output; + } + + cur = build_lora_mm(output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); + + if (model.dspark_markov_w1) { + build_dspark_markov_head(*this, model, inp_tokens); + } +} diff --git a/src/models/models.h b/src/models/models.h index bb372ece81ee..2d5de5d432ee 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1107,6 +1107,7 @@ struct llama_model_deepseek4 : public llama_model_base { void load_arch_tensors(llama_model_loader & ml) override; struct graph : public llm_graph_context { + graph(const llm_graph_params & params) : llm_graph_context(params) {} graph(const llama_model & model, const llm_graph_params & params); ggml_tensor * build_hc_pre( @@ -1138,6 +1139,21 @@ struct llama_model_deepseek4 : public llama_model_base { ggml_tensor * inp_pos, int il) const; + ggml_tensor * build_attention( + const llama_model & model, + llm_graph_input_attn_k_iswa * inp_mtp, + ggml_tensor * cur, + ggml_tensor * inp_pos, + int il) const; + + ggml_tensor * build_attention_impl( + const llama_model & model, + llm_graph_input_dsv4 * inp_dsv4, + llm_graph_input_attn_k_iswa * inp_mtp, + ggml_tensor * cur, + ggml_tensor * inp_pos, + int il) const; + ggml_tensor * build_hca_compressed_kv_from_state( ggml_tensor * kv_state, ggml_tensor * score_state, @@ -1213,6 +1229,10 @@ struct llama_model_deepseek4 : public llama_model_base { int il) const; }; + struct graph_mtp : public graph { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; @@ -1272,6 +1292,10 @@ struct llama_model_dflash : public llama_model_base { ggml_tensor * build_inp_embd_enc() const; }; + struct graph_dsv4 : public llama_model_deepseek4::graph { + graph_dsv4(const llama_model & model, const llm_graph_params & params); + }; + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; diff --git a/tests/test-recurrent-state-rollback.cpp b/tests/test-recurrent-state-rollback.cpp index 8e2eace6a195..5d1f0140b623 100644 --- a/tests/test-recurrent-state-rollback.cpp +++ b/tests/test-recurrent-state-rollback.cpp @@ -83,27 +83,33 @@ int main(int argc, char ** argv) { if (llama_vocab_type(vocab) == LLAMA_VOCAB_TYPE_NONE) { tokens = { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; } else { - tokens = common_tokenize(ctx_src, "The quick brown fox jumps", true); + tokens = common_tokenize(ctx_src, "The quick brown fox jumps over the lazy dog", true); } const uint32_t n_rs_seq = llama_n_rs_seq(ctx_src); - if (tokens.size() > n_rs_seq + 1) { - tokens.resize(n_rs_seq + 1); + constexpr uint32_t n_rollback = 3; + if (n_rs_seq < n_rollback) { + fprintf(stderr, "%s : skipping because n_rs_seq is too small\n", __func__); + llama_free(ctx_src); + llama_free(ctx_dst); + return 0; } - if (tokens.size() < 2) { + if (tokens.empty()) { fprintf(stderr, "%s : not enough prompt tokens\n", __func__); return 1; } - const uint32_t n_tokens = tokens.size(); - const llama_token last_tok = tokens.back(); - const llama_pos last_pos = (llama_pos) n_tokens - 2; + tokens.resize(n_rs_seq + 1, tokens.back()); - // Decode the full prompt on the source, then roll back the last position. + const uint32_t n_tokens = tokens.size(); + const llama_pos rollback_pos = (llama_pos) n_tokens - n_rollback; + + // Decode the full prompt on the source, then roll back three positions. + // Replaying them crosses DSV4's ratio-4 compressor boundary. // Rollback leaves the recurrent memory in a snapshot state (rs_idx != 0). if (!decode_tokens(ctx_src, tokens, n_tokens)) { fprintf(stderr, "%s : failed to decode prompt\n", __func__); return 1; } - if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, last_pos, -1)) { + if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1)) { fprintf(stderr, "%s : rollback failed\n", __func__); return 1; } @@ -113,31 +119,56 @@ int main(int argc, char ** argv) { ckpt.update_tgt(ctx_src, 0, 0); ckpt.load_tgt(ctx_dst, 0, 0); - // Replay the rolled-back token on both contexts and compare logits. - if (!decode_one(ctx_src, last_tok, last_pos) || - !decode_one(ctx_dst, last_tok, last_pos)) { - fprintf(stderr, "%s : replay failed\n", __func__); + constexpr float eps = 1e-5f; + std::vector> logits_src_replay(n_rollback); + const auto replay_and_compare = [&](const char * mode) { + for (uint32_t i = 0; i < n_rollback; ++i) { + const llama_pos pos = rollback_pos + i; + if (!decode_one(ctx_src, tokens[pos], pos) || + !decode_one(ctx_dst, tokens[pos], pos)) { + fprintf(stderr, "%s : %s replay failed at position %d\n", __func__, mode, pos); + return false; + } + + const float * logits_src = llama_get_logits_ith(ctx_src, 0); + const float * logits_dst = llama_get_logits_ith(ctx_dst, 0); + if (logits_src == nullptr || logits_dst == nullptr) { + fprintf(stderr, "%s : missing %s logits at position %d\n", __func__, mode, pos); + return false; + } + + logits_src_replay[i].assign(logits_src, logits_src + n_vocab); + for (int token = 0; token < n_vocab; ++token) { + if (std::fabs(logits_src[token] - logits_dst[token]) > eps) { + fprintf(stderr, "%s : %s logits mismatch at position %d, token %d (%g != %g)\n", + __func__, mode, pos, token, (double) logits_src[token], (double) logits_dst[token]); + return false; + } + } + } + return true; + }; + if (!replay_and_compare("full")) { return 1; } - const float * logits_src = llama_get_logits_ith(ctx_src, 0); - const float * logits_dst = llama_get_logits_ith(ctx_dst, 0); - if (logits_src == nullptr || logits_dst == nullptr) { - fprintf(stderr, "%s : missing logits\n", __func__); + if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) || + !llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) { + fprintf(stderr, "%s : partial rollback failed\n", __func__); return 1; } - constexpr float eps = 1e-5f; - for (int i = 0; i < n_vocab; ++i) { - if (std::fabs(logits_src[i] - logits_dst[i]) > eps) { - fprintf(stderr, "%s : logits mismatch at token %d (%g != %g)\n", - __func__, i, (double) logits_src[i], (double) logits_dst[i]); - return 1; - } + constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY; + common_prompt_checkpoint ckpt_partial; + ckpt_partial.update_tgt(ctx_src, 0, partial_flags); + ckpt_partial.load_tgt(ctx_dst, 0, partial_flags); + + if (!replay_and_compare("partial")) { + return 1; } // Repeat the load into a context that already has its own rollback state: - // groups 1..n_rs_seq hold a *different* prompt's history, and rs_idx[0] is + // groups 1..n_rs_seq hold a different prompt's history, and rs_idx[0] is // non-zero at load time. The restore must wipe that state and still match. llama_context * ctx_dirty = make_ctx(params, model); if (ctx_dirty == nullptr) { @@ -156,30 +187,33 @@ int main(int argc, char ** argv) { fprintf(stderr, "%s : dirty prompt decode failed\n", __func__); return 1; } - if (!llama_memory_seq_rm(llama_get_memory(ctx_dirty), 0, last_pos, -1)) { + if (!llama_memory_seq_rm(llama_get_memory(ctx_dirty), 0, rollback_pos, -1)) { fprintf(stderr, "%s : dirty rollback failed\n", __func__); return 1; } ckpt.load_tgt(ctx_dirty, 0, 0); - if (!decode_one(ctx_dirty, last_tok, last_pos)) { - fprintf(stderr, "%s : dirty replay failed\n", __func__); - return 1; - } - - const float * logits_dirty = llama_get_logits_ith(ctx_dirty, 0); - if (logits_dirty == nullptr) { - fprintf(stderr, "%s : missing dirty logits\n", __func__); - return 1; - } + for (uint32_t i = 0; i < n_rollback; ++i) { + const llama_pos pos = rollback_pos + i; + if (!decode_one(ctx_dirty, tokens[pos], pos)) { + fprintf(stderr, "%s : dirty replay failed at position %d\n", __func__, pos); + return 1; + } - for (int i = 0; i < n_vocab; ++i) { - if (std::fabs(logits_src[i] - logits_dirty[i]) > eps) { - fprintf(stderr, "%s : dirty-ctx logits mismatch at token %d (%g != %g)\n", - __func__, i, (double) logits_src[i], (double) logits_dirty[i]); + const float * logits_dirty = llama_get_logits_ith(ctx_dirty, 0); + if (logits_dirty == nullptr) { + fprintf(stderr, "%s : missing dirty logits at position %d\n", __func__, pos); return 1; } + + for (int token = 0; token < n_vocab; ++token) { + if (std::fabs(logits_src_replay[i][token] - logits_dirty[token]) > eps) { + fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, token %d (%g != %g)\n", + __func__, pos, token, (double) logits_src_replay[i][token], (double) logits_dirty[token]); + return 1; + } + } } fprintf(stderr, "%s : recurrent rollback checkpoint restored successfully\n", __func__); From c745be2a2c5aefbf9f3ced0440804373731890b6 Mon Sep 17 00:00:00 2001 From: akleine Date: Sun, 2 Aug 2026 15:43:00 +0200 Subject: [PATCH 25/98] opencl: bugfix increment ref_count in ggml_backend_opencl_init() (#26162) Incrementing `ref_count` at the beginning is important later in the `free()` method of the `ggml_backend_opencl_context` at program end. If we do not increment the `ref_count`, the result would be -1 here, and consequently, the profiling data would not be flushed and written. ( #ifdef GGML_OPENCL_PROFILING ) --- ggml/src/ggml-opencl/ggml-opencl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index d07b8fe41a31..388c585fa121 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -7495,6 +7495,7 @@ static ggml_backend_i ggml_backend_opencl_i = { ggml_backend_t ggml_backend_opencl_init(void) { ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_opencl_reg(), 0); ggml_backend_opencl_context *backend_ctx = ggml_cl_init(dev); + backend_ctx->ref_count++; ggml_backend_t backend = new ggml_backend { /* .guid = */ ggml_backend_opencl_guid(), From 3581ba0cf591b3f772fbb002de0f70e294bc0396 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Sun, 2 Aug 2026 23:16:31 +0800 Subject: [PATCH 26/98] convert: add option to create separate dspark GGUF (#26452) * convert: add option to create separate dspark GGUF * add --no-nextn * fix convert bug --- conversion/__init__.py | 1 + conversion/deepseek.py | 105 +++++++++++++++++++++++++++++++++++++- convert_hf_to_gguf.py | 21 ++++++-- gguf-py/gguf/constants.py | 25 +++++++++ 4 files changed, 146 insertions(+), 6 deletions(-) diff --git a/conversion/__init__.py b/conversion/__init__.py index 1a47b851a0e6..534f9e309a13 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -55,6 +55,7 @@ "DFlashDraftModel": "qwen", "Qwen3DSparkModel": "qwen", "DeepseekV4ForCausalLM": "deepseek", + "DeepseekV4DSparkModel": "deepseek", "DistilBertForMaskedLM": "bert", "DistilBertForSequenceClassification": "bert", "DistilBertModel": "bert", diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 0bf69be3be98..5b69e23437f3 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -620,7 +620,8 @@ def set_gguf_parameters(self): self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"]) self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"]) self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"]) - self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"]) + if self.model_arch == gguf.MODEL_ARCH.DEEPSEEK4: + self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"]) if self.mtp_only and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers", 0)) > 0: self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers) @@ -878,3 +879,105 @@ def prepare_tensors(self): super().prepare_tensors() self._is_mxfp4 = True self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE + + +@ModelBase.register("DeepseekV4DSparkModel") +class DeepseekV4DSparkModel(DeepseekV4Model): + model_arch = gguf.MODEL_ARCH.DFLASH + + _DSPARK_ROOT_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { + "main_proj.weight": (gguf.MODEL_TENSOR.FC, ".weight"), + "main_norm.weight": (gguf.MODEL_TENSOR.ENC_OUTPUT_NORM, ".weight"), + "markov_head.markov_w1.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W1, ".weight"), + "markov_head.markov_w2.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W2, ".weight"), + "confidence_head.proj.weight": (gguf.MODEL_TENSOR.DSPARK_CONF_PROJ, ".weight"), + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.block_count = 1 + max( + int(match.group(1)) for name in self.model_tensors + if (match := re.match(r"layers\.(\d+)\.", name)) + ) + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + self.hparams["compress_ratios"] = [0] * self.block_count + self.hparams["num_hash_layers"] = 0 + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + if remote_hf_model_id is None: + return super().index_tensors() + + with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: + weight_map = json.load(f)["weight_map"] + + part_names = sorted({ + part_name for name, part_name in weight_map.items() + if name.startswith("mtp.") + }) + tensors: dict[str, Callable[[], Tensor]] = {} + + for part_name in part_names: + from huggingface_hub import hf_hub_download + + logger.info("gguf: caching remote DSpark part '%s'", part_name) + part_path = Path(hf_hub_download(repo_id=remote_hf_model_id, filename=part_name)) + with gguf.utility.SafetensorsLocal(part_path) as model_part: + for name in model_part: + data = model_part[name] + data_gen = lambda data=data: LazyTorchTensor.from_local_tensor(data) # noqa: E731 + if titem := self.filter_tensors((name, data_gen)): + tensor_name, tensor_gen = titem + tensors[tensor_name] = tensor_gen + + return tensors + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if not name.startswith("mtp."): + return None + return super().filter_tensors((cls._rekey_mtp_tensor_name(name), gen)) + + @staticmethod + def _rekey_mtp_tensor_name(name: str) -> str: + match = re.match(r"mtp\.(\d+)\.(.+)$", name) + if match is None: + raise ValueError(f"Unexpected DSpark tensor {name!r}") + + stage, rest = match.group(1), match.group(2) + root_names = ( + "main_proj.scale", + "norm.weight", + "hc_head_fn", + "hc_head_base", + "hc_head_scale", + ) + if rest in DeepseekV4DSparkModel._DSPARK_ROOT_MAP or rest in root_names: + return rest + return f"layers.{stage}.{rest}" + + def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]: + if name in self._DSPARK_ROOT_MAP: + return self._DSPARK_ROOT_MAP[name] + return super()._map_dsv4_tensor_name(name, bid) + + def set_vocab(self): + if self.target_model_dir is None: + raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer") + + original_dir = self.dir_model + try: + self.dir_model = self.target_model_dir + super().set_vocab() + finally: + self.dir_model = original_dir + + self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"]) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + self.gguf_writer.add_block_size(self.hparams["dspark_block_size"]) + self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]]) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 2c5e62a16fbe..78ad26c65630 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -122,8 +122,12 @@ def parse_args() -> argparse.Namespace: help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.", ) parser.add_argument( - "--no-mtp", action="store_true", - help="Exclude the multi-token prediction (MTP) head from the converted GGUF. Pair with --mtp on a second run to publish trunk and MTP as two files. Note: the split form duplicates embeddings, but even though the bundled default is more space-efficient overall, this allows differing quantization which may be more performant.", + "--no-nextn", "--no-mtp", dest="no_mtp", action="store_true", + help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.", + ) + parser.add_argument( + "--dspark", action="store_true", + help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.", ) parser.add_argument( "--mistral-format", action="store_true", @@ -254,13 +258,20 @@ def main() -> None: from conversion.mistral import MistralModel model_class = MistralModel - if args.mtp and args.no_mtp: - logger.error("--mtp and --no-mtp are mutually exclusive") + if sum((args.mtp, args.no_mtp, args.dspark)) > 1: + logger.error("--mtp, --no-nextn, and --dspark are mutually exclusive") sys.exit(1) + if args.dspark: + if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM": + logger.error("--dspark is only supported for DeepseekV4ForCausalLM") + sys.exit(1) + from conversion.deepseek import DeepseekV4DSparkModel + model_class = DeepseekV4DSparkModel + if args.mtp or args.no_mtp: if not model_class.supports_mtp_export: - logger.error("--mtp / --no-mtp are not supported for %s", model_architecture) + logger.error("--mtp / --no-nextn are not supported for %s", model_architecture) sys.exit(1) if args.no_mtp: model_class.no_mtp = True diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 978231091c87..a0e55781bac2 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4383,10 +4383,35 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_SINKS, + MODEL_TENSOR.ATTN_Q_A, + MODEL_TENSOR.ATTN_Q_B, + MODEL_TENSOR.ATTN_Q_A_NORM, + MODEL_TENSOR.ATTN_KV, + MODEL_TENSOR.ATTN_KV_NORM, + MODEL_TENSOR.ATTN_OUT_A, + MODEL_TENSOR.ATTN_OUT_B, + MODEL_TENSOR.HC_ATTN_FN, + MODEL_TENSOR.HC_ATTN_BASE, + MODEL_TENSOR.HC_ATTN_SCALE, + MODEL_TENSOR.HC_FFN_FN, + MODEL_TENSOR.HC_FFN_BASE, + MODEL_TENSOR.HC_FFN_SCALE, + MODEL_TENSOR.HC_HEAD_FN, + MODEL_TENSOR.HC_HEAD_BASE, + MODEL_TENSOR.HC_HEAD_SCALE, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, MODEL_TENSOR.FC, MODEL_TENSOR.ENC_OUTPUT_NORM, # optional DSpark heads From bb4e0e1b3f6bb38960769a1c9bcd2081016154cd Mon Sep 17 00:00:00 2001 From: Pascal Date: Sun, 2 Aug 2026 19:25:27 +0200 Subject: [PATCH 27/98] common: support the DSpark sidecar resolution (#26458) The dspark- files resolve like the other speculative sidecars: the -hfd tag applies to them, a requested sidecar resolves without a full model at the tag, and an explicit -md selection disables the discovery. When no type is requested, dspark outranks dflash in the auto-selection since its sidecar carries the extra Markov head. --- common/arg.cpp | 36 +++++++++++++++++++++++++++++++++++- common/download.cpp | 19 +++++++++++++++---- common/download.h | 2 ++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 79480e06f9d2..772422f6820a 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -374,6 +374,10 @@ common_models_handler common_models_handler_init(const common_params & params, l params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end(); + const bool spec_type_draft_dspark = std::find(params.speculative.types.begin(), + params.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) != params.speculative.types.end(); + // only download mmproj if the current example is using it bool use_mmproj = false; for (const auto & ex : mmproj_examples) { @@ -388,6 +392,7 @@ common_models_handler common_models_handler_init(const common_params & params, l opts.download_mtp = spec_type_draft_mtp; opts.download_eagle3 = spec_type_draft_eagle3; opts.download_dflash = spec_type_draft_dflash; + opts.download_dspark = spec_type_draft_dspark; opts.download_mmproj = use_mmproj && !params.no_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty(); @@ -402,6 +407,7 @@ common_models_handler common_models_handler_init(const common_params & params, l opts_spec.download_mtp = true; opts_spec.download_dflash = true; opts_spec.download_eagle3 = true; + opts_spec.download_dspark = true; } plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec); } @@ -544,12 +550,19 @@ void common_models_handler_apply(common_models_handler & handler, common_params plan_spec.mtp = {}; plan_spec.dflash = {}; plan_spec.eagle3 = {}; + plan_spec.dspark = {}; } // infer the speculative type from the sidecar shipped by the draft repo when none is requested if (spec_types_is_default(params)) { if (!plan_spec.mtp.local_path.empty()) { params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP }; + plan_spec.dspark = {}; + plan_spec.dflash = {}; + plan_spec.eagle3 = {}; + } else if (!plan_spec.dspark.local_path.empty()) { + // dspark outranks dflash, its sidecar carries the extra Markov head + params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK }; plan_spec.dflash = {}; plan_spec.eagle3 = {}; } else if (!plan_spec.dflash.local_path.empty()) { @@ -563,7 +576,8 @@ void common_models_handler_apply(common_models_handler & handler, common_params // when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() || !plan_spec.dflash.local_path.empty() || - !plan_spec.eagle3.local_path.empty(); + !plan_spec.eagle3.local_path.empty() || + !plan_spec.dspark.local_path.empty(); if (!plan_spec.mtp.local_path.empty() && !had_spec_url) { tasks.emplace_back(plan_spec.mtp, opts, [&]() { // only use the discovered MTP head when no draft path is set yet @@ -594,6 +608,16 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }); } + if (!plan_spec.dspark.local_path.empty() && !had_spec_url) { + tasks.emplace_back(plan_spec.dspark, opts, [&]() { + // only use the discovered DSpark sidecar when no draft path is set yet + if (params.speculative.draft.mparams.path.empty()) { + params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dspark); + } else { + hf_cache::finalize_file(plan_spec.dspark); + } + }); + } // a wired draft sidecar counts as an explicit draft for the main plan fallback below if (spec_sidecar_found) { @@ -649,6 +673,16 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }); } + if (!plan.dspark.local_path.empty() && !had_spec_url) { + tasks.emplace_back(plan.dspark, opts, [&]() { + // only fall back to the discovered DSpark sidecar when no draft was explicitly provided + if (params.speculative.draft.mparams.empty()) { + params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dspark); + } else { + hf_cache::finalize_file(plan.dspark); + } + }); + } if (!plan.preset.local_path.empty()) { tasks.emplace_back(plan.preset, opts, [&]() { // if HF repo is a preset repo, we simply run server in router mode with the preset.ini file diff --git a/common/download.cpp b/common/download.cpp index 3776c6c7eb68..44c6cea4249b 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -656,6 +656,12 @@ static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, return find_best_sibling(files, model, "dflash-", tag); } +static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files, + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "dspark-", tag); +} + static bool gguf_filename_is_model(const std::string & filepath) { if (!string_ends_with(filepath, ".gguf")) { return false; @@ -670,7 +676,8 @@ static bool gguf_filename_is_model(const std::string & filepath) { filename.find("imatrix") == std::string::npos && filename.find("mtp-") == std::string::npos && filename.find("eagle3-") == std::string::npos && - filename.find("dflash-") == std::string::npos; + filename.find("dflash-") == std::string::npos && + filename.find("dspark-") == std::string::npos; } static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, @@ -763,7 +770,7 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & } else { primary = find_best_model(all, tag); // a requested sidecar can resolve on its own, without a full model of the same tag - if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) { + if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); return plan; @@ -787,9 +794,12 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & if (opts.download_eagle3) { plan.eagle3 = find_best_eagle3(all, primary.path, tag); } + if (opts.download_dspark) { + plan.dspark = find_best_dspark(all, primary.path, tag); + } if (primary.path.empty() && - plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) { + plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); } @@ -967,7 +977,8 @@ std::vector common_list_cached_models() { split.prefix.find("mmproj") != std::string::npos || split.prefix.find("mtp-") != std::string::npos || split.prefix.find("eagle3-") != std::string::npos || - split.prefix.find("dflash-") != std::string::npos) { + split.prefix.find("dflash-") != std::string::npos || + split.prefix.find("dspark-") != std::string::npos) { continue; } if (seen.insert(f.repo_id + ":" + split.tag).second) { diff --git a/common/download.h b/common/download.h index 3e789e9e9369..9a03f5e91475 100644 --- a/common/download.h +++ b/common/download.h @@ -59,6 +59,7 @@ struct common_download_opts { bool download_mtp = false; bool download_eagle3 = false; bool download_dflash = false; + bool download_dspark = false; common_download_callback * callback = nullptr; }; @@ -110,6 +111,7 @@ struct common_download_hf_plan { hf_cache::hf_file mtp; hf_cache::hf_file eagle3; hf_cache::hf_file dflash; + hf_cache::hf_file dspark; hf_cache::hf_file preset; // if set, only this file is downloaded }; common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts); From fffbcbdb9d5e56105a8842867a59bb9736520ca8 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 2 Aug 2026 21:06:02 +0300 Subject: [PATCH 28/98] metal: implement DeepSeek V4 hyper-connections (#26459) - Implement GGML_OP_DSV4_HC_COMB, GGML_OP_DSV4_HC_PRE, and GGML_OP_DSV4_HC_POST with SIMDgroup register and shuffle optimized kernels. - Add Metal dispatch and support plumbing and test the production Sinkhorn iteration count and embedding width. Assisted-by: Codex Co-authored-by: Thiago Padilha --- ggml/src/ggml-metal/ggml-metal-device.cpp | 18 +++ ggml/src/ggml-metal/ggml-metal-device.h | 1 + ggml/src/ggml-metal/ggml-metal-device.m | 36 +++++ ggml/src/ggml-metal/ggml-metal-impl.h | 43 ++++++ ggml/src/ggml-metal/ggml-metal-ops.cpp | 137 +++++++++++++++++++ ggml/src/ggml-metal/ggml-metal-ops.h | 1 + ggml/src/ggml-metal/ggml-metal.metal | 159 ++++++++++++++++++++++ tests/test-backend-ops.cpp | 2 + 8 files changed, 397 insertions(+) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 16e98eb519ea..ec3ac5140e48 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -477,6 +477,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max(ggml_me return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, ggml_op op) { + const char * name = nullptr; + + switch (op) { + case GGML_OP_DSV4_HC_COMB: name = "kernel_dsv4_hc_comb_f32"; break; + case GGML_OP_DSV4_HC_PRE: name = "kernel_dsv4_hc_pre_f32"; break; + case GGML_OP_DSV4_HC_POST: name = "kernel_dsv4_hc_post_f32"; break; + default: GGML_ABORT("fatal error"); + } + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv(ggml_metal_library_t lib, const ggml_tensor * op) { GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32); GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 91b841b67b7d..2d712f0bfc64 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -124,6 +124,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_bl struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_add (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, enum ggml_op op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan (ggml_metal_library_t lib, const struct ggml_tensor * op); diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 7d2a686850d5..6bc60a0db7f7 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1299,6 +1299,42 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return false; } return has_simdgroup_mm; // TODO: over-restricted for vec-kernels + case GGML_OP_DSV4_HC_COMB: + return has_simdgroup_reduction && + op->src[0]->type == GGML_TYPE_F32 && + op->src[1]->type == GGML_TYPE_F32 && + op->src[2]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32 && + op->src[0]->ne[0] == 24 && + op->src[1]->ne[0] >= 3 && + op->src[2]->ne[0] == 24 && + ggml_is_contiguous_rows(op->src[0]) && + ggml_is_contiguous_rows(op->src[1]) && + ggml_is_contiguous_rows(op->src[2]); + case GGML_OP_DSV4_HC_PRE: + return has_simdgroup_reduction && + op->src[0]->type == GGML_TYPE_F32 && + op->src[1]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32 && + op->src[0]->ne[1] == 4 && + op->src[1]->ne[0] == 4 && + ggml_is_contiguous_rows(op->src[0]) && + ggml_is_contiguous_rows(op->src[1]); + case GGML_OP_DSV4_HC_POST: + return has_simdgroup_reduction && + op->src[0]->type == GGML_TYPE_F32 && + op->src[1]->type == GGML_TYPE_F32 && + op->src[2]->type == GGML_TYPE_F32 && + op->src[3]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32 && + op->src[1]->ne[1] == 4 && + op->src[2]->ne[0] == 4 && + op->src[3]->ne[0] == 4 && + op->src[3]->ne[1] == 4 && + ggml_is_contiguous_rows(op->src[0]) && + ggml_is_contiguous_rows(op->src[1]) && + ggml_is_contiguous_rows(op->src[2]) && + ggml_is_contiguous_rows(op->src[3]); case GGML_OP_SSM_CONV: case GGML_OP_SSM_SCAN: return has_simdgroup_reduction; diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 9f350aad5b74..0fa0c2d6ad08 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -1171,6 +1171,49 @@ typedef struct { int64_t val; } ggml_metal_kargs_memset; +typedef struct { + int32_t n_tokens; + int32_t n_iter; + uint64_t nb_m0; + uint64_t nb_m1; + uint64_t nb_s0; + uint64_t nb_b0; + uint64_t nb_d0; + uint64_t nb_d1; + uint64_t nb_d2; + float eps; +} ggml_metal_kargs_dsv4_hc_comb; + +typedef struct { + int32_t n_embd; + int32_t n_tokens; + uint64_t nb_x0; + uint64_t nb_x1; + uint64_t nb_x2; + uint64_t nb_w0; + uint64_t nb_w1; + uint64_t nb_d0; + uint64_t nb_d1; +} ggml_metal_kargs_dsv4_hc_pre; + +typedef struct { + int32_t n_embd; + int32_t n_tokens; + uint64_t nb_x0; + uint64_t nb_x1; + uint64_t nb_r0; + uint64_t nb_r1; + uint64_t nb_r2; + uint64_t nb_p0; + uint64_t nb_p1; + uint64_t nb_c0; + uint64_t nb_c1; + uint64_t nb_c2; + uint64_t nb_d0; + uint64_t nb_d1; + uint64_t nb_d2; +} ggml_metal_kargs_dsv4_hc_post; + typedef struct { int32_t ne00; int32_t ne01; diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 76626a451836..49f4ee37bb27 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -316,6 +316,12 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_cumsum(ctx, idx); } break; + case GGML_OP_DSV4_HC_COMB: + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + { + n_fuse = ggml_metal_op_dsv4_hc(ctx, idx); + } break; case GGML_OP_SOFT_MAX: { n_fuse = ggml_metal_op_soft_max(ctx, idx); @@ -1297,6 +1303,137 @@ int ggml_metal_op_diag(ggml_metal_op_t ctx, int idx) { return 1; } +int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_encoder_t enc = ctx->enc; + auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op->op); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + + switch (op->op) { + case GGML_OP_DSV4_HC_COMB: + { + const ggml_tensor * mixes = op->src[0]; + const ggml_tensor * scale = op->src[1]; + const ggml_tensor * base = op->src[2]; + + GGML_ASSERT(mixes->type == GGML_TYPE_F32); + GGML_ASSERT(scale->type == GGML_TYPE_F32); + GGML_ASSERT(base->type == GGML_TYPE_F32); + GGML_ASSERT(op->type == GGML_TYPE_F32); + GGML_ASSERT(mixes->ne[0] == 24); + GGML_ASSERT(op->ne[0] == 4 && op->ne[1] == 4); + + ggml_metal_kargs_dsv4_hc_comb args = { + /*.n_tokens =*/ (int32_t) mixes->ne[1], + /*.n_iter =*/ ggml_get_op_params_i32(op, 1), + /*.nb_m0 =*/ mixes->nb[0], + /*.nb_m1 =*/ mixes->nb[1], + /*.nb_s0 =*/ scale->nb[0], + /*.nb_b0 =*/ base->nb[0], + /*.nb_d0 =*/ op->nb[0], + /*.nb_d1 =*/ op->nb[1], + /*.nb_d2 =*/ op->nb[2], + /*.eps =*/ ggml_get_op_params_f32(op, 0), + }; + + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(mixes), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(scale), 2); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(base), 3); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4); + + // One SIMDgroup owns one 4x4 Sinkhorn matrix. Packing up to four + // independent tokens per threadgroup keeps both decode and prompt + // dispatches compact without any threadgroup-memory synchronization. + const int nsg = std::min(4, args.n_tokens); + ggml_metal_encoder_dispatch_threadgroups( + enc, (args.n_tokens + nsg - 1)/nsg, 1, 1, 32, nsg, 1); + } break; + case GGML_OP_DSV4_HC_PRE: + { + const ggml_tensor * x = op->src[0]; + const ggml_tensor * weights = op->src[1]; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(weights->type == GGML_TYPE_F32); + GGML_ASSERT(op->type == GGML_TYPE_F32); + GGML_ASSERT(x->ne[1] == 4); + + ggml_metal_kargs_dsv4_hc_pre args = { + /*.n_embd =*/ (int32_t) x->ne[0], + /*.n_tokens =*/ (int32_t) x->ne[2], + /*.nb_x0 =*/ x->nb[0], + /*.nb_x1 =*/ x->nb[1], + /*.nb_x2 =*/ x->nb[2], + /*.nb_w0 =*/ weights->nb[0], + /*.nb_w1 =*/ weights->nb[1], + /*.nb_d0 =*/ op->nb[0], + /*.nb_d1 =*/ op->nb[1], + }; + + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(weights), 2); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); + + const int n_tiles = (args.n_embd + 31)/32; + const int nsg = std::min(4, n_tiles); + ggml_metal_encoder_dispatch_threadgroups( + enc, (n_tiles + nsg - 1)/nsg, args.n_tokens, 1, 32, nsg, 1); + } break; + case GGML_OP_DSV4_HC_POST: + { + const ggml_tensor * x = op->src[0]; + const ggml_tensor * residual = op->src[1]; + const ggml_tensor * post = op->src[2]; + const ggml_tensor * comb = op->src[3]; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(residual->type == GGML_TYPE_F32); + GGML_ASSERT(post->type == GGML_TYPE_F32); + GGML_ASSERT(comb->type == GGML_TYPE_F32); + GGML_ASSERT(op->type == GGML_TYPE_F32); + GGML_ASSERT(residual->ne[1] == 4); + + ggml_metal_kargs_dsv4_hc_post args = { + /*.n_embd =*/ (int32_t) x->ne[0], + /*.n_tokens =*/ (int32_t) x->ne[1], + /*.nb_x0 =*/ x->nb[0], + /*.nb_x1 =*/ x->nb[1], + /*.nb_r0 =*/ residual->nb[0], + /*.nb_r1 =*/ residual->nb[1], + /*.nb_r2 =*/ residual->nb[2], + /*.nb_p0 =*/ post->nb[0], + /*.nb_p1 =*/ post->nb[1], + /*.nb_c0 =*/ comb->nb[0], + /*.nb_c1 =*/ comb->nb[1], + /*.nb_c2 =*/ comb->nb[2], + /*.nb_d0 =*/ op->nb[0], + /*.nb_d1 =*/ op->nb[1], + /*.nb_d2 =*/ op->nb[2], + }; + + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(residual), 2); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(post), 3); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(comb), 4); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5); + + const int n_tiles = (args.n_embd + 31)/32; + const int nsg = std::min(4, n_tiles); + ggml_metal_encoder_dispatch_threadgroups( + enc, (n_tiles + nsg - 1)/nsg, args.n_tokens, 1, 32, nsg, 1); + } break; + default: + GGML_ABORT("fatal error"); + } + + return 1; +} + int ggml_metal_op_soft_max(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index 2783ecb8b611..bd20bdfbcd69 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -54,6 +54,7 @@ int ggml_metal_op_cumsum (ggml_metal_op_t ctx, int idx); int ggml_metal_op_get_rows (ggml_metal_op_t ctx, int idx); int ggml_metal_op_set_rows (ggml_metal_op_t ctx, int idx); int ggml_metal_op_diag (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_dsv4_hc (ggml_metal_op_t ctx, int idx); int ggml_metal_op_soft_max (ggml_metal_op_t ctx, int idx); int ggml_metal_op_ssm_conv (ggml_metal_op_t ctx, int idx); int ggml_metal_op_ssm_scan (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index f14ee0792ba5..3c8c0ecd41eb 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -11278,3 +11278,162 @@ kernel void kernel_count_equal( typedef decltype(kernel_count_equal) kernel_count_equal_t; template [[host_name("kernel_count_equal_i32")]] kernel kernel_count_equal_t kernel_count_equal; + +kernel void kernel_dsv4_hc_comb_f32( + constant ggml_metal_kargs_dsv4_hc_comb & args, + device const char * mixes, + device const char * scale, + device const char * base, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constexpr ushort hc = 4; + constexpr ushort comb_offset = 2*hc; + + const int it = tgpig.x*ntg.y + sgitg; + if (it >= args.n_tokens) { + return; + } + + float scale_lane = 0.0f; + if (tiisg == 0) { + scale_lane = *(device const float *) (scale + 2*args.nb_s0); + } + const float scale_comb = simd_shuffle(scale_lane, 0); + + float v = 0.0f; + if (tiisg < hc*hc) { + v = *(device const float *) (mixes + (comb_offset + tiisg)*args.nb_m0 + it*args.nb_m1)*scale_comb + + *(device const float *) (base + (comb_offset + tiisg)*args.nb_b0); + } + + // Softmax across destinations (the four contiguous lanes for each source). + float vmax = max(v, simd_shuffle_xor(v, 1)); + vmax = max(vmax, simd_shuffle_xor(vmax, 2)); + v = exp(v - vmax); + + float sum = v + simd_shuffle_xor(v, 1); + sum += simd_shuffle_xor(sum, 2); + v = v/sum + args.eps; + + // Normalize columns: equal destination indices are four lanes apart. + sum = v + simd_shuffle_xor(v, 4); + sum += simd_shuffle_xor(sum, 8); + v /= sum + args.eps; + + for (int i = 1; i < args.n_iter; ++i) { + sum = v + simd_shuffle_xor(v, 1); + sum += simd_shuffle_xor(sum, 2); + v /= sum + args.eps; + + sum = v + simd_shuffle_xor(v, 4); + sum += simd_shuffle_xor(sum, 8); + v /= sum + args.eps; + } + + if (tiisg < hc*hc) { + const ushort idst = tiisg & 3; + const ushort isrc = tiisg >> 2; + *(device float *) (dst + idst*args.nb_d0 + isrc*args.nb_d1 + it*args.nb_d2) = v; + } +} + +kernel void kernel_dsv4_hc_pre_f32( + constant ggml_metal_kargs_dsv4_hc_pre & args, + device const char * x, + device const char * weights, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constexpr ushort hc = 4; + + const int it = tgpig.y; + const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; + + float weight_lane = 0.0f; + if (tiisg < hc) { + weight_lane = *(device const float *) (weights + tiisg*args.nb_w0 + it*args.nb_w1); + } + + float w[hc]; + FOR_UNROLL (ushort ih = 0; ih < hc; ++ih) { + w[ih] = simd_shuffle(weight_lane, ih); + } + + if (i0 >= args.n_embd) { + return; + } + + device const char * xb = x + i0*args.nb_x0 + it*args.nb_x2; + float result = 0.0f; + FOR_UNROLL (ushort ih = 0; ih < hc; ++ih) { + result = fma(*(device const float *) (xb + ih*args.nb_x1), w[ih], result); + } + + *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = result; +} + +kernel void kernel_dsv4_hc_post_f32( + constant ggml_metal_kargs_dsv4_hc_post & args, + device const char * x, + device const char * residual, + device const char * post, + device const char * comb, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constexpr ushort hc = 4; + + const int it = tgpig.y; + const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; + + float coeff_lane = 0.0f; + if (tiisg < hc) { + coeff_lane = *(device const float *) (post + tiisg*args.nb_p0 + it*args.nb_p1); + } else if (tiisg < hc + hc*hc) { + const ushort idx = tiisg - hc; + const ushort idst = idx & 3; + const ushort isrc = idx >> 2; + coeff_lane = *(device const float *) (comb + idst*args.nb_c0 + isrc*args.nb_c1 + it*args.nb_c2); + } + + float post_reg[hc]; + float comb_reg[hc][hc]; + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + post_reg[idst] = simd_shuffle(coeff_lane, idst); + } + FOR_UNROLL (ushort isrc = 0; isrc < hc; ++isrc) { + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + comb_reg[isrc][idst] = simd_shuffle(coeff_lane, hc + idst + hc*isrc); + } + } + + if (i0 >= args.n_embd) { + return; + } + + const float xv = *(device const float *) (x + i0*args.nb_x0 + it*args.nb_x1); + float result[hc]; + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + result[idst] = xv*post_reg[idst]; + } + + device const char * rb = residual + i0*args.nb_r0 + it*args.nb_r2; + FOR_UNROLL (ushort isrc = 0; isrc < hc; ++isrc) { + const float rv = *(device const float *) (rb + isrc*args.nb_r1); + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + result[idst] = fma(rv, comb_reg[isrc][idst], result[idst]); + } + } + + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = result[idst]; + } +} + diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 8f3fb45dab65..32468bf22d41 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8069,6 +8069,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_dsv4_hc_comb(1, 1)); test_cases.emplace_back(new test_dsv4_hc_comb(17, 4)); test_cases.emplace_back(new test_dsv4_hc_comb(257, 8)); + test_cases.emplace_back(new test_dsv4_hc_comb(17, 20)); test_cases.emplace_back(new test_dsv4_hc_pre(1, 1)); test_cases.emplace_back(new test_dsv4_hc_pre(31, 17)); @@ -8078,6 +8079,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_dsv4_hc_post(1, 1)); test_cases.emplace_back(new test_dsv4_hc_post(31, 17)); test_cases.emplace_back(new test_dsv4_hc_post(128, 257)); + test_cases.emplace_back(new test_dsv4_hc_post(4096, 21)); // glu ops for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { From 0ab9d6fed73dbc5dc8026c868cb10a6728c4ed48 Mon Sep 17 00:00:00 2001 From: mgroeber9110 <45620825+mgroeber9110@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:44:00 +0200 Subject: [PATCH 29/98] opencl: limit local workgroup size for GLU operation (#26383) --- ggml/src/ggml-opencl/ggml-opencl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 388c585fa121..915c8e7b903f 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -24260,7 +24260,7 @@ static void ggml_cl_glu(ggml_backend_t backend, const ggml_tensor * src0, const } const size_t nrows = ggml_nrows(src0); - size_t nth = 512; + size_t nth = backend_ctx->max_workgroup_size < 512 ? backend_ctx->max_workgroup_size : 512; size_t global_work_size[] = {nrows*nth, 1, 1}; size_t local_work_size[] = {nth, 1, 1}; From 9d21b57f2e8e526ae57f2e10d1b565bd8bd160de Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 2 Aug 2026 22:28:17 +0300 Subject: [PATCH 30/98] metal : add F16 support for bin ops (#26465) --- ggml/src/ggml-metal/ggml-metal-device.m | 1 + ggml/src/ggml-metal/ggml-metal-ops.cpp | 3 --- ggml/src/ggml-metal/ggml-metal.metal | 2 ++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 6bc60a0db7f7..20a2fdcfa443 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1181,6 +1181,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_MUL: case GGML_OP_DIV: case GGML_OP_ADD_ID: + return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && (op->src[0]->type == op->src[1]->type); case GGML_OP_ACC: return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_REPEAT: diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 49f4ee37bb27..caa162a5c92b 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -3334,9 +3334,6 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); - GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32); - GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); GGML_ASSERT(ggml_is_contiguous_rows(op->src[1])); diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 3c8c0ecd41eb..60442411930a 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -1418,6 +1418,8 @@ typedef decltype(kernel_bin_fuse_impl) kernel_bin_fuse_t; template [[host_name("kernel_bin_fuse_f32_f32_f32")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl; template [[host_name("kernel_bin_fuse_f32_f32_f32_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl; +template [[host_name("kernel_bin_fuse_f16_f16_f16")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl; +template [[host_name("kernel_bin_fuse_f16_f16_f16_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl; kernel void kernel_add_id( constant ggml_metal_kargs_add_id & args, From 221f0f6356efe2260023208365705ec5d5a7c8f5 Mon Sep 17 00:00:00 2001 From: Talha Adnan Date: Sun, 2 Aug 2026 14:39:28 -0500 Subject: [PATCH 31/98] metal : add SILU_BACK (#25982) * feat(silu_back): implemented silu_back op for f32 * fix(silu_back): removed redundant asserts in ggml-metal-ops.cpp function ggml_metal_op_silu_back. --- ggml/src/ggml-metal/ggml-metal-device.cpp | 17 ++++++++++++ ggml/src/ggml-metal/ggml-metal-device.h | 1 + ggml/src/ggml-metal/ggml-metal-device.m | 8 ++++++ ggml/src/ggml-metal/ggml-metal-impl.h | 4 +++ ggml/src/ggml-metal/ggml-metal-ops.cpp | 34 +++++++++++++++++++++++ ggml/src/ggml-metal/ggml-metal-ops.h | 1 + ggml/src/ggml-metal/ggml-metal.metal | 14 ++++++++++ 7 files changed, 79 insertions(+) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index ec3ac5140e48..e5ef797555cb 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -2135,6 +2135,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_sgd(ggm return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_silu_back(ggml_metal_library_t lib, const ggml_tensor * op) { + assert(op->op == GGML_OP_SILU_BACK); + + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_silu_back_%s", ggml_type_name(op->src[0]->type)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_memset(ggml_metal_library_t lib, const ggml_tensor * op) { GGML_ASSERT(op->type == GGML_TYPE_I64); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 2d712f0bfc64..63e6d0c8aa07 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -117,6 +117,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_diag struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat (ggml_metal_library_t lib, enum ggml_type tsrc); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat (ggml_metal_library_t lib, enum ggml_type tsrc); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_silu_back (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum_rows (ggml_metal_library_t lib, const struct ggml_tensor * op); diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 20a2fdcfa443..827d7ca32087 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1137,6 +1137,14 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te default: return false; } + case GGML_OP_SILU_BACK: + return (op->src[0]->type == GGML_TYPE_F32) && + (op->src[1]->type == GGML_TYPE_F32) && + (op->type == GGML_TYPE_F32) && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op->src[1]) && + ggml_is_contiguous(op) && + ggml_are_same_shape(op->src[0], op->src[1]); case GGML_OP_GLU: switch (ggml_get_glu_op(op)) { case GGML_GLU_OP_REGLU: diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 0fa0c2d6ad08..8e8926cda4b7 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -1265,4 +1265,8 @@ typedef struct { int64_t np; } ggml_metal_kargs_opt_step_sgd; +typedef struct { + int64_t ne; +} ggml_metal_kargs_silu_back; + #endif // GGML_METAL_IMPL diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index caa162a5c92b..2d84804b9217 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -299,6 +299,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_unary(ctx, idx); } break; + case GGML_OP_SILU_BACK: + { + n_fuse = ggml_metal_op_silu_back(ctx, idx); + } break; case GGML_OP_GLU: { n_fuse = ggml_metal_op_glu(ctx, idx); @@ -3473,6 +3477,36 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { return n_fuse; } +int ggml_metal_op_silu_back(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + auto pipeline = ggml_metal_library_get_pipeline_silu_back(lib, op); + + const int64_t ne = ggml_nelements(op); + + ggml_metal_kargs_silu_back args = { + /*.ne =*/ ne, + }; + + int arg_idx{0}; + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), arg_idx++); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), arg_idx++); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), arg_idx++); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), arg_idx++); + + const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne); + const int64_t n = (ne + nth - 1) / nth; + + ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, nth, 1, 1); + + return 1; +} + int ggml_metal_op_l2_norm(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index bd20bdfbcd69..4fdad1dd59d2 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -71,6 +71,7 @@ int ggml_metal_op_mul_mat_id (ggml_metal_op_t ctx, int idx); int ggml_metal_op_add_id (ggml_metal_op_t ctx, int idx); int ggml_metal_op_flash_attn_ext (ggml_metal_op_t ctx, int idx); int ggml_metal_op_bin (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_silu_back (ggml_metal_op_t ctx, int idx); int ggml_metal_op_l2_norm (ggml_metal_op_t ctx, int idx); int ggml_metal_op_group_norm (ggml_metal_op_t ctx, int idx); int ggml_metal_op_norm (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 60442411930a..47d4f685a020 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -1255,6 +1255,20 @@ template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_un template [[host_name("kernel_unary_f16_f16")]] kernel kernel_unary_t kernel_unary_impl; template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl; +kernel void kernel_silu_back_f32( + constant ggml_metal_kargs_silu_back & args, + device const float * dy, + device const float * x, + device float * dx, + uint gid [[thread_position_in_grid]]) { + if (gid >= args.ne) { + return; + } + + const float s = 1.0f / (1.0f + exp(-x[gid])); + dx[gid] = dy[gid] * s * (1.0f + x[gid] * (1.0f - s)); +} + // OP: 0 - add, 1 - sub, 2 - mul, 3 - div constant short FC_bin_op [[function_constant(FC_BIN + 0)]]; constant short FC_bin_f [[function_constant(FC_BIN + 1)]]; From 1464c62d88f699ec9700c8010bbfdbc603a9efd6 Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Mon, 3 Aug 2026 01:33:37 -0300 Subject: [PATCH 32/98] metal: implement DSv4 Lightning Indexer (#25893) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * metal: implement F16 Lightning Indexer - Implement GGML_OP_LIGHTNING_INDEXER for 128-dimensional, 64-head inputs with F32 queries and weights plus F16 keys and masks. - Add tiled and tail kernels and test KV lengths around 8- and 64-element boundaries. llama-bench (--mmap 1, -fa 1, -p 512, -n 128; d=0/10k/20k/30k): Before: - pp512: 153.73 ± 0.87 t/s - tg128: 8.91 ± 0.04 t/s - pp512 @ d10000: 73.90 ± 0.39 t/s - tg128 @ d10000: 8.66 ± 0.03 t/s - pp512 @ d20000: 45.83 ± 0.18 t/s - tg128 @ d20000: 8.26 ± 0.03 t/s - pp512 @ d30000: 33.40 ± 0.21 t/s - tg128 @ d30000: 7.94 ± 0.01 t/s After: - pp512: 155.19 ± 0.91 t/s - tg128: 8.95 ± 0.04 t/s - pp512 @ d10000: 86.95 ± 0.69 t/s - tg128 @ d10000: 9.00 ± 0.05 t/s - pp512 @ d20000: 62.01 ± 0.45 t/s - tg128 @ d20000: 8.68 ± 0.04 t/s - pp512 @ d30000: 49.18 ± 0.33 t/s - tg128 @ d30000: 8.60 ± 0.02 t/s Assisted-by: Codex * metal: stage Lightning Indexer K tiles - Stage and dequantize K in F16 threadgroup memory before simdgroup matrix loads. - Zero-fill partial tiles and guard stores so all KV segments use the same numerical path. - Support F32, F16, BF16, Q4_0, Q4_1, Q5_0, Q5_1, and Q8_0 K caches. llama-bench (--mmap 1, -fa on, -p 512, -n 128; d=0/10k/20k): - pp512: 160.38 +/- 1.01 t/s - tg128: 9.08 +/- 0.03 t/s - pp512 @ d10000: 88.37 +/- 0.46 t/s - tg128 @ d10000: 9.07 +/- 0.04 t/s - pp512 @ d20000: 62.53 +/- 0.46 t/s - tg128 @ d20000: 8.84 +/- 0.03 t/s Assisted-by: Codex * dedup Lightning Indexer constants, fix flaky test * cont : fix whitespace --------- Co-authored-by: forforever73 <690105611@qq.com> Co-authored-by: Georgi Gerganov --- ggml/src/ggml-metal/ggml-metal-device.cpp | 17 +++ ggml/src/ggml-metal/ggml-metal-device.h | 1 + ggml/src/ggml-metal/ggml-metal-device.m | 31 +++++ ggml/src/ggml-metal/ggml-metal-impl.h | 24 ++++ ggml/src/ggml-metal/ggml-metal-ops.cpp | 70 ++++++++++ ggml/src/ggml-metal/ggml-metal-ops.h | 1 + ggml/src/ggml-metal/ggml-metal.metal | 150 +++++++++++++++++++++- tests/test-backend-ops.cpp | 6 + 8 files changed, 299 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index e5ef797555cb..c153bd82177b 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -477,6 +477,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max(ggml_me return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer( + ggml_metal_library_t lib, + const ggml_tensor * op) { + GGML_ASSERT(op->op == GGML_OP_LIGHTNING_INDEXER); + + char name[256]; + + snprintf(name, 256, "kernel_lightning_indexer_%s", ggml_type_name(op->src[1]->type)); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, ggml_op op) { const char * name = nullptr; diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 63e6d0c8aa07..7e1deeaa2102 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -125,6 +125,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_bl struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_add (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, enum ggml_op op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs); diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 827d7ca32087..2dc6eb8fdbc0 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -2,6 +2,7 @@ #import "ggml-impl.h" #import "ggml-backend-impl.h" +#import "ggml-metal-impl.h" #include @@ -1308,6 +1309,36 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return false; } return has_simdgroup_mm; // TODO: over-restricted for vec-kernels + case GGML_OP_LIGHTNING_INDEXER: + if (op->src[0]->ne[0] != OP_LIGHTNING_INDEXER_DK || + op->src[0]->ne[1] != OP_LIGHTNING_INDEXER_NH) { + return false; + } + if (!has_simdgroup_mm || + op->src[0]->type != GGML_TYPE_F32 || + op->src[2]->type != GGML_TYPE_F32 || + op->src[3]->type != GGML_TYPE_F16 || + op->type != GGML_TYPE_F32 || + !ggml_is_contiguous_rows(op->src[0]) || + !ggml_is_contiguous_rows(op->src[1]) || + !ggml_is_contiguous_rows(op->src[2]) || + !ggml_is_contiguous_rows(op->src[3])) { + return false; + } + switch (op->src[1]->type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return true; + case GGML_TYPE_BF16: + return has_bfloat; + default: + return false; + } case GGML_OP_DSV4_HC_COMB: return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32 && diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 8e8926cda4b7..e173b91c0c5c 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -112,6 +112,13 @@ #define OP_FLASH_ATTN_EXT_VEC_NQPSG 1 #define OP_FLASH_ATTN_EXT_VEC_NCPSG 32 +#define OP_LIGHTNING_INDEXER_DK 128 +#define OP_LIGHTNING_INDEXER_NH 64 +#define OP_LIGHTNING_INDEXER_NHPTG 8 +#define OP_LIGHTNING_INDEXER_NKPSG 8 +#define OP_LIGHTNING_INDEXER_NSG 8 +#define OP_LIGHTNING_INDEXER_NBPTG 8 + #define OP_UNARY_NUM_SCALE 10 #define OP_UNARY_NUM_FILL 11 #define OP_UNARY_NUM_CLAMP 12 @@ -1171,6 +1178,23 @@ typedef struct { int64_t val; } ggml_metal_kargs_memset; +typedef struct { + int32_t n_kv; + int32_t n_batch; + int32_t mask_ne3; + uint64_t nb1; + uint64_t nb3; + uint64_t nbq1; + uint64_t nbq2; + uint64_t nbq3; + uint64_t nbk2; + uint64_t nbk3; + uint64_t nbw1; + uint64_t nbw3; + uint64_t nbm1; + uint64_t nbm3; +} ggml_metal_kargs_lightning_indexer; + typedef struct { int32_t n_tokens; int32_t n_iter; diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 2d84804b9217..c5d7619c12fa 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -320,6 +320,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_cumsum(ctx, idx); } break; + case GGML_OP_LIGHTNING_INDEXER: + { + n_fuse = ggml_metal_op_lightning_indexer(ctx, idx); + } break; case GGML_OP_DSV4_HC_COMB: case GGML_OP_DSV4_HC_PRE: case GGML_OP_DSV4_HC_POST: @@ -1307,6 +1311,72 @@ int ggml_metal_op_diag(ggml_metal_op_t ctx, int idx) { return 1; } +int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_encoder_t enc = ctx->enc; + + GGML_ASSERT(op->op == GGML_OP_LIGHTNING_INDEXER); + + const ggml_tensor * q = op->src[0]; + const ggml_tensor * k = op->src[1]; + const ggml_tensor * w = op->src[2]; + const ggml_tensor * m = op->src[3]; + + GGML_ASSERT(q->type == GGML_TYPE_F32); + GGML_ASSERT(k->type == GGML_TYPE_F32 || + k->type == GGML_TYPE_F16 || + k->type == GGML_TYPE_BF16 || + k->type == GGML_TYPE_Q4_0 || + k->type == GGML_TYPE_Q4_1 || + k->type == GGML_TYPE_Q5_0 || + k->type == GGML_TYPE_Q5_1 || + k->type == GGML_TYPE_Q8_0); + GGML_ASSERT(w->type == GGML_TYPE_F32); + GGML_ASSERT(m->type == GGML_TYPE_F16); + GGML_ASSERT(op->type == GGML_TYPE_F32); + + GGML_ASSERT(q->ne[0] == OP_LIGHTNING_INDEXER_DK); + GGML_ASSERT(q->ne[1] == OP_LIGHTNING_INDEXER_NH); + + ggml_metal_kargs_lightning_indexer args = { + /*.n_kv =*/ (int32_t) k->ne[2], + /*.n_batch =*/ (int32_t) q->ne[2], + /*.mask_ne3 =*/ (int32_t) m->ne[3], + /*.nb1 =*/ op->nb[1], + /*.nb3 =*/ op->nb[3], + /*.nbq1 =*/ q->nb[1], + /*.nbq2 =*/ q->nb[2], + /*.nbq3 =*/ q->nb[3], + /*.nbk2 =*/ k->nb[2], + /*.nbk3 =*/ k->nb[3], + /*.nbw1 =*/ w->nb[1], + /*.nbw3 =*/ w->nb[3], + /*.nbm1 =*/ m->nb[1], + /*.nbm3 =*/ m->nb[3], + }; + + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(q), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(k), 2); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(w), 3); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(m), 4); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5); + + const int nsg = OP_LIGHTNING_INDEXER_NSG; + const int nkptg = OP_LIGHTNING_INDEXER_NKPSG*nsg; + const int nbptg = OP_LIGHTNING_INDEXER_NBPTG; + + auto pipeline = ggml_metal_library_get_pipeline_lightning_indexer(ctx->lib, op); + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); + ggml_metal_encoder_dispatch_threadgroups(enc, + (k->ne[2] + nkptg - 1)/nkptg, + (q->ne[2] + nbptg - 1)/nbptg, + q->ne[3], 32, nsg, 1); + + return 1; +} + int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index 4fdad1dd59d2..b03b59e0bd92 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -54,6 +54,7 @@ int ggml_metal_op_cumsum (ggml_metal_op_t ctx, int idx); int ggml_metal_op_get_rows (ggml_metal_op_t ctx, int idx); int ggml_metal_op_set_rows (ggml_metal_op_t ctx, int idx); int ggml_metal_op_diag (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_lightning_indexer (ggml_metal_op_t ctx, int idx); int ggml_metal_op_dsv4_hc (ggml_metal_op_t ctx, int idx); int ggml_metal_op_soft_max (ggml_metal_op_t ctx, int idx); int ggml_metal_op_ssm_conv (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 47d4f685a020..7d12cb0fe39b 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -11295,6 +11295,155 @@ typedef decltype(kernel_count_equal) kernel_count_equal_t; template [[host_name("kernel_count_equal_i32")]] kernel kernel_count_equal_t kernel_count_equal; +template< + typename kd4x4_t, + short nl_k, + void (*deq_k)(device const kd4x4_t *, short, thread half4x4 &)> +kernel void kernel_lightning_indexer( + constant ggml_metal_kargs_lightning_indexer & args, + device const char * q, + device const char * k, + device const char * w, + device const char * m, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + constexpr short DK = OP_LIGHTNING_INDEXER_DK; + constexpr short NH = OP_LIGHTNING_INDEXER_NH; + constexpr short NHPTG = OP_LIGHTNING_INDEXER_NHPTG; + constexpr short NKPSG = OP_LIGHTNING_INDEXER_NKPSG; + constexpr short NSG = OP_LIGHTNING_INDEXER_NSG; + constexpr short NBPTG = OP_LIGHTNING_INDEXER_NBPTG; + + constexpr short DK4 = DK/4; + constexpr short DK8 = DK/8; + constexpr short DK16 = DK/16; + + constexpr short NK = NKPSG*NSG; // keys per threadgroup + constexpr short NTG = 32*NSG; // threads per threadgroup + + const int i_stream = tgpig.z; + const int i_kv_0 = tgpig.x*NK; // first key of this threadgroup + const int i_kv = i_kv_0 + sgitg*NKPSG; // first key of this simdgroup + + threadgroup half4x4 sk4x4[NK*DK16]; + threadgroup half * sk = (threadgroup half *) sk4x4; + + for (short i = tiitg; i < NK*DK16; i += NTG) { + const short ik = i/DK16; + const short i16 = i%DK16; + + half4x4 tmp; + + if (i_kv_0 + ik < args.n_kv) { + device const kd4x4_t * kr = (device const kd4x4_t *) (k + (i_kv_0 + ik)*args.nbk2 + i_stream*args.nbk3); + + deq_k(kr + i16/nl_k, i16%nl_k, tmp); + } else { + FOR_UNROLL (short j = 0; j < 4; ++j) { + tmp[j] = half4(0.0h); + } + } + + sk4x4[i] = tmp; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // K tile of this simdgroup, transposed to [DK, NKPSG] + simdgroup_half8x8 mk[DK8]; + + FOR_UNROLL (short i = 0; i < DK8; ++i) { + simdgroup_load(mk[i], sk + sgitg*NKPSG*DK + 8*i, DK, 0, true); + } + + threadgroup half4 sq4[NHPTG*DK4]; + threadgroup half * sq = (threadgroup half *) sq4; + + threadgroup float sw [NHPTG]; + threadgroup float sqk[NSG*NHPTG*NKPSG]; + + const int i_batch_0 = tgpig.y*NBPTG; + const int n_batch = min((int) NBPTG, args.n_batch - i_batch_0); + + for (short ib = 0; ib < n_batch; ++ib) { + const int i_batch = i_batch_0 + ib; + + device const char * pq = q + i_batch*args.nbq2 + i_stream*args.nbq3; + device const char * pw = w + i_batch*args.nbw1 + i_stream*args.nbw3; + + float score = 0.0f; + + FOR_UNROLL (short i_head = 0; i_head < NH; i_head += NHPTG) { + // stage the Q tile [DK, NHPTG] and the (prescaled) head weights + for (short i = tiitg; i < NHPTG*DK4; i += NTG) { + const short ih = i/DK4; + const short i4 = i%DK4; + + device const float4 * q4 = (device const float4 *) (pq + (i_head + ih)*args.nbq1); + + sq4[ih*DK4 + i4] = half4(q4[i4]); + } + + if (tiitg < NHPTG) { + sw[tiitg] = ((device const float *) pw)[i_head + tiitg]; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + simdgroup_float8x8 mqk = make_filled_simdgroup_matrix(0.0f); + + FOR_UNROLL (short i = 0; i < DK8; ++i) { + simdgroup_half8x8 mq; + + simdgroup_load(mq, sq + 8*i, DK, 0, false); + simdgroup_multiply_accumulate(mqk, mq, mk[i], mqk); + } + + threadgroup float * pqk = sqk + sgitg*NHPTG*NKPSG; + + simdgroup_store(mqk, pqk, NKPSG, 0, false); + simdgroup_barrier(mem_flags::mem_threadgroup); + + // one lane per key: ReLU, apply the head weight and accumulate over the head tile + if (tiisg < NKPSG) { + FOR_UNROLL (short ih = 0; ih < NHPTG; ++ih) { + score += max(pqk[ih*NKPSG + tiisg], 0.0f)*sw[ih]; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + if (tiisg < NKPSG) { + const int ik = i_kv + tiisg; + if (ik < args.n_kv) { + device const half * pm = (device const half *) (m + i_batch*args.nbm1 + (i_stream % args.mask_ne3)*args.nbm3); + device float * pd = (device float *) (dst + i_batch*args.nb1 + i_stream*args.nb3); + + pd[ik] = score + (float) pm[ik]; + } + } + } +} + +typedef decltype(kernel_lightning_indexer) kernel_lightning_indexer_t; + +template [[host_name("kernel_lightning_indexer_f32")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +template [[host_name("kernel_lightning_indexer_f16")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; + +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_lightning_indexer_bf16")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +#endif + +template [[host_name("kernel_lightning_indexer_q4_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +template [[host_name("kernel_lightning_indexer_q4_1")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +template [[host_name("kernel_lightning_indexer_q5_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +template [[host_name("kernel_lightning_indexer_q5_1")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +template [[host_name("kernel_lightning_indexer_q8_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; + kernel void kernel_dsv4_hc_comb_f32( constant ggml_metal_kargs_dsv4_hc_comb & args, device const char * mixes, @@ -11452,4 +11601,3 @@ kernel void kernel_dsv4_hc_post_f32( *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = result[idst]; } } - diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 32468bf22d41..8cb598935861 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9731,6 +9731,12 @@ static std::vector> make_test_cases_eval() { } } + for (int kv : { 1, 7, 8, 63, 64, 65 }) { + for (ggml_type type_K : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0}) { + test_cases.emplace_back(new test_lightning_indexer(128, 64, kv, 32, 4, 1, type_K)); + } + } + return test_cases; } #ifdef _MSC_VER From 2b63e0610bbc2be990ae1360d5256efcdc3f9efb Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Mon, 3 Aug 2026 08:25:01 +0200 Subject: [PATCH 33/98] llama : MTP support for DeepSeek V3.2 (#26457) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * llama : MTP support for DeepSeek V3.2 * model : no need to include MTP layers during DeepSeek V3.2 model type discovery --------- Co-authored-by: Stanisław Szymczyk --- conversion/deepseek.py | 35 ++++- src/llama-model.cpp | 20 +-- src/models/deepseek32.cpp | 279 ++++++++++++++++++++++++++++++++++++-- src/models/models.h | 4 + 4 files changed, 308 insertions(+), 30 deletions(-) diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 5b69e23437f3..1c9b325d5e6a 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -447,12 +447,43 @@ def prepare_tensors(self): class DeepseekV32Model(DeepseekV2Model): model_arch = gguf.MODEL_ARCH.DEEPSEEK32 skip_mtp = False + supports_mtp_export = True + _n_main_layers: int | None = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0) + self.block_count = self.hparams["num_hidden_layers"] + if not self.no_mtp: + self.block_count += self.hparams.get("num_nextn_predict_layers", 0) self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + def index_tensors(self, remote_hf_model_id: str | None = None): + type(self)._n_main_layers = self.hparams["num_hidden_layers"] + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem + + # DeepSeek V3.2 appends the NextN/MTP block past num_hidden_layers + # (model.layers.61 -> blk.61 in the 62-block file). + assert cls._n_main_layers is not None + is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers + + # --no-mtp: drop the appended NextN block entirely. + if is_mtp and cls.no_mtp: + return None + # --mtp: keep ONLY NextN-block tensors plus the shared embeddings/ + # norm/lm_head (so the resulting GGUF carries just the draft head). + if cls.mtp_only and not is_mtp and name not in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + ): + return None + + return name, gen + def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) @@ -463,7 +494,7 @@ def set_gguf_parameters(self): super().set_gguf_parameters() # NextN/MTP prediction layers - if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None: + if not self.no_mtp and (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None: self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers) # DSA indexer parameters diff --git a/src/llama-model.cpp b/src/llama-model.cpp index e93641b63ddc..13023c643f4e 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2071,24 +2071,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, { res = nullptr; } break; - case LLM_ARCH_DEEPSEEK32: - { - res = new llama_kv_cache_dsa( - *this, - params.type_k, - params.type_v, - !cparams.flash_attn, - cparams.offload_kqv, - cparams.kv_unified, - cparams.n_ctx_seq, - cparams.n_seq_max, - 1, - hparams.n_swa, - hparams.swa_type, - nullptr, - nullptr); - } break; case LLM_ARCH_GLM_DSA: + case LLM_ARCH_DEEPSEEK32: { if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && hparams.n_layer_nextn > 0) { // The NextN/MTP draft head runs dense MLA (no DSA indexer), so the @@ -2313,7 +2297,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, } if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA || - arch == LLM_ARCH_MIMO2) && + arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_DEEPSEEK32) && hparams.n_layer_nextn > 0) { if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp index 32262e6840b0..8a07a0b71cae 100644 --- a/src/models/deepseek32.cpp +++ b/src/models/deepseek32.cpp @@ -44,13 +44,24 @@ void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) { GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer"); switch (hparams.n_layer()) { - case 62: type = LLM_TYPE_685B_A37B; break; + case 61: type = LLM_TYPE_685B_A37B; break; default: type = LLM_TYPE_UNKNOWN; } } -void llama_model_deepseek32::load_arch_tensors(llama_model_loader &) { +void llama_model_deepseek32::load_arch_tensors(llama_model_loader & ml) { LLAMA_LOAD_LOCALS; + + const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); + const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; + const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); + const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } + const bool is_mla = hparams.is_mla(); if (!is_mla) { throw std::runtime_error("DEEPSEEK32 architecture requires MLA"); @@ -80,12 +91,7 @@ void llama_model_deepseek32::load_arch_tensors(llama_model_loader &) { } for (int i = 0; i < n_layer_all; ++i) { - int flags = 0; - if (i >= n_layer) { - // skip all tensors in the NextN layers - // TODO @ngxson : TENSOR_NOT_REQUIRED was a hack, need to remove it later - flags |= TENSOR_SKIP | TENSOR_NOT_REQUIRED; - } + const int flags = (i >= n_layer) ? mtp_flags : trunk_flags; auto & layer = layers[i]; @@ -138,7 +144,7 @@ void llama_model_deepseek32::load_arch_tensors(llama_model_loader &) { layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); } - // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers + // NextN/MTP tensors - conditionally load for last nextn_predict_layers if (i >= n_layer) { layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags); layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags); @@ -153,6 +159,9 @@ void llama_model_deepseek32::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_deepseek32::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique(*this, params); + } return std::make_unique(*this, params); } @@ -430,7 +439,9 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, top_k, kq_scale, il); } } - if (il == n_layer - 1 && inp_out_ids) { + // when unmasked nextn embeddings are requested, t_h_nextn must keep all rows, + // so the early output masking has to be skipped (it is applied after the final norm instead) + if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); } @@ -493,6 +504,14 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + // post-norm hidden state feeds the NextN/MTP draft head + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + cb(cur, "result_norm", -1); res->t_embd = cur; @@ -504,3 +523,243 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ ggml_build_forward_expand(gf, cur); } + +// LLM_GRAPH_TYPE_DECODER_MTP draft head for DeepSeek V3.2 (DEEPSEEK32). +// Semantics mirror the deepseek-family NextN/MTP layer: +// enorm(embed) + hnorm(prev_hidden) -> concat(e, h) -> eh_proj -> +// full deepseek32 decoder block (dense MLA attention + sigmoid-gated MoE FFN +// with shared expert, exactly as the trunk deepseek2 graph builds it) -> +// shared_head_norm (fallback output_norm) -> shared LM head. +// The DSA indexer is not used at runtime. +llama_model_deepseek32::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) + : llm_graph_context(params) { + GGML_ASSERT(hparams.n_layer_nextn > 0 && "DEEPSEEK32 MTP requires n_layer_nextn > 0"); + GGML_ASSERT(hparams.n_layer_nextn == 1 && "DEEPSEEK32 MTP currently only supports a single MTP block"); + GGML_ASSERT(hparams.is_mla() && "DEEPSEEK32 MTP requires MLA"); + + const int il = hparams.n_layer() + cparams.nextn_layer_offset; + GGML_ASSERT(cparams.nextn_layer_offset >= 0 && + cparams.nextn_layer_offset < (int) hparams.n_layer_nextn && + "nextn_layer_offset out of range [0, n_layer_nextn)"); + const auto & layer = model.layers[il]; + + GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj"); + GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm"); + GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); + GGML_ASSERT(layer.ffn_gate_inp && "MTP block missing ffn_gate_inp"); + + // note: these are the actual head sizes you get when treating as MHA or after "decompression" using wv_b for MLA + const int64_t n_embd_head_k = hparams.n_embd_head_k_mla(); + + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k - n_embd_head_qk_rope; + + const uint32_t kv_lora_rank = hparams.n_lora_kv; + + // We have to pre-scale kq_scale and attn_factor to make the YaRN RoPE work correctly. + // See the deepseek2 trunk graph for the detailed explanation - this must match it EXACTLY. + GGML_ASSERT(ext_factor >= 0.0f); + const float attn_factor_org = attn_factor * (1.0f + 0.1f * logf(1.0f / freq_scale)); + + const float mscale = attn_factor_org * (1.0f + 0.1f * hparams.rope_yarn_log_mul * logf(1.0f / freq_scale)); + const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k)); + + // TODO: extract in a common llm_graph_context::build_inp_embd_h() + auto inp = std::make_unique(hparams.n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); + ggml_set_input(inp->embd); + + ggml_tensor * tok_embd; + if (ubatch.token) { + ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; + + tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens); + } else { + tok_embd = inp->embd; + } + cb(tok_embd, "mtp_tok_embd", il); + + inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); + ggml_set_input(inp->h); + ggml_set_name(inp->h, "mtp_h_input"); + + ggml_tensor * h_embd = inp->h; + + res->add_input(std::move(inp)); + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // MLA with the absorption optimization uses a K-only cache (V is a view of K) + auto * inp_attn = build_attn_inp_k(); + + ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + + ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il); + cb(e_norm, "mtp_enorm", il); + + ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0); + cb(concat, "mtp_concat", il); + + ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s); + cb(cur, "mtp_eh_proj", il); + + ggml_tensor * inpSA = cur; + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_norm", il); + + // self-attention: dense MLA, same construction as the deepseek2 trunk graph + { + ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_a, cur); + cb(q, "mtp_q", il); + + q = build_norm(q, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(q, "mtp_q", il); + + q = ggml_mul_mat(ctx0, layer.wq_b, q); + cb(q, "mtp_q", il); + + // split into {n_embd_head_qk_nope, n_head, n_tokens} + ggml_tensor * q_nope = + ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), + ggml_row_size(q->type, n_embd_head_k) * n_head, 0); + cb(q_nope, "mtp_q_nope", il); + + // and {n_embd_head_qk_rope, n_head, n_tokens} + ggml_tensor * q_pe = ggml_view_3d( + ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), + ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "mtp_q_pe", il); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur); + cb(kv_cmpr_pe, "mtp_kv_cmpr_pe", il); + + // split into {kv_lora_rank, n_tokens} + ggml_tensor * kv_cmpr = + ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + cb(kv_cmpr, "mtp_kv_cmpr", il); + + // and {n_embd_head_qk_rope, 1, n_tokens} + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + cb(k_pe, "mtp_k_pe", il); + + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(q_pe, "mtp_q_pe", il); + + k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(k_pe, "mtp_k_pe", il); + + kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + cb(kv_cmpr, "mtp_kv_cmpr", il); + + // {n_embd_head_qk_nope, n_tokens, n_head} + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + cb(q_nope, "mtp_q_nope_perm", il); + + // {n_embd_head_qk_nope, kv_lora_rank, n_head} x {n_embd_head_qk_nope, n_tokens, n_head} + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope); + cb(q_nope_absorbed, "mtp_q_nope_absorbed", il); + + // {kv_lora_rank, n_head, n_tokens} + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + cb(q_nope_absorbed, "mtp_q_nope_absorbed_perm", il); + + // {n_embd_head_qk_rope + kv_lora_rank, n_head, n_tokens} + // note: rope must go first for in-place context shifting in build_rope_shift() + ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + cb(Qcur, "mtp_Qcur", il); + + kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens); + cb(kv_cmpr, "mtp_kv_cmpr_reshape", il); + + // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens} + ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); + cb(Kcur, "mtp_Kcur", il); + + // {kv_lora_rank, 1, n_tokens} + ggml_tensor * Vcur = kv_cmpr; + cb(Vcur, "mtp_Vcur", il); + + // note: MLA with the absorption optimization converts into MQA (ie: GQA with 1 group) + cur = build_attn(inp_attn, + layer.wo, NULL, layer.wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale, il); + cb(cur, "mtp_attn_out", il); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "mtp_ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "mtp_ffn_norm", il); + + // MoE FFN with shared expert - same construction as the deepseek2 trunk graph + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, + layer.ffn_gate_up_exps, + layer.ffn_up_exps_s, + layer.ffn_gate_exps_s, + layer.ffn_down_exps_s); + cb(moe_out, "mtp_ffn_moe_out", il); + + // FFN shared expert + ggml_tensor * ffn_shexp = + build_ffn(cur, + layer.ffn_up_shexp, NULL, layer.ffn_up_shexp_s, + layer.ffn_gate_shexp, NULL, layer.ffn_gate_shexp_s, + layer.ffn_down_shexp, NULL, layer.ffn_down_shexp_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "mtp_ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "mtp_ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "mtp_post_ffn", il); + + // shared_head_norm applied after the decoder block, before the shared LM head. + // The post-norm hidden state seeds the next MTP step. + ggml_tensor * head_norm_w = layer.nextn.shared_head_norm + ? layer.nextn.shared_head_norm + : model.output_norm; + GGML_ASSERT(head_norm_w && "DEEPSEEK32 MTP: missing both nextn.shared_head_norm and output_norm"); + cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1); + + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + cb(cur, "mtp_shared_head_norm", -1); + + ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output; + ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s; + GGML_ASSERT(head_w && "DEEPSEEK32 MTP: missing LM head (nextn.shared_head_head or model.output)"); + cur = build_lora_mm(head_w, cur, head_s); + cb(cur, "result_output", -1); + + res->t_logits = cur; + ggml_build_forward_expand(gf, cur); +} + diff --git a/src/models/models.h b/src/models/models.h index 2d5de5d432ee..55072f17e97b 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1097,6 +1097,10 @@ struct llama_model_deepseek32 : public llama_model_base { graph(const llama_model & model, const llm_graph_params & params); }; + struct graph_mtp : public llm_graph_context { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; From 4ed2b13f758ea467282cf0b0e1a938fca7f51211 Mon Sep 17 00:00:00 2001 From: Masashi Yoshimura Date: Mon, 3 Aug 2026 17:15:01 +0900 Subject: [PATCH 34/98] model: MTP support for Qwen3-Next (#25589) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * mtp for qwen3nex * fix for python type-check * Fix to compute num_mtp from directly mtp layer * define opt_num_mtp_layers in _QwenMtpMixin and fix some comments * Fix for python type check * Update gguf-py/gguf/constants.py Co-authored-by: Sigbjørn Skjæret * rebase and add load_mtp flags * Update src/models/qwen3next.cpp Co-authored-by: Sigbjørn Skjæret * Update src/models/qwen3next.cpp Co-authored-by: Sigbjørn Skjæret --------- Co-authored-by: Sigbjørn Skjæret --- conversion/qwen.py | 193 +++++++++++------------ gguf-py/gguf/constants.py | 8 +- src/llama-model.cpp | 12 +- src/models/models.h | 4 + src/models/qwen3next.cpp | 324 ++++++++++++++++++++++++++++++++------ 5 files changed, 389 insertions(+), 152 deletions(-) diff --git a/conversion/qwen.py b/conversion/qwen.py index d1127f7431f6..7e3d8c0d12d3 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -268,8 +268,101 @@ def set_vocab(self): super().set_vocab() +class _QwenMtpMixin: + """Shared MTP wiring for Qwen3-Next and Qwen3.5/3.6 text variants. The HF + config carries the MTP block under `mtp_num_hidden_layers` (computed from + the checkpoint when absent, e.g. Qwen3-Next) and the tensors under + `mtp.*`; we extend block_count, emit the nextn metadata key, and remap + `mtp.*` to the standard layer-indexed nextn naming so the existing + tensor_map handles them.""" + + supports_mtp_export = True + hparams: dict[str, Any] + model_arch: gguf.MODEL_ARCH + gguf_writer: gguf.GGUFWriter + block_count: int + tensor_map: gguf.TensorNameMap + no_mtp: bool + mtp_only: bool + _original_block_count: int | None = None + opt_num_mtp_layers: int = 0 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.block_count = self.hparams["num_hidden_layers"] + if not self.no_mtp: + n_mtp = self.hparams.get("mtp_num_hidden_layers", 0) + # Qwen-3-Next doesn't include `mtp_num_hidden_layers` in config. + if n_mtp == 0: + assert self.opt_num_mtp_layers != 0 + n_mtp = self.opt_num_mtp_layers + self.block_count += n_mtp + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + hparams = {**self.hparams, **self.hparams.get("text_config", {})} + key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None) + type(self)._original_block_count = hparams.get(key) + type(self).opt_num_mtp_layers = 0 + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute] + + @classmethod + def filter_tensors(cls, item): + assert cls._original_block_count is not None + # TODO: change TextModel to super() + if (titem := TextModel.filter_tensors(item)) is None: + return None + name, gen = titem + if name.startswith("model.mtp."): + name = name.replace("model.", "", 1) + if name.startswith("mtp."): + if cls.no_mtp: + return None + remapper = { + "fc": "eh_proj", + "pre_fc_norm_embedding": "enorm", + "pre_fc_norm_hidden": "hnorm", + "norm": "shared_head.norm", + } + parts = name.split(".", 3) + if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal(): + mtp_idx = int(parts[2]) + name = f"model.layers.{cls._original_block_count + mtp_idx}.{parts[3]}" + cls.opt_num_mtp_layers = max(cls.opt_num_mtp_layers, mtp_idx + 1) + elif len(parts) == 3 and parts[1] in remapper: + name = f"model.layers.{cls._original_block_count}.{remapper[parts[1]]}.{parts[2]}" + elif cls.mtp_only: + keep = name in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + "embed_tokens.weight", "norm.weight", + ) + if not keep: + return None + return name, gen + + def set_gguf_parameters(self): + super().set_gguf_parameters() # ty: ignore[unresolved-attribute] + if self.no_mtp: + return + if (n := self.block_count - self.hparams["num_hidden_layers"]) > 0: + self.gguf_writer.add_nextn_predict_layers(n) + + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute] + + if not self.mtp_only or not from_dir: + return + + output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + + @ModelBase.register("Qwen3NextForCausalLM") -class Qwen3NextModel(Qwen2MoeModel): +class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel): model_arch = gguf.MODEL_ARCH.QWEN3NEXT def set_gguf_parameters(self): @@ -284,16 +377,6 @@ def set_gguf_parameters(self): rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.25))) - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, gen = item - - if name.startswith("mtp"): - # ignore MTP layers for now - return None - - return super().filter_tensors(item) - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name.endswith(".A_log"): data_torch = -torch.exp(data_torch) @@ -536,97 +619,13 @@ def set_gguf_parameters(self): self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION) -class _Qwen35MtpMixin: - """Shared MTP wiring for Qwen3.5/3.6 text variants. The HF config carries - the MTP block under `mtp_num_hidden_layers` and the tensors under - `mtp.*`; we extend block_count, emit the nextn metadata key, and remap - `mtp.*` to the standard layer-indexed nextn naming so the existing - tensor_map handles them.""" - - supports_mtp_export = True - hparams: dict[str, Any] - model_arch: gguf.MODEL_ARCH - gguf_writer: gguf.GGUFWriter - block_count: int - tensor_map: gguf.TensorNameMap - no_mtp: bool - mtp_only: bool - _original_block_count: int | None = None - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.block_count = self.hparams["num_hidden_layers"] - if not self.no_mtp: - self.block_count += self.hparams.get("mtp_num_hidden_layers", 0) - self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) - - def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: - hparams = {**self.hparams, **self.hparams.get("text_config", {})} - key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None) - type(self)._original_block_count = hparams.get(key) - return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute] - - @classmethod - def filter_tensors(cls, item): - assert cls._original_block_count is not None - # TODO: change TextModel to super() - if (titem := TextModel.filter_tensors(item)) is None: - return None - name, gen = titem - if name.startswith("model.mtp."): - name = name.replace("model.", "", 1) - if name.startswith("mtp."): - if cls.no_mtp: - return None - remapper = { - "fc": "eh_proj", - "pre_fc_norm_embedding": "enorm", - "pre_fc_norm_hidden": "hnorm", - "norm": "shared_head.norm", - } - parts = name.split(".", 3) - if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal(): - mtp_idx = int(parts[2]) - name = f"model.layers.{cls._original_block_count + mtp_idx}.{parts[3]}" - elif len(parts) == 3 and parts[1] in remapper: - name = f"model.layers.{cls._original_block_count}.{remapper[parts[1]]}.{parts[2]}" - elif cls.mtp_only: - keep = name in ( - "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", - "embed_tokens.weight", "norm.weight", - ) - if not keep: - return None - return name, gen - - def set_gguf_parameters(self): - super().set_gguf_parameters() # ty: ignore[unresolved-attribute] - if self.no_mtp: - return - if (n := self.hparams.get("mtp_num_hidden_layers", 0)) > 0: - self.gguf_writer.add_nextn_predict_layers(n) - - def prepare_metadata(self, vocab_only: bool): - from_dir = self.fname_out.is_dir() - super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute] - - if not self.mtp_only or not from_dir: - return - - output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - fname_default: str = gguf.naming_convention( - self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" - - @ModelBase.register("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM") -class Qwen3_5TextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase): +class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35 @ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM") -class Qwen3_5MoeTextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase): +class Qwen3_5MoeTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35MOE diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index a0e55781bac2..7df984432d43 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -2329,7 +2329,13 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.SSM_NORM, MODEL_TENSOR.SSM_IN, MODEL_TENSOR.SSM_BETA_ALPHA, - MODEL_TENSOR.SSM_OUT + MODEL_TENSOR.SSM_OUT, + MODEL_TENSOR.NEXTN_EH_PROJ, + MODEL_TENSOR.NEXTN_EMBED_TOKENS, + MODEL_TENSOR.NEXTN_ENORM, + MODEL_TENSOR.NEXTN_HNORM, + MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, + MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], MODEL_ARCH.QWEN3VL: [ MODEL_TENSOR.TOKEN_EMBD, diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 13023c643f4e..4b4fe4712c9e 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2195,11 +2195,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, // checks default: { - // The MTP head is dense-attention only on hybrid Qwen3.5/3.6, so use a plain + // The MTP head is dense-attention only on hybrid Qwen3-Next/3.5/3.6, so use a plain // attention KV cache for the MTP context instead of the hybrid wrapper. - const bool mtp_on_hybrid_qwen35 = + const bool mtp_on_hybrid_qwen = params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && - (arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE); + (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE); if (llm_arch_is_recurrent(arch)) { res = new llama_memory_recurrent( @@ -2211,7 +2211,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, cparams.n_seq_max, cparams.n_rs_seq, nullptr); - } else if (llm_arch_is_hybrid(arch) && !mtp_on_hybrid_qwen35) { + } else if (llm_arch_is_hybrid(arch) && !mtp_on_hybrid_qwen) { // The main difference between hybrid architectures is the // layer filters, so pick the right one here llama_memory_hybrid::layer_filter_cb filter_attn = nullptr; @@ -2226,7 +2226,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, filter_recr = [&](uint32_t il) { return hparams.is_recr(il) && hparams.n_ff(il) == 0; }; - } else if (arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE) { + } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE) { filter_attn = [&](uint32_t il) { return il < hparams.n_layer() && !hparams.is_recr(il); }; @@ -2292,7 +2292,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, }; } - if (mtp_on_hybrid_qwen35) { + if (mtp_on_hybrid_qwen) { filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } diff --git a/src/models/models.h b/src/models/models.h index 55072f17e97b..930cc3184233 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2037,6 +2037,10 @@ struct llama_model_qwen3next : public llama_model_base { const llama_model & model; }; + struct graph_mtp : public llm_graph_context { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp index 09b66423d5a8..0808fd87aa0e 100644 --- a/src/models/qwen3next.cpp +++ b/src/models/qwen3next.cpp @@ -13,7 +13,11 @@ void llama_model_qwen3next::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group); - // Mark recurrent layers (linear attention layers) + // NextN/MTP: extra decoder block appended beyond the main stack + ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false); + GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all"); + + // Mark recurrent layers (linear attention layers). if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) { uint32_t full_attn_interval = 4; ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false); @@ -28,13 +32,17 @@ void llama_model_qwen3next::load_arch_hparams(llama_model_loader & ml) { } } -void llama_model_qwen3next::load_arch_tensors(llama_model_loader &) { +void llama_model_qwen3next::load_arch_tensors(llama_model_loader & ml) { LLAMA_LOAD_LOCALS; if (n_expert == 0) { throw std::runtime_error(arch_name() + " model cannot have zero experts"); } + const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); + const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0; + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); // output @@ -61,49 +69,73 @@ void llama_model_qwen3next::load_arch_tensors(llama_model_loader &) { const int64_t qkvz_dim = key_dim * 2 + value_dim * 2; const int64_t ba_dim = n_v_heads * 2; - for (int i = 0; i < n_layer; ++i) { - auto & layer = layers[i]; - const uint32_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : hparams.n_ff(i); + auto load_block_trunk = [&](int il, int flags) { + auto & layer = layers[il]; + const uint32_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : hparams.n_ff(il); - layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0); - layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), { n_embd }, 0); + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, flags); + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, flags); - if (!hparams.is_recr(i)) { + if (!hparams.is_recr(il)) { // Attention layers - create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0); - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); - + create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, flags); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, flags); // Q/K normalization for attention layers - layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), { n_embd_head_k }, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), { n_embd_head_k }, 0); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, flags); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, flags); } else { // Linear attention (gated delta net) specific tensors // Create tensors with calculated dimensions // note: ssm_in is used by legacy GGUF - layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", i), { n_embd, qkvz_dim }, TENSOR_NOT_REQUIRED); - layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), { n_embd, key_dim * 2 + value_dim }, TENSOR_NOT_REQUIRED); - layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), { n_embd, value_dim }, TENSOR_NOT_REQUIRED); - layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", i), { hparams.ssm_d_conv, conv_dim }, 0); - layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), { hparams.ssm_dt_rank }, 0); - layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), { hparams.ssm_dt_rank }, 0); - layer.ssm_beta_alpha = create_tensor(tn(LLM_TENSOR_SSM_BETA_ALPHA, "weight", i), { n_embd, ba_dim }, 0); - layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), { head_v_dim }, 0); - layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", i), { value_dim, n_embd }, 0); + layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", il), { n_embd, qkvz_dim }, TENSOR_NOT_REQUIRED | flags); + layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), { n_embd, key_dim * 2 + value_dim }, TENSOR_NOT_REQUIRED | flags); + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", il), { n_embd, value_dim }, TENSOR_NOT_REQUIRED | flags); + layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), { hparams.ssm_d_conv, conv_dim }, flags); + layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), { hparams.ssm_dt_rank }, flags); + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, il), { hparams.ssm_dt_rank }, flags); + layer.ssm_beta_alpha = create_tensor(tn(LLM_TENSOR_SSM_BETA_ALPHA, "weight", il), { n_embd, ba_dim }, flags); + layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), { head_v_dim }, flags); + layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), { value_dim, n_embd }, flags); } - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert }, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff_exp, n_embd, n_expert }, 0); - create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, flags); + create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, flags); // Shared experts - layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", i), { n_embd }, 0); - layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), { n_embd, n_ff_shexp }, 0); - layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), { n_embd, n_ff_shexp }, 0); - layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_shexp, n_embd }, 0); + layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, flags); + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, flags); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, flags); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, flags); + }; + + auto load_block_mtp = [&](int il) { + // MTP head is identical to the trunk block (full attention + FFN) + load_block_trunk(il, mtp_flags); + + auto & layer = layers[il]; + + // NextN-specific tensors that define the MTP block. + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, mtp_flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, mtp_flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, mtp_flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, mtp_flags | TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, mtp_flags | TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, mtp_flags | TENSOR_NOT_REQUIRED); + }; + + for (int i = 0; i < n_layer; i++) { + load_block_trunk(i, trunk_flags); + } + for (int i = n_layer; i < n_layer_all; i++) { + load_block_mtp(i); } } std::unique_ptr llama_model_qwen3next::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique(*this, params); + } return std::make_unique(*this, params); } @@ -120,6 +152,7 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p ggml_tensor * inp_pos = build_inp_pos(); ggml_tensor * inp_out_ids = build_inp_out_ids(); + // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass. for (int il = 0; il < n_layer; ++il) { res->t_layer_inp[il] = inpL; @@ -139,7 +172,7 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il); } - if (il == n_layer - 1 && inp_out_ids) { + if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); } @@ -171,9 +204,16 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p } cur = inpL; - // Final norm + // post-norm hidden state is input to both the LM head and the MTP head cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (!cparams.embeddings_nextn_masked && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + cb(cur, "result_norm", -1); res->t_embd = cur; @@ -186,14 +226,6 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p ggml_build_forward_expand(gf, cur); } -// utility to get one slice from the third dimension -// input dim: [x, y, c, b] -// output dim: [x, y, 1, b] -static ggml_tensor * get_slice_2d(ggml_context * ctx0, ggml_tensor * t, int64_t c) { - return ggml_view_4d(ctx0, t, t->ne[0], t->ne[1], 1, t->ne[3], - t->nb[1], t->nb[2], t->nb[3], t->nb[2] * c); -} - ggml_tensor * llama_model_qwen3next::graph::build_norm_gated( ggml_tensor * input, ggml_tensor * weights, @@ -216,7 +248,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn( // Order: joint QG projection, QG split, Q norm, KV projection, K norm, RoPE, attention // Qwen3Next uses a single Q projection that outputs query + gate - ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur); + ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); cb(Qcur_full, "Qcur_full", il); Qcur_full = ggml_reshape_4d(ctx0, Qcur_full, n_embd_head * 2, n_head, n_tokens, 1); @@ -232,10 +264,10 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn( Qcur_full->nb[1], Qcur_full->nb[2], Qcur_full->nb[3], n_embd_head * ggml_element_size(Qcur_full)); cb(gate, "gate", il); - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s); cb(Kcur, "Kcur", il); - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s); cb(Vcur, "Vcur", il); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); @@ -274,8 +306,6 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn( gate = ggml_sigmoid(ctx0, gate); cb(gate, "gate_sigmoid", il); - gate = ggml_reshape_2d(ctx0, gate, n_embd_head * n_head, n_tokens); - cur = ggml_mul(ctx0, cur, gate); cb(cur, "attn_gated", il); @@ -550,16 +580,19 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_ffn(ggml_tensor * cur, c LLM_FFN_SILU, true, hparams.expert_weights_scale, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il, - nullptr, model.layers[il].ffn_gate_up_exps); + nullptr, model.layers[il].ffn_gate_up_exps, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); cb(moe_out, "ffn_moe_out", il); // Add shared experts if present - following Qwen3Next reference implementation if (model.layers[il].ffn_up_shexp != nullptr) { ggml_tensor * ffn_shexp = build_ffn(cur, - model.layers[il].ffn_up_shexp, NULL, NULL, - model.layers[il].ffn_gate_shexp, NULL, NULL, - model.layers[il].ffn_down_shexp, NULL, NULL, + model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s, + model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s, + model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); cb(ffn_shexp, "ffn_shexp", il); @@ -593,3 +626,198 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_ffn(ggml_tensor * cur, c } return cur; } + +// LLM_GRAPH_TYPE_DECODER_MTP draft head for Qwen3-Next +llama_model_qwen3next::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) + : llm_graph_context(params) { + GGML_ASSERT(hparams.n_layer_nextn > 0 && "QWEN3NEXT MTP requires n_layer_nextn > 0"); + GGML_ASSERT(hparams.n_layer_nextn == 1 && "QWEN3NEXT MTP currently only supports a single MTP block"); + + const int64_t n_embd_head = hparams.n_embd_head_v(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + const int il = hparams.n_layer(); + const auto & layer = model.layers[il]; + + GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj"); + GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm"); + GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); + GGML_ASSERT(layer.ffn_gate_inp && "MTP block missing ffn_gate_inp"); + + // TODO: extract in a common llm_graph_context::build_inp_embd_h() + auto inp = std::make_unique(hparams.n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); + ggml_set_input(inp->embd); + + // TODO: make static using `ggml_build_forward_select()` + // see llm_graph_context::build_inp_embd() for reference + ggml_tensor * tok_embd; + if (ubatch.token) { + ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; + + tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens); + } else { + tok_embd = inp->embd; + } + cb(tok_embd, "mtp_tok_embd", il); + + inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); + ggml_set_input(inp->h); + ggml_set_name(inp->h, "mtp_h_input"); + + ggml_tensor * h_embd = inp->h; + + res->add_input(std::move(inp)); + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + auto * inp_attn = build_attn_inp_kv(); + + ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + + ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il); + cb(e_norm, "mtp_enorm", il); + + ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0); + cb(concat, "mtp_concat", il); + + ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s); + cb(cur, "mtp_eh_proj", il); + + ggml_tensor * inpSA = cur; + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_norm", il); + + ggml_tensor * Qcur_full = build_lora_mm(layer.wq, cur, layer.wq_s); + cb(Qcur_full, "mtp_Qcur_full", il); + + ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, + n_embd_head, n_head, n_tokens, + ggml_element_size(Qcur_full) * n_embd_head * 2, + ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head, + 0); + Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il); + cb(Qcur, "mtp_Qcur_normed", il); + + ggml_tensor * Kcur = build_lora_mm(layer.wk, cur, layer.wk_s); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); + cb(Kcur, "mtp_Kcur_normed", il); + + ggml_tensor * Vcur = build_lora_mm(layer.wv, cur, layer.wv_s); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + + cb(Qcur, "mtp_Qcur", il); + cb(Kcur, "mtp_Kcur", il); + cb(Vcur, "mtp_Vcur", il); + + const float kq_scale = hparams.f_attention_scale == 0.0f + ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + + cur = build_attn(inp_attn, + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "mtp_attn_pregate", il); + + ggml_tensor * gate = ggml_view_3d(ctx0, Qcur_full, + n_embd_head, n_head, n_tokens, + ggml_element_size(Qcur_full) * n_embd_head * 2, + ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head, + ggml_element_size(Qcur_full) * n_embd_head); + + // TODO: CUDA is missing non-contiguous unary ops. when implemented: remove this cont + gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens); + cb(gate, "mtp_gate", il); + + cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, gate)); + cur = build_lora_mm(layer.wo, cur, layer.wo_s); + cb(cur, "mtp_attn_out", il); + + if (inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + cur = ggml_add(ctx0, cur, inpSA); + cb(cur, "mtp_attn_residual", il); + + ggml_tensor * ffn_residual = cur; + cur = build_norm(cur, layer.attn_post_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_post_norm", il); + + // MoE FFN — routed experts plus gated shared expert (mirrors the trunk). + ggml_tensor * moe_out = + build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, true, + hparams.expert_weights_scale, + LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il, + nullptr, layer.ffn_gate_up_exps, + layer.ffn_up_exps_s, + layer.ffn_gate_exps_s, + layer.ffn_down_exps_s); + cb(moe_out, "mtp_ffn_moe_out", il); + + if (layer.ffn_up_shexp != nullptr) { + ggml_tensor * ffn_shexp = + build_ffn(cur, + layer.ffn_up_shexp, nullptr, layer.ffn_up_shexp_s, + layer.ffn_gate_shexp, nullptr, layer.ffn_gate_shexp_s, + layer.ffn_down_shexp, nullptr, layer.ffn_down_shexp_s, + nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "mtp_ffn_shexp", il); + + ggml_tensor * shared_gate = build_lora_mm(layer.ffn_gate_inp_shexp, cur); + shared_gate = ggml_sigmoid(ctx0, shared_gate); + cb(shared_gate, "mtp_shared_expert_gate_sigmoid", il); + + ffn_shexp = ggml_mul(ctx0, ffn_shexp, shared_gate); + cb(ffn_shexp, "mtp_ffn_shexp_gated", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + } else { + cur = moe_out; + } + cb(cur, "mtp_ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_residual); + cb(cur, "mtp_post_ffn", il); + + ggml_tensor * head_norm_w = layer.nextn.shared_head_norm + ? layer.nextn.shared_head_norm + : model.output_norm; + GGML_ASSERT(head_norm_w && "QWEN3NEXT MTP: missing both nextn.shared_head_norm and output_norm"); + cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1); + + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output; + ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s; + GGML_ASSERT(head_w && "QWEN3NEXT MTP: missing LM head (nextn.shared_head_head or model.output)"); + cur = build_lora_mm(head_w, cur, head_s); + cb(cur, "result_output", -1); + + res->t_logits = cur; + ggml_build_forward_expand(gf, cur); +} From f2b52a87e82fa461191565a19d4ef9fd8b8fbd87 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Mon, 3 Aug 2026 10:47:21 +0200 Subject: [PATCH 35/98] server: (tools) add x-tool-cwd header (#26420) * server: (tools) add x-tool-cwd header * reuse str_to_lower from server-models --- tools/server/README-dev.md | 3 + tools/server/server-tools.cpp | 58 +++++++++++++++---- tools/server/tests/unit/test_tools_builtin.py | 27 ++++++++- 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/tools/server/README-dev.md b/tools/server/README-dev.md index b41d70c63ac8..45bcdcca7699 100644 --- a/tools/server/README-dev.md +++ b/tools/server/README-dev.md @@ -199,6 +199,9 @@ Invoke a tool call, request body is a JSON object with: - `tool` (string): the name of the tool - `params` (object): a mapping from argument name (string) to argument value +Headers: +- `x-tool-cwd`: optional; if set, use as the CWD for tool; this is not part of tool's params because it's meant to be set by the runtime, not the LLM itself + Returns JSON object. There are two response formats (MCP tools use the same two formats: their result content is concatenated into `plain_text_response`, and RPC or tool errors are surfaced as the `error` string): Format 1: Plain text. The text will be placed into a field called `plain_text_response`, example: diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index 90b7e2a9f0ee..4a6c5ed4420a 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -64,24 +64,27 @@ class tools_io { class tools_io_basic : public tools_io { public: + // cwd, if non-empty, is used to resolve relative paths and as the working directory for run() + explicit tools_io_basic(std::string cwd = "") : cwd(std::move(cwd)) {} + bool is_directory(const std::string & path) const override { std::error_code ec; - return fs::is_directory(path, ec) && !ec; + return fs::is_directory(resolve(path), ec) && !ec; } bool is_regular_file(const std::string & path) const override { std::error_code ec; - return fs::is_regular_file(path, ec) && !ec; + return fs::is_regular_file(resolve(path), ec) && !ec; } bool file_size(const std::string & path, uintmax_t & out_size) const override { std::error_code ec; - out_size = fs::file_size(path, ec); + out_size = fs::file_size(resolve(path), ec); return !ec; } bool read_file(const std::string & path, std::string & out) const override { - std::ifstream f(path, std::ios::binary); + std::ifstream f(resolve(path), std::ios::binary); if (!f) return false; std::ostringstream ss; ss << f.rdbuf(); @@ -91,12 +94,12 @@ class tools_io_basic : public tools_io { bool write_file(const std::string & path, const std::string & content) const override { std::error_code ec; - fs::path fpath(path); + fs::path fpath(resolve(path)); if (fpath.has_parent_path()) { fs::create_directories(fpath.parent_path(), ec); if (ec) return false; } - std::ofstream f(path, std::ios::binary); + std::ofstream f(fpath, std::ios::binary); if (!f) return false; f << content; return (bool) f; @@ -104,13 +107,14 @@ class tools_io_basic : public tools_io { std::vector list_files(const std::string & base, std::string & err) const override { err.clear(); + std::string abs_base = resolve(base); if (!is_directory(base)) { err = "path does not exist or is not a directory: " + base; return {}; } auto res = run( - {"git", "-C", base, "ls-files", "--cached", "--others", "--exclude-standard"}, + {"git", "-C", abs_base, "ls-files", "--cached", "--others", "--exclude-standard"}, SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, SERVER_TOOL_GIT_LS_FILES_TIMEOUT); if (res.exit_code == 0 && !res.timed_out) { @@ -128,7 +132,7 @@ class tools_io_basic : public tools_io { return result; } - return list_files_fallback(base); + return list_files_fallback(abs_base); } exec_result run( @@ -145,7 +149,7 @@ class tools_io_basic : public tools_io { | subprocess_option_inherit_environment | subprocess_option_search_user_path; - if (!proc.create(args, options)) { + if (!proc.create(args, options, {}, cwd.empty() ? nullptr : cwd.c_str())) { res.output = "failed to spawn process"; return res; } @@ -205,6 +209,16 @@ class tools_io_basic : public tools_io { } private: + std::string cwd; + + // resolves `path` against `cwd` if `path` is relative and `cwd` is set; otherwise returns `path` unchanged + std::string resolve(const std::string & path) const { + if (cwd.empty() || fs::path(path).is_absolute()) { + return path; + } + return (fs::path(cwd) / path).string(); + } + static const std::unordered_set & junk_dir_names() { static const std::unordered_set names = { ".git", ".svn", ".hg", "node_modules", "__pycache__", @@ -244,8 +258,8 @@ class tools_io_basic : public tools_io { }; static std::unique_ptr make_tools_io(const json & params) { - GGML_UNUSED(params); // TODO in follow-up PR - return std::make_unique(); + std::string cwd = json_value(params, "cwd", std::string()); + return std::make_unique(cwd); } // no '/' in pattern -> match basename at any depth; else match full relative path @@ -1188,6 +1202,22 @@ static std::vector> build_tools() { return tools; } +static std::string str_to_lower(const std::string & value) { + std::string lowered(value.size(), '\0'); + std::transform(value.begin(), value.end(), lowered.begin(), [](unsigned char c) { return std::tolower(c); }); + return lowered; +} + +static std::string get_header(const std::map & headers, const std::string & key, std::string default_value = "") { + const auto lowered_key = str_to_lower(key); + for (const auto & h : headers) { + if (str_to_lower(h.first) == lowered_key) { + return h.second; + } + } + return default_value; +} + void server_tools::setup(const std::vector & enabled_tools, server_mcp & mcp_mgr) { if (!enabled_tools.empty()) { @@ -1271,6 +1301,12 @@ void server_tools::setup(const std::vector & enabled_tools, json params = body.value("params", json::object()); bool stream = body.value("stream", false); + // accept x-tool-cwd header to override of the process + auto cwd = get_header(req.headers, "x-tool-cwd"); + if (!cwd.empty()) { + params["cwd"] = cwd; + } + server_tool & tool = find_tool(tools, tool_name, stream); if (stream) { diff --git a/tools/server/tests/unit/test_tools_builtin.py b/tools/server/tests/unit/test_tools_builtin.py index 1b2d0db43231..fb194cac669a 100755 --- a/tools/server/tests/unit/test_tools_builtin.py +++ b/tools/server/tests/unit/test_tools_builtin.py @@ -19,8 +19,8 @@ def create_server(): server.server_tools = "all" -def call_tool(name: str, params: dict) -> dict: - res = server.make_request("POST", "/tools", data={"tool": name, "params": params}) +def call_tool(name: str, params: dict, headers: dict | None = None) -> dict: + res = server.make_request("POST", "/tools", data={"tool": name, "params": params}, headers=headers) assert res.status_code == 200, res.body assert "error" not in res.body, res.body return res.body @@ -123,6 +123,29 @@ def test_tools_builtin_exec_shell_command_stream(): assert "[exit code: 0]" in chunks +def test_tools_builtin_cwd_header(): + global server + server.start() + + cwd_dir = os.path.join(PROJECT_ROOT, "tools", "server", "tests", "unit") + headers = {"x-tool-cwd": cwd_dir} + + res = call_tool("read_file", {"path": "test_tools_builtin.py"}, headers=headers) + assert GREP_MARKER in res["plain_text_response"] + + # exec_shell_command should also run with that directory as its working directory: + # writing to a relative filename must land inside cwd_dir + marker_name = "llama_cpp_test_tools_builtin_cwd_marker.txt" + marker_path = os.path.join(cwd_dir, marker_name) + try: + command = f"echo hello > {marker_name}" + call_tool("exec_shell_command", {"command": command}, headers=headers) + assert os.path.exists(marker_path) + finally: + if os.path.exists(marker_path): + os.remove(marker_path) + + def test_tools_builtin_edit_file_rejects_overlapping_edits(): global server server.start() From 0b14b87d7c20cb753b94b96854dd7b45306fc696 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Mon, 3 Aug 2026 12:45:24 +0200 Subject: [PATCH 36/98] server: add notice for upcoming default port change 8080 --> 9931 (#26508) * server: add notice for upcoming default port change 8080 --> 6631 * add link to PR * correct to 9931 --- tools/server/server.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/server/server.cpp b/tools/server/server.cpp index a3b2a8b0fe1b..aafb1f307966 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -486,6 +486,13 @@ int llama_server(common_params & params, int argc, char ** argv) { SRV_INF("listening on %s\n", ctx_http.listening_address.c_str()); + // TODO: remove this in the future + // check the string to also handle the .sock case + if (string_ends_with(ctx_http.listening_address, ":8080")) { + SRV_WRN("%s", "NOTICE: server default port will be changed to :9931 in a future release\n"); + SRV_WRN("%s", " ref: https://github.com/ggml-org/llama.cpp/pull/26508\n"); + } + if (is_router_server) { if (!params.models_preset_hf.empty()) { SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str()); From 9bd4c09ea571a9020f30eeef169b552625b5b5a4 Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Mon, 3 Aug 2026 14:22:44 +0200 Subject: [PATCH 37/98] CUDA: Fix data-races when reusing SMEM in block_reduce (#26385) * CUDA: Fix data-races when reusing block_reduce block_reduce currently doesn't resync after reading from SMEM, causing potential data-races when reusing SMEM for multiple reductions. One may consider simply always adding this in block_reduce, but this comes at a potential perf cost * double-buffering for single-row softmax * double-buffering for norm as well * Add comment * Add explanatory comment to block_reduce * Specify need for + do memory barrier only in multi-warp scenario * Implement review-suggestion from @gaugarg-nv --- ggml/src/ggml-cuda/common.cuh | 3 ++- ggml/src/ggml-cuda/norm.cu | 4 ++-- ggml/src/ggml-cuda/softmax.cu | 20 ++++++++++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 33be16dc5cce..d27d8acb1d37 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -627,7 +627,8 @@ template struct block_reduce_policy { }; template -static __device__ T block_reduce(T val, T * shared_vals) { +static __device__ T block_reduce(T val, [[maybe_unused]] T * shared_vals) { + // for multi-warp reductions, callers must not reuse shared_vals until all reads from this invocation have completed val = block_reduce_policy::reduce(val); const unsigned int block_size = block_size_template == 0 ? blockDim.x : block_size_template; if (block_size > WARP_SIZE) { diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index 09d9f3a7d624..c3758cd50cfe 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -64,7 +64,7 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr tmp += xi * xi; } - tmp = block_reduce(tmp, s_sum); + tmp = block_reduce(tmp, s_sum + 32); const float variance = tmp / group_size; const float scale = rsqrtf(variance + eps); @@ -297,7 +297,7 @@ static void group_norm_f32_cuda( group_norm_f32<<>>(x, dst, group_size, ne_elements, eps); } else { const dim3 block_dims(1024, 1, 1); - group_norm_f32<1024><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, group_size, ne_elements, eps); + group_norm_f32<1024><< WARP_SIZE ? 2 * 32 * sizeof(float): 0, stream>>>(x, dst, group_size, ne_elements, eps); } } diff --git a/ggml/src/ggml-cuda/softmax.cu b/ggml/src/ggml-cuda/softmax.cu index 285c0e9543a2..f320c6f004ea 100644 --- a/ggml/src/ggml-cuda/softmax.cu +++ b/ggml/src/ggml-cuda/softmax.cu @@ -116,6 +116,11 @@ static __global__ void soft_max_f32( vals[col] = val; } + if (block_size > WARP_SIZE) { + // sync is needed as we reuse buf_iw across block_reduce invocations, see #26385 + // for block_size <= WARP_SIZE, block_reduce does not access buf_iw + __syncthreads(); + } // find the sum of exps in the block tmp = block_reduce(tmp, buf_iw); @@ -142,6 +147,8 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ float * __restrict__ dst, float * __restrict__ tmp_maxs, float * __restrict__ tmp_sums, + float * shared_vals_max, + float * shared_vals_sum, const soft_max_params p) { namespace cg = cooperative_groups; @@ -154,7 +161,6 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ float local_vals[n_elem_per_thread] = { -INFINITY, -INFINITY, -INFINITY, -INFINITY }; float local_max = -INFINITY; const int step_size = gridDim.x * blockDim.x; - __shared__ float shared_vals[32]; // Compute thread-local max for (int col = col_start; col < p.ncols;) { @@ -171,7 +177,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ } // Compute CTA-level max - local_max = block_reduce(local_max, shared_vals); + local_max = block_reduce(local_max, shared_vals_max); // Store CTA-level max to GMEM if (tid == 0) { @@ -186,7 +192,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ } else { local_max = -INFINITY; } - local_max = block_reduce(local_max, shared_vals); + local_max = block_reduce(local_max, shared_vals_max); // Compute softmax dividends, accumulate divisor float tmp_expf = 0.0f; @@ -209,7 +215,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ } // Reduce divisor within CTA - tmp_expf = block_reduce(tmp_expf, shared_vals); + tmp_expf = block_reduce(tmp_expf, shared_vals_sum); // Store CTA-level sum to GMEM if (tid == 0) { @@ -223,7 +229,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ } else { tmp_expf = 0.0f; } - tmp_expf = block_reduce(tmp_expf, shared_vals); + tmp_expf = block_reduce(tmp_expf, shared_vals_sum); // Divide dividend by global sum + store data for (int col = col_start; col < p.ncols;) { @@ -310,9 +316,11 @@ __launch_bounds__(8*WARP_SIZE, 1) static __global__ void soft_max_f32_paralleliz // https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/device-callable-apis.html#grid-synchronization // https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/device-callable-apis.html#class-cluster-group { + __shared__ float shared_vals[2][32]; + for (int rowx = 0; rowx < p.ne01 * p.ne02 * p.ne03; rowx++) { soft_max_f32_parallelize_cols_single_row(x + int64_t(rowx) * p.ncols, dst + int64_t(rowx) * p.ncols, tmp_maxs, - tmp_sums, p); + tmp_sums, shared_vals[0], shared_vals[1], p); } } From 96278e39fc83e1d97c881e34bcec39ac7ea98820 Mon Sep 17 00:00:00 2001 From: Konrad Moren Date: Mon, 3 Aug 2026 14:26:09 +0200 Subject: [PATCH 38/98] CUDA: Add backend sampler for penalties sampler (#25262) * sampling: enhance penalty handling in common_sampler_init - Set default value for penalty_last_n based on model context if not specified. - Ensure penalty_last_n and n_prev are non-negative. - Update llama_sampler_penalties structure to inherit from llama_sampler_backend and add backend input handling for penalties. - Implement backend initialization and application logic for penalties, including frequency and presence adjustments. * tests: add backend penalties sampling tests and utility functions - Introduced `accept_prompt` and `unique_prompt_tokens` functions to handle prompt acceptance and token uniqueness. - Implemented `compare_penalties_logits` to compare logits from backend and CPU samplers with penalties. - Added `test_backend_penalties_sampling` to validate backend penalties with various configurations. - Enhanced the test suite for better coverage of penalty handling in sampling. * sampling: add support for top-k penalties in backend sampling * sampling: add fix to ensure stable numerical results. Preserve masked logits as -Inf and no longer generate NaN. * sampling: enhance penalty comparison tests with masking penalties logic * add comments on padding * sampling: add comments on modifications * add the unit test to cover masked-out token as -INF * validate repeat penalty to ensure it is finite and greater than 0; add tests for invalid values * refactor: test functions to share logic and be less verbose * add test to cover case where previously penalized token is not part of candidates * remove comments * remove redundant penalty_last_n initialization and validation in common_sampler_init * add support for penalties in sampler chain with configurable positions * add validation for penalty parameters and enhance tests for non-finite values * add context parameter to common_sampler_init and set default for penalty_last_n * add llama_n_ctx parameter to common_sampler_init for improved sampler initialization * replace penalty_last_n x n_candidates comparison matrix with a vocabulary-sized count tensor * add tests for backend penalties sampling without filler entries , token_count.size() == n_active == n_max == 64 * add test for backend penalties sampling after top-p with large history window * remove as unused * add is_disabled method, tensor logits reshape, add rest review suggestions * clarify comment --- common/arg.cpp | 21 +- common/common.cpp | 3 +- common/sampling.cpp | 21 +- common/sampling.h | 5 +- include/llama.h | 7 +- src/llama-graph.cpp | 1 + src/llama-sampler.cpp | 241 ++++++++++++-- tests/test-arg-parser.cpp | 28 ++ tests/test-backend-sampler.cpp | 561 ++++++++++++++++++++++++++++++++ tools/server/server-context.cpp | 3 +- 10 files changed, 860 insertions(+), 31 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 772422f6820a..305938fcb2b4 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -2036,7 +2037,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--repeat-penalty"}, "N", string_format("penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)", (double)params.sampling.penalty_repeat), [](common_params & params, const std::string & value) { - params.sampling.penalty_repeat = std::stof(value); + const float penalty_repeat = std::stof(value); + if (!std::isfinite(penalty_repeat) || + penalty_repeat <= 0.0f || + !std::isfinite(1.0f/penalty_repeat)) { + throw std::runtime_error("error: repeat-penalty must be finite and greater than 0\n"); + } + params.sampling.penalty_repeat = penalty_repeat; params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT; } ).set_sampling()); @@ -2044,14 +2051,22 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--presence-penalty"}, "N", string_format("repeat alpha presence penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_present), [](common_params & params, const std::string & value) { - params.sampling.penalty_present = std::stof(value); + const float penalty_present = std::stof(value); + if (!std::isfinite(penalty_present)) { + throw std::runtime_error("error: presence-penalty must be finite\n"); + } + params.sampling.penalty_present = penalty_present; } ).set_sampling()); add_opt(common_arg( {"--frequency-penalty"}, "N", string_format("repeat alpha frequency penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_freq), [](common_params & params, const std::string & value) { - params.sampling.penalty_freq = std::stof(value); + const float penalty_freq = std::stof(value); + if (!std::isfinite(penalty_freq)) { + throw std::runtime_error("error: frequency-penalty must be finite\n"); + } + params.sampling.penalty_freq = penalty_freq; } ).set_sampling()); add_opt(common_arg( diff --git a/common/common.cpp b/common/common.cpp index ff27d392fb2e..c941fd505ab3 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1299,8 +1299,9 @@ common_init_result::common_init_result(common_params & params, bool model_only) pimpl->samplers.resize(cparams.n_seq_max); pimpl->samplers_seq_config.resize(cparams.n_seq_max); + const int32_t n_ctx = cparams.n_ctx > 0 ? (int32_t) cparams.n_ctx : llama_model_n_ctx_train(model); for (int i = 0; i < (int) cparams.n_seq_max; ++i) { - pimpl->samplers[i].reset(common_sampler_init(model, params.sampling)); + pimpl->samplers[i].reset(common_sampler_init(model, params.sampling, n_ctx)); pimpl->samplers_seq_config[i] = { i, common_sampler_get(pimpl->samplers[i].get()) }; } diff --git a/common/sampling.cpp b/common/sampling.cpp index 256ac161e20f..5698c0263b24 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -184,9 +184,26 @@ std::string common_params_sampling::print() const { return std::string(result); } -struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params) { - const llama_vocab * vocab = llama_model_get_vocab(model); +struct common_sampler * common_sampler_init( + const struct llama_model * model, + struct common_params_sampling & params, + int32_t n_ctx) { + if (!std::isfinite(params.penalty_repeat) || + params.penalty_repeat <= 0.0f || + !std::isfinite(1.0f/params.penalty_repeat)) { + throw std::invalid_argument("penalty_repeat must be finite and greater than 0"); + } + if (!std::isfinite(params.penalty_freq)) { + throw std::invalid_argument("penalty_freq must be finite"); + } + if (!std::isfinite(params.penalty_present)) { + throw std::invalid_argument("penalty_present must be finite"); + } + if (params.penalty_last_n == -1) { + params.penalty_last_n = n_ctx > 0 ? n_ctx : llama_model_n_ctx_train(model); + } + const llama_vocab * vocab = llama_model_get_vocab(model); llama_sampler_chain_params lparams = llama_sampler_chain_default_params(); lparams.no_perf = params.no_perf; diff --git a/common/sampling.h b/common/sampling.h index 4191988bb877..91e2cea787fb 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -37,7 +37,10 @@ struct common_sampler; // llama_sampler API overloads // note: can mutate params in some cases -struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params); +struct common_sampler * common_sampler_init( + const struct llama_model * model, + struct common_params_sampling & params, + int32_t n_ctx = 0); void common_sampler_free(struct common_sampler * gsmpl); diff --git a/include/llama.h b/include/llama.h index 6e53e2297235..f2d7e3885837 100644 --- a/include/llama.h +++ b/include/llama.h @@ -1256,6 +1256,7 @@ extern "C" { struct ggml_tensor * probs; struct ggml_tensor * sampled; struct ggml_tensor * candidates; + int64_t n_vocab; }; // user code can implement the interface below in order to create custom llama_sampler @@ -1425,9 +1426,9 @@ extern "C" { /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. LLAMA_API struct llama_sampler * llama_sampler_init_penalties( int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) - float penalty_repeat, // 1.0 = disabled - float penalty_freq, // 0.0 = disabled - float penalty_present); // 0.0 = disabled + float penalty_repeat, // must be > 0.0, 1.0 = disabled + float penalty_freq, // must be finite, 0.0 = disabled + float penalty_present); // must be finite, 0.0 = disabled /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 LLAMA_API struct llama_sampler * llama_sampler_init_dry( diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index e12a8cdc2aac..1a35692300c1 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -3620,6 +3620,7 @@ void llm_graph_context::build_sampling() const { /*.probs =*/ nullptr, /*.sampled =*/ nullptr, /*.candidates =*/ nullptr, + /*.n_vocab =*/ logits_seq->ne[0], }; assert(sampler->iface->backend_apply); diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cpp index a9cb6bee5fd7..b2f1abe73785 100644 --- a/src/llama-sampler.cpp +++ b/src/llama-sampler.cpp @@ -589,6 +589,7 @@ static bool llama_sampler_backend_support( /*.probs = */ nullptr, /*.sampled = */ nullptr, /*.candidates = */ ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n), + /*.n_vocab = */ n, }; ggml_cgraph * gf = ggml_new_graph(ctx); @@ -2638,7 +2639,7 @@ struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( // penalties -struct llama_sampler_penalties { +struct llama_sampler_penalties : public llama_sampler_backend { const int32_t penalty_last_n; const float penalty_repeat; const float penalty_freq; @@ -2648,10 +2649,49 @@ struct llama_sampler_penalties { // a frequency map to count token occurrences std::unordered_map token_count; + + // backend graph inputs + ggml_tensor * inp_token_ids = nullptr; + ggml_tensor * inp_counts = nullptr; + + // backend helpers + int32_t n_vocab = 0; + int32_t n_max = 0; + bool has_candidates = false; + + std::vector host_token_ids; + std::vector host_counts; + + static bool is_disabled( + int32_t penalty_last_n, + float penalty_repeat, + float penalty_freq, + float penalty_present) { + return penalty_last_n == 0 || + (penalty_repeat == 1.0f && penalty_freq == 0.0f && penalty_present == 0.0f); + } + + bool is_disabled() const { + return is_disabled(penalty_last_n, penalty_repeat, penalty_freq, penalty_present); + } + + llama_sampler_penalties( + int32_t penalty_last_n, + float penalty_repeat, + float penalty_freq, + float penalty_present) + : llama_sampler_backend("penalties") + , penalty_last_n (penalty_last_n) + , penalty_repeat (penalty_repeat) + , penalty_freq (penalty_freq) + , penalty_present (penalty_present) + , prev (penalty_last_n) { + } }; -static const char * llama_sampler_penalties_name(const struct llama_sampler * /*smpl*/) { - return "penalties"; +static const char * llama_sampler_penalties_name(const struct llama_sampler * smpl) { + auto * ctx = (llama_sampler_penalties *) smpl->ctx; + return ctx->get_name(); } static void llama_sampler_penalties_accept(struct llama_sampler * smpl, llama_token token) { @@ -2688,8 +2728,7 @@ static void llama_sampler_penalties_accept(struct llama_sampler * smpl, llama_to static void llama_sampler_penalties_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) { auto * ctx = (llama_sampler_penalties *) smpl->ctx; - if ((ctx->penalty_last_n == 0) || - (ctx->penalty_repeat == 1.0f && ctx->penalty_freq == 0.0f && ctx->penalty_present == 0.0f)) { + if (ctx->is_disabled()) { return; } @@ -2736,7 +2775,8 @@ static struct llama_sampler * llama_sampler_penalties_clone(const struct llama_s { auto * result_ctx = (llama_sampler_penalties *) result->ctx; - result_ctx->prev = ctx->prev; + result_ctx->prev = ctx->prev; + result_ctx->token_count = ctx->token_count; } return result; @@ -2746,6 +2786,171 @@ static void llama_sampler_penalties_free(struct llama_sampler * smpl) { delete (llama_sampler_penalties *) smpl->ctx; } +static bool llama_sampler_penalties_backend_init( + struct llama_sampler * smpl, + ggml_backend_buffer_type_t buft) { + auto * sctx = (llama_sampler_penalties *) smpl->ctx; + + const bool res = llama_sampler_backend_support(smpl, buft); + + sctx->init(res); + + return res; +} + +static void llama_sampler_penalties_backend_apply( + struct llama_sampler * smpl, + struct ggml_context * ctx, + struct ggml_cgraph * gf, + struct llama_sampler_data * data) { + GGML_UNUSED(gf); + + auto * sctx = (llama_sampler_penalties *) smpl->ctx; + + if (sctx->is_disabled()) { + return; + } + + GGML_ASSERT(data->n_vocab > 0 && data->n_vocab <= INT32_MAX); + + sctx->has_candidates = data->candidates != nullptr; + sctx->n_vocab = (int32_t) data->n_vocab; + sctx->n_max = std::min(sctx->penalty_last_n, sctx->n_vocab); + + sctx->inp_token_ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, sctx->n_max); + ggml_set_name(sctx->inp_token_ids, "penalties_token_ids"); + ggml_set_input(sctx->inp_token_ids); + + sctx->inp_counts = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, sctx->n_max); + ggml_set_name(sctx->inp_counts, "penalties_counts"); + ggml_set_input(sctx->inp_counts); + + if ((int32_t) sctx->host_token_ids.size() != sctx->n_max) { + sctx->host_token_ids.assign(sctx->n_max, 0); + sctx->host_counts.assign(sctx->n_max, 0); + } + + // flatten + ggml_tensor * logits = ggml_reshape_1d(ctx, data->logits, ggml_nelements(data->logits)); + ggml_tensor * gathered = logits; + ggml_tensor * counts_f32 = ggml_cast(ctx, sctx->inp_counts, GGML_TYPE_F32); + + if (sctx->has_candidates) { + ggml_tensor * candidates = ggml_reshape_1d( + ctx, data->candidates, ggml_nelements(data->candidates)); + const int64_t n_candidates = candidates->ne[0]; + GGML_ASSERT(n_candidates == ggml_nelements(logits)); + + ggml_tensor * counts_rows = ggml_fill( + ctx, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, sctx->n_vocab), 0.0f); + ggml_tensor * scatter_rows = ggml_reshape_2d(ctx, counts_f32, 1, sctx->n_max); + counts_rows = ggml_set_rows(ctx, counts_rows, scatter_rows, sctx->inp_token_ids); + counts_f32 = ggml_get_rows(ctx, counts_rows, candidates); + counts_f32 = ggml_reshape_1d(ctx, counts_f32, n_candidates); + } else { + ggml_tensor * logits_rows = ggml_reshape_2d(ctx, logits, 1, ggml_nelements(logits)); + gathered = ggml_get_rows(ctx, logits_rows, sctx->inp_token_ids); + gathered = ggml_reshape_1d(ctx, gathered, sctx->n_max); + } + + ggml_tensor * active_mask = ggml_step(ctx, counts_f32); + ggml_tensor * inactive_mask = ggml_sub(ctx, ggml_fill(ctx, active_mask, 1.0f), active_mask); + + ggml_tensor * penalized = gathered; + + if (sctx->penalty_repeat != 1.0f) { + ggml_tensor * pos_mask = ggml_step(ctx, penalized); + ggml_tensor * neg_mask = ggml_sub(ctx, ggml_fill(ctx, pos_mask, 1.0f), pos_mask); + + ggml_tensor * pos_scale = ggml_scale(ctx, pos_mask, 1.0f/sctx->penalty_repeat); + ggml_tensor * neg_scale = ggml_scale(ctx, neg_mask, sctx->penalty_repeat); + ggml_tensor * repeat_scale = ggml_add(ctx, pos_scale, neg_scale); + + // scale inactive entries with 1 to avoid -INF * 0 = NaN for values masked by top-p + repeat_scale = ggml_mul(ctx, repeat_scale, active_mask); + repeat_scale = ggml_add(ctx, repeat_scale, inactive_mask); + penalized = ggml_mul(ctx, gathered, repeat_scale); + } + + if (sctx->penalty_freq != 0.0f) { + ggml_tensor * penalty_freq = ggml_scale(ctx, counts_f32, sctx->penalty_freq); + penalized = ggml_sub(ctx, penalized, penalty_freq); + } + + if (sctx->penalty_present != 0.0f) { + ggml_tensor * penalty_present = ggml_scale(ctx, active_mask, sctx->penalty_present); + penalized = ggml_sub(ctx, penalized, penalty_present); + } + + if (sctx->has_candidates) { + data->logits = penalized; + } else { + ggml_tensor * logits_rows = ggml_reshape_2d(ctx, logits, 1, ggml_nelements(logits)); + ggml_tensor * scatter_rows = ggml_reshape_2d(ctx, penalized, 1, sctx->n_max); + logits_rows = ggml_set_rows(ctx, logits_rows, scatter_rows, sctx->inp_token_ids); + data->logits = ggml_reshape_1d(ctx, logits_rows, ggml_nelements(logits)); + } +} + +static void llama_sampler_penalties_backend_set_input(struct llama_sampler * smpl) { + auto * sctx = (llama_sampler_penalties *) smpl->ctx; + + if (!sctx->inp_token_ids || !sctx->inp_counts || sctx->n_max <= 0 || sctx->n_vocab <= 0) { + return; + } + + if (sctx->is_disabled()) { + return; + } + + // fill active entries from the map + int32_t n_active = 0; + + for (const auto & it : sctx->token_count) { + GGML_ASSERT(n_active < sctx->n_max); + sctx->host_token_ids[n_active] = it.first; + sctx->host_counts [n_active] = it.second; + ++n_active; + } + + // Sorting is required because backend_apply uses ggml_set_rows (a scatter-back operation) + std::vector> entries; + entries.reserve(n_active); + for (int32_t i = 0; i < n_active; ++i) { + entries.emplace_back(sctx->host_token_ids[i], sctx->host_counts[i]); + } + std::sort(entries.begin(), entries.end(), [](const auto & a, const auto & b) { + return a.first < b.first; + }); + for (int32_t i = 0; i < n_active; ++i) { + sctx->host_token_ids[i] = entries[i].first; + sctx->host_counts [i] = entries[i].second; + } + + // Padding: Finds a filler token id that is not present in token_count. + // Use it to do padding for the arrays, it avoids resizing every time. + // The arrays must always have exactly n_max entries (the GPU tensor is a fixed size). + int32_t filler = 0; + if (n_active < sctx->n_max) { + while (sctx->token_count.find(filler) != sctx->token_count.end()) { + ++filler; + } + GGML_ASSERT(filler < sctx->n_vocab); + } + + // Fill the rest of the arrays with the filler token id and count 0. + // Inactive slots are padded with a unique dummy token ID (count = 0). + // The uniqueness matters because ggml_set_rows with duplicate indices can produce non-deterministic or incorrect results. + // Using a filler token with count 0 that isn't in the active set is safe, because the active_mask step in backend_apply filters them out via ggml_step(counts_f32) + for (int32_t i = n_active; i < sctx->n_max; ++i) { + sctx->host_token_ids[i] = filler; + sctx->host_counts [i] = 0; + } + + ggml_backend_tensor_set(sctx->inp_token_ids, sctx->host_token_ids.data(), 0, sctx->n_max * sizeof(int32_t)); + ggml_backend_tensor_set(sctx->inp_counts, sctx->host_counts.data(), 0, sctx->n_max * sizeof(int32_t)); +} + static struct llama_sampler_i llama_sampler_penalties_i = { /* .name = */ llama_sampler_penalties_name, /* .accept = */ llama_sampler_penalties_accept, @@ -2753,10 +2958,10 @@ static struct llama_sampler_i llama_sampler_penalties_i = { /* .reset = */ llama_sampler_penalties_reset, /* .clone = */ llama_sampler_penalties_clone, /* .free = */ llama_sampler_penalties_free, - /* .backend_init = */ nullptr, + /* .backend_init = */ llama_sampler_penalties_backend_init, /* .backend_accept = */ nullptr, - /* .backend_apply = */ nullptr, - /* .backend_set_input = */ nullptr, + /* .backend_apply = */ llama_sampler_penalties_backend_apply, + /* .backend_set_input = */ llama_sampler_penalties_backend_set_input, }; struct llama_sampler * llama_sampler_init_penalties( @@ -2766,22 +2971,18 @@ struct llama_sampler * llama_sampler_init_penalties( float penalty_present) { penalty_last_n = std::max(penalty_last_n, 0); - const bool is_empty = (penalty_last_n == 0 || (penalty_repeat == 1.0f && penalty_freq == 0.0f && penalty_present == 0.0f)); - - if (is_empty) { + if (llama_sampler_penalties::is_disabled( + penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) { return llama_sampler_init_empty("?penalties"); } return llama_sampler_init( /* .iface = */ &llama_sampler_penalties_i, - /* .ctx = */ new llama_sampler_penalties { - /* .penalty_last_n = */ penalty_last_n, - /* .penalty_repeat = */ penalty_repeat, - /* .penalty_freq = */ penalty_freq, - /* .penalty_present = */ penalty_present, - /* .prev = */ ring_buffer(penalty_last_n), - /* .token_count = */ {}, - } + /* .ctx = */ new llama_sampler_penalties( + penalty_last_n, + penalty_repeat, + penalty_freq, + penalty_present) ); } diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp index 1d3584f903c4..fd5adb740eab 100644 --- a/tests/test-arg-parser.cpp +++ b/tests/test-arg-parser.cpp @@ -99,6 +99,34 @@ static void test(void) { argv = {"binary_name", "-sm", "hello"}; assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); + { + common_params penalty_params; + + argv = {"binary_name", "--repeat-penalty", "0"}; + assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON)); + + argv = {"binary_name", "--repeat-penalty", "-1"}; + assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON)); + + argv = {"binary_name", "--repeat-penalty", "nan"}; + assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON)); + + argv = {"binary_name", "--repeat-penalty", "inf"}; + assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON)); + + argv = {"binary_name", "--repeat-penalty", "-inf"}; + assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON)); + + const char * penalty_options[] = {"--frequency-penalty", "--presence-penalty"}; + const char * nonfinite_values[] = {"nan", "inf", "-inf"}; + for (const char * option : penalty_options) { + for (const char * value : nonfinite_values) { + argv = {"binary_name", option, value}; + assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON)); + } + } + } + // non-existence arg in specific example (--draft cannot be used outside llama-speculative) argv = {"binary_name", "--draft", "123"}; assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_EMBEDDING)); diff --git a/tests/test-backend-sampler.cpp b/tests/test-backend-sampler.cpp index c24076e31372..1a46468ba264 100644 --- a/tests/test-backend-sampler.cpp +++ b/tests/test-backend-sampler.cpp @@ -8,12 +8,15 @@ #endif #include +#include #include #include #include +#include #include #include #include +#include #include struct test_args { @@ -761,6 +764,563 @@ static void test_backend_logit_bias_sampling(const test_params & params) { printf("backend logit bias sampling test PASSED\n"); } +static void accept_prompt(llama_sampler * smpl, const llama_vocab * vocab, const std::string & prompt) { + const llama_token bos = llama_vocab_bos(vocab); + if (bos != LLAMA_TOKEN_NULL) { + llama_sampler_accept(smpl, bos); + } + + std::vector tokens(64); + int32_t n_tokens = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), + tokens.data(), (int32_t) tokens.size(), false, false); + if (n_tokens < 0) { + tokens.resize(-n_tokens); + n_tokens = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), + tokens.data(), (int32_t) tokens.size(), false, false); + } + + for (int32_t i = 0; i < n_tokens; ++i) { + llama_sampler_accept(smpl, tokens[i]); + } +} + +static std::vector decode_raw_logits(const test_params & params, const std::string & prompt) { + const int seq_id = 0; + const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(params.model.get())); + std::vector empty_configs; + test_context ctx(params, empty_configs); + + GGML_ASSERT(ctx.decode({{ seq_id, prompt }})); + + float * logits = llama_get_logits_ith(ctx.ctx.get(), ctx.idx_for_seq(seq_id)); + GGML_ASSERT(logits != nullptr); + return std::vector(logits, logits + n_vocab); +} + +static std::vector apply_cpu_sampler( + const std::vector & raw_logits, + llama_sampler * sampler) { + std::vector data; + data.reserve(raw_logits.size()); + for (llama_token token = 0; token < (llama_token) raw_logits.size(); ++token) { + data.push_back({ token, raw_logits[token], 0.0f }); + } + + llama_token_data_array cur_p = { data.data(), data.size(), -1, false }; + llama_sampler_apply(sampler, &cur_p); + data.resize(cur_p.size); + return data; +} + +using sampler_setup_fn = std::function; +using sampler_init_fn = std::function; + +enum class penalties_position { + before_filter, + after_filter, +}; + +static void add_filter_and_penalties( + llama_sampler * chain, + const sampler_init_fn & init_filter, + int32_t penalty_last_n, + float penalty_repeat, + float penalty_freq, + float penalty_present, + penalties_position position) { + const auto add_penalties = [&]() { + llama_sampler_chain_add(chain, llama_sampler_init_penalties( + penalty_last_n, penalty_repeat, penalty_freq, penalty_present)); + }; + + if (position == penalties_position::before_filter) { + add_penalties(); + llama_sampler_chain_add(chain, init_filter()); + } else { + llama_sampler_chain_add(chain, init_filter()); + add_penalties(); + } +} + +static llama_sampler_ptr make_sampler_chain( + const sampler_setup_fn & add_samplers, + const sampler_setup_fn & accept_history) { + llama_sampler_ptr chain(llama_sampler_chain_init(llama_sampler_chain_default_params())); + add_samplers(chain.get()); + accept_history(chain.get()); + return chain; +} + +struct backend_sampler_output { + std::vector logits; + std::vector candidates; +}; + +static backend_sampler_output run_backend_sampler( + const test_params & params, + const std::string & prompt, + llama_sampler * sampler) { + const int seq_id = 0; + std::vector configs = {{ seq_id, sampler }}; + test_context ctx(params, configs); + + GGML_ASSERT(ctx.decode({{ seq_id, prompt }})); + llama_synchronize(ctx.ctx.get()); + + const int32_t idx = ctx.idx_for_seq(seq_id); + const uint32_t n_logits = llama_get_sampled_logits_count_ith(ctx.ctx.get(), idx); + const uint32_t n_candidates = llama_get_sampled_candidates_count_ith(ctx.ctx.get(), idx); + float * logits = llama_get_sampled_logits_ith(ctx.ctx.get(), idx); + llama_token * candidates = llama_get_sampled_candidates_ith(ctx.ctx.get(), idx); + GGML_ASSERT(logits != nullptr); + + backend_sampler_output result; + result.logits.assign(logits, logits + n_logits); + result.candidates.resize(n_logits); + + if (n_candidates == 0) { + for (uint32_t i = 0; i < n_logits; ++i) { + result.candidates[i] = (llama_token) i; + } + } else { + GGML_ASSERT(candidates != nullptr); + GGML_ASSERT(n_candidates == n_logits); + std::memcpy(result.candidates.data(), candidates, n_candidates * sizeof(llama_token)); + } + + return result; +} + +struct sampler_comparison_output { + std::vector expected; + backend_sampler_output actual; +}; + +static sampler_comparison_output run_sampler_comparison( + const test_params & params, + const std::string & prompt, + const std::vector & raw_logits, + const sampler_setup_fn & add_samplers, + const sampler_setup_fn & accept_history) { + llama_sampler_ptr cpu_chain = make_sampler_chain(add_samplers, accept_history); + llama_sampler_ptr backend_chain = make_sampler_chain(add_samplers, accept_history); + return { + apply_cpu_sampler(raw_logits, cpu_chain.get()), + run_backend_sampler(params, prompt, backend_chain.get()), + }; +} + +static std::unordered_map map_logits(const std::vector & data) { + std::unordered_map result; + result.reserve(data.size()); + for (const auto & item : data) { + result[item.id] = item.logit; + } + return result; +} + +struct sampler_comparison_stats { + int n_mismatch = 0; + int n_masked = 0; + float max_diff = 0.0f; +}; + +static sampler_comparison_stats compare_sampler_outputs( + const char * name, + const std::unordered_map & expected, + const backend_sampler_output & actual, + bool allow_extra_candidates = false) { + GGML_ASSERT(actual.logits.size() == actual.candidates.size()); + + sampler_comparison_stats result; + std::unordered_set seen; + seen.reserve(actual.candidates.size()); + + for (size_t i = 0; i < actual.logits.size(); ++i) { + const llama_token token = actual.candidates[i]; + const float logit = actual.logits[i]; + if (!seen.insert(token).second || std::isnan(logit)) { + if (result.n_mismatch < 5) { + printf("%s token %d has invalid backend output\n", name, token); + } + ++result.n_mismatch; + continue; + } + + const auto it = expected.find(token); + if (it == expected.end()) { + if (std::isinf(logit) && logit < 0.0f) { + ++result.n_masked; + } else if (!allow_extra_candidates) { + if (result.n_mismatch < 5) { + printf("%s token %d was not masked\n", name, token); + } + ++result.n_mismatch; + } + continue; + } + + const float diff = fabsf(it->second - logit); + result.max_diff = std::max(result.max_diff, diff); + if (!std::isfinite(logit) || diff > 1e-3f) { + if (result.n_mismatch < 5) { + printf("%s mismatch token %d: cpu=%.6f backend=%.6f diff=%.6f\n", + name, token, it->second, logit, diff); + } + ++result.n_mismatch; + } + } + + for (const auto & item : expected) { + if (seen.find(item.first) == seen.end()) { + if (result.n_mismatch < 5) { + printf("%s missing backend token %d\n", name, item.first); + } + ++result.n_mismatch; + } + } + + printf("%s logits: max_diff=%.6f n_masked=%d n_mismatch=%d\n", + name, result.max_diff, result.n_masked, result.n_mismatch); + return result; +} + +static float find_backend_logit(const backend_sampler_output & output, llama_token token) { + for (size_t i = 0; i < output.candidates.size(); ++i) { + if (output.candidates[i] == token) { + return output.logits[i]; + } + } + GGML_ABORT("backend token not found"); +} + +static sampler_comparison_output run_penalties_comparison( + const test_params & params, + int32_t penalty_last_n, + float penalty_repeat, + float penalty_freq, + float penalty_present, + const std::string & prompt, + const std::function & extra_accept = {}) { + const auto * vocab = llama_model_get_vocab(params.model.get()); + const std::vector raw_logits = decode_raw_logits(params, prompt); + const auto add_samplers = [&](llama_sampler * chain) { + llama_sampler_chain_add(chain, llama_sampler_init_penalties( + penalty_last_n, penalty_repeat, penalty_freq, penalty_present)); + }; + const auto accept_history = [&](llama_sampler * chain) { + accept_prompt(chain, vocab, prompt); + if (extra_accept) { + extra_accept(chain); + } + }; + + return run_sampler_comparison( + params, prompt, raw_logits, add_samplers, accept_history); +} + +static void compare_penalties_logits( + const test_params & params, + int32_t penalty_last_n, + float penalty_repeat, + float penalty_freq, + float penalty_present, + const std::string & prompt, + const std::function & extra_accept = {}) { + const sampler_comparison_output output = run_penalties_comparison( + params, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, prompt, extra_accept); + + GGML_ASSERT(output.expected.size() == output.actual.logits.size()); + + const sampler_comparison_stats stats = compare_sampler_outputs( + "penalties", map_logits(output.expected), output.actual); + GGML_ASSERT(stats.n_masked == 0); + GGML_ASSERT(stats.n_mismatch == 0); +} + +static void test_penalty_parameter_values(const test_params & params) { + struct penalty_test_case { + const char * name; + float repeat; + float frequency; + float presence; + }; + + const penalty_test_case cases[] = { + { "frequency -1", 1.0f, -1.0f, 0.0f }, + { "frequency 0", 1.0f, 0.0f, 0.0f }, + { "frequency 1", 1.0f, 1.0f, 0.0f }, + { "presence -1", 1.0f, 0.0f, -1.0f }, + { "presence 0", 1.0f, 0.0f, 0.0f }, + { "presence 1", 1.0f, 0.0f, 1.0f }, + { "repeat 1", 1.0f, 0.0f, 0.0f }, + }; + + int n_failed = 0; + for (const auto & test : cases) { + const sampler_comparison_output output = run_penalties_comparison( + params, 64, test.repeat, test.frequency, test.presence, "Hello Hello world"); + GGML_ASSERT(output.expected.size() == output.actual.logits.size()); + const sampler_comparison_stats stats = compare_sampler_outputs( + test.name, map_logits(output.expected), output.actual); + n_failed += stats.n_mismatch != 0; + } + + GGML_ASSERT(n_failed == 0); +} + +static void compare_top_k_penalties_logits( + const test_params & params, + int32_t k, + int32_t penalty_last_n, + float penalty_repeat, + float penalty_freq, + float penalty_present, + const std::string & prompt, + penalties_position position) { + const auto * vocab = llama_model_get_vocab(params.model.get()); + const std::vector raw_logits = decode_raw_logits(params, prompt); + const int n_vocab = (int) raw_logits.size(); + + GGML_ASSERT(n_vocab > k); + + const sampler_init_fn init_top_k = [k]() { + return llama_sampler_init_top_k(k); + }; + llama_sampler_ptr top_k(init_top_k()); + const std::vector top_k_data = apply_cpu_sampler(raw_logits, top_k.get()); + GGML_ASSERT(top_k_data.size() == (size_t) k); + const llama_token retained_history_token = top_k_data[0].id; + + llama_token excluded_history_token = LLAMA_TOKEN_NULL; + for (llama_token token = 0; token < n_vocab; ++token) { + const auto it = std::find_if(top_k_data.begin(), top_k_data.end(), [token](const llama_token_data & data) { + return data.id == token; + }); + if (it == top_k_data.end()) { + excluded_history_token = token; + break; + } + } + GGML_ASSERT(excluded_history_token != LLAMA_TOKEN_NULL); + + const auto add_samplers = [&](llama_sampler * chain) { + add_filter_and_penalties(chain, init_top_k, + penalty_last_n, penalty_repeat, penalty_freq, penalty_present, position); + }; + + auto accept_history = [&](llama_sampler * smpl) { + accept_prompt(smpl, vocab, prompt); + llama_sampler_accept(smpl, excluded_history_token); + llama_sampler_accept(smpl, excluded_history_token); + llama_sampler_accept(smpl, retained_history_token); + llama_sampler_accept(smpl, retained_history_token); + }; + + const sampler_comparison_output output = run_sampler_comparison( + params, prompt, raw_logits, add_samplers, accept_history); + + GGML_ASSERT(output.expected.size() == (size_t) k); + GGML_ASSERT(output.actual.logits.size() == (size_t) k); + + const std::unordered_map expected_logits = map_logits(output.expected); + + if (position == penalties_position::after_filter) { + GGML_ASSERT(expected_logits.find(retained_history_token) != expected_logits.end()); + GGML_ASSERT(fabsf(expected_logits.at(retained_history_token) - raw_logits[retained_history_token]) > 1e-6f); + GGML_ASSERT(expected_logits.find(excluded_history_token) == expected_logits.end()); + GGML_ASSERT(std::find(output.actual.candidates.begin(), output.actual.candidates.end(), + excluded_history_token) == output.actual.candidates.end()); + } else { + const std::unordered_map unpenalized_logits = map_logits(top_k_data); + bool changed = false; + for (const auto & item : expected_logits) { + const auto it = unpenalized_logits.find(item.first); + if (it == unpenalized_logits.end() || fabsf(it->second - item.second) > 1e-6f) { + changed = true; + break; + } + } + GGML_ASSERT(changed); + } + + const char * name = position == penalties_position::before_filter + ? "penalties top-k" + : "top-k penalties"; + const sampler_comparison_stats stats = compare_sampler_outputs( + name, expected_logits, output.actual); + GGML_ASSERT(stats.n_masked == 0); + GGML_ASSERT(stats.n_mismatch == 0); +} + +static void compare_masking_penalties_logits( + const test_params & params, + const char * filter_name, + const sampler_init_fn & init_filter, + int32_t penalty_last_n, + float penalty_repeat, + float penalty_freq, + float penalty_present, + const std::string & prompt, + penalties_position position, + bool allow_extra_candidates, + bool add_history = true) { + const auto * vocab = llama_model_get_vocab(params.model.get()); + const std::vector raw_logits = decode_raw_logits(params, prompt); + const int n_vocab = (int) raw_logits.size(); + llama_sampler_ptr filter(init_filter()); + const std::vector filtered_data = apply_cpu_sampler(raw_logits, filter.get()); + GGML_ASSERT(!filtered_data.empty()); + GGML_ASSERT(filtered_data.size() < (size_t) n_vocab); + + const llama_token penalized_token = filtered_data[0].id; + std::unordered_set retained_tokens; + retained_tokens.reserve(filtered_data.size()); + for (const auto & data : filtered_data) { + retained_tokens.insert(data.id); + } + + llama_token masked_token = LLAMA_TOKEN_NULL; + for (llama_token token = 0; token < n_vocab; ++token) { + if (retained_tokens.find(token) == retained_tokens.end()) { + masked_token = token; + break; + } + } + GGML_ASSERT(masked_token != LLAMA_TOKEN_NULL); + + const auto add_samplers = [&](llama_sampler * chain) { + add_filter_and_penalties(chain, init_filter, + penalty_last_n, penalty_repeat, penalty_freq, penalty_present, position); + }; + auto accept_history = [&](llama_sampler * smpl) { + if (!add_history) { + return; + } + accept_prompt(smpl, vocab, prompt); + llama_sampler_accept(smpl, penalized_token); + llama_sampler_accept(smpl, penalized_token); + llama_sampler_accept(smpl, masked_token); + llama_sampler_accept(smpl, masked_token); + }; + + const sampler_comparison_output output = run_sampler_comparison( + params, prompt, raw_logits, add_samplers, accept_history); + + GGML_ASSERT(output.actual.logits.size() == (size_t) n_vocab); + + const std::unordered_map expected_logits = map_logits(output.expected); + + GGML_ASSERT(expected_logits.find(masked_token) == expected_logits.end()); + if (add_history) { + if (position == penalties_position::after_filter) { + GGML_ASSERT(expected_logits.find(penalized_token) != expected_logits.end()); + GGML_ASSERT(fabsf(expected_logits.at(penalized_token) - raw_logits[penalized_token]) > 1e-6f); + } else { + llama_sampler_ptr penalties(llama_sampler_init_penalties( + penalty_last_n, penalty_repeat, penalty_freq, penalty_present)); + accept_history(penalties.get()); + const std::unordered_map penalized_logits = + map_logits(apply_cpu_sampler(raw_logits, penalties.get())); + GGML_ASSERT(fabsf(penalized_logits.at(penalized_token) - raw_logits[penalized_token]) > 1e-6f); + } + } + + const std::string name = position == penalties_position::before_filter + ? "penalties " + std::string(filter_name) + : std::string(filter_name) + " penalties"; + const sampler_comparison_stats stats = compare_sampler_outputs( + name.c_str(), expected_logits, output.actual, allow_extra_candidates); + const float masked_logit = find_backend_logit(output.actual, masked_token); + GGML_ASSERT(stats.n_masked > 0); + GGML_ASSERT(std::isinf(masked_logit) && masked_logit < 0.0f); + GGML_ASSERT(stats.n_mismatch == 0); +} + +static void test_backend_penalties_sampling(const test_params & params) { + printf("Testing backend penalties (repeat + freq + presence)\n"); + compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello Hello world"); + + printf("Testing backend penalties with penalty_last_n > 64\n"); + const auto * vocab = llama_model_get_vocab(params.model.get()); + std::vector tokens(8); + int32_t n_tok = llama_tokenize(vocab, "a", 1, tokens.data(), (int32_t) tokens.size(), false, false); + if (n_tok < 0) { + tokens.resize(-n_tok); + n_tok = llama_tokenize(vocab, "a", 1, tokens.data(), (int32_t) tokens.size(), false, false); + } + GGML_ASSERT(n_tok > 0); + const llama_token tok = tokens[0]; + + compare_penalties_logits(params, 80, 1.15f, 0.1f, 0.05f, "a", [tok](llama_sampler * smpl) { + // accept_prompt already accepted BOS + one 'a'; fill the ring to n=80 + for (int i = 0; i < 78; ++i) { + llama_sampler_accept(smpl, tok); + } + }); + + printf("Testing backend penalties without filler entries\n"); + compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello", [](llama_sampler * smpl) { + for (llama_token token = 0; token < 64; ++token) { + llama_sampler_accept(smpl, token); + } + }); + + printf("Testing backend top-k followed by penalties\n"); + compare_top_k_penalties_logits(params, 8, 64, 1.1f, 0.5f, 0.25f, "Hello", + penalties_position::after_filter); + + printf("Testing backend penalties followed by top-k\n"); + compare_top_k_penalties_logits(params, 8, 64, 1.1f, 0.5f, 0.25f, "Hello", + penalties_position::before_filter); + + printf("Testing backend top-p followed by penalties\n"); + compare_masking_penalties_logits(params, "top-p", []() { + return llama_sampler_init_top_p(0.9f, 0); + }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true); + + printf("Testing backend top-p followed by penalties with a large history window\n"); + compare_masking_penalties_logits(params, "top-p large-window", []() { + return llama_sampler_init_top_p(0.9f, 0); + }, 4096, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true); + + printf("Testing backend penalties followed by top-p\n"); + compare_masking_penalties_logits(params, "top-p", []() { + return llama_sampler_init_top_p(0.9f, 0); + }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter, true); + + printf("Testing backend min-p followed by penalties\n"); + compare_masking_penalties_logits(params, "min-p", []() { + return llama_sampler_init_min_p(0.1f, 0); + }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, false); + + printf("Testing backend penalties followed by min-p\n"); + compare_masking_penalties_logits(params, "min-p", []() { + return llama_sampler_init_min_p(0.1f, 0); + }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter, false); + + printf("Testing backend top-p followed by penalties with empty history\n"); + compare_masking_penalties_logits(params, "top-p empty", []() { + return llama_sampler_init_top_p(0.9f, 0); + }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true, false); + + printf("Testing backend top-p followed by individual penalties\n"); + compare_masking_penalties_logits(params, "top-p repeat", []() { + return llama_sampler_init_top_p(0.9f, 0); + }, 64, 1.1f, 0.0f, 0.0f, "Hello", penalties_position::after_filter, true); + compare_masking_penalties_logits(params, "top-p frequency", []() { + return llama_sampler_init_top_p(0.9f, 0); + }, 64, 1.0f, 0.5f, 0.0f, "Hello", penalties_position::after_filter, true); + compare_masking_penalties_logits(params, "top-p presence", []() { + return llama_sampler_init_top_p(0.9f, 0); + }, 64, 1.0f, 0.0f, 0.25f, "Hello", penalties_position::after_filter, true); + + printf("Testing backend penalty parameter values\n"); + test_penalty_parameter_values(params); + + printf("backend penalties sampling test PASSED\n"); +} + // This test verifies that it is possible to have two different backend samplers, // one that uses the backend dist sampler, and another that uses CPU dist sampler. static void test_backend_mixed_sampling(const test_params & params) { @@ -1014,6 +1574,7 @@ struct backend_test_case { static const backend_test_case BACKEND_TESTS[] = { { "greedy", test_backend_greedy_sampling, true }, { "logit_bias", test_backend_logit_bias_sampling, true }, + { "penalties", test_backend_penalties_sampling, true }, { "temp", test_backend_temp_sampling, true }, { "temp_ext", test_backend_temp_ext_sampling, true }, { "top_k", test_backend_top_k_sampling, true }, diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 4655b518e21f..5d2798cc14e9 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1807,7 +1807,8 @@ struct server_context_impl { // initialize samplers if (task.need_sampling()) { try { - slot.smpl.reset(common_sampler_init(model_tgt, task.params.sampling)); + slot.smpl.reset(common_sampler_init( + model_tgt, task.params.sampling, (int32_t) llama_n_ctx(ctx_tgt))); } catch (std::exception & e) { std::string err_msg = std::string("Failed to initialize samplers: ") + e.what(); send_error(task, err_msg, ERROR_TYPE_INVALID_REQUEST); From 563dec81c1c538aac0fad465ea933eb2a621a183 Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:56:30 +0200 Subject: [PATCH 39/98] llama : allocate indexer cache only in "full" indexer layers (#26474) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stanisław Szymczyk --- src/llama-kv-cache-dsa.cpp | 7 ++++--- src/llama-kv-cache-dsa.h | 3 ++- src/llama-model.cpp | 8 +++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/llama-kv-cache-dsa.cpp b/src/llama-kv-cache-dsa.cpp index 241c50365a13..96cb045d2e5d 100644 --- a/src/llama-kv-cache-dsa.cpp +++ b/src/llama-kv-cache-dsa.cpp @@ -23,7 +23,8 @@ llama_kv_cache_dsa::llama_kv_cache_dsa( uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, - const layer_filter_cb & filter, + const layer_filter_cb & filter_mla, + const layer_filter_cb & filter_lid, const layer_reuse_cb & reuse) : hparams_lid(model.hparams), n_stream(unified ? 1 : n_seq_max) { @@ -32,7 +33,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa( kv_mla = std::make_unique( model, model.hparams, type_k, type_v, v_trans, offload, unified, kv_size, n_seq_max, n_pad, - n_swa, swa_type, nullptr, filter, reuse, nullptr); + n_swa, swa_type, nullptr, filter_mla, reuse, nullptr); // we use llama_kv_cache for caching indexer keys // by hand-tweaking some hparams we fool it to create @@ -49,7 +50,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa( kv_lid = std::make_unique( model, hparams_lid, type_k, type_v, v_trans, offload, unified, kv_size, n_seq_max, n_pad, - n_swa, swa_type, nullptr, filter, reuse, nullptr); + n_swa, swa_type, nullptr, filter_lid, reuse, nullptr); } void llama_kv_cache_dsa::clear(bool data) { diff --git a/src/llama-kv-cache-dsa.h b/src/llama-kv-cache-dsa.h index e2b330993b84..e74fc4d9100f 100644 --- a/src/llama-kv-cache-dsa.h +++ b/src/llama-kv-cache-dsa.h @@ -26,7 +26,8 @@ class llama_kv_cache_dsa : public llama_memory_i { uint32_t n_pad, uint32_t n_swa, llama_swa_type swa_type, - const layer_filter_cb & filter, + const layer_filter_cb & filter_mla, + const layer_filter_cb & filter_lid, const layer_reuse_cb & reuse); ~llama_kv_cache_dsa() = default; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 4b4fe4712c9e..938d98798cdb 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2101,10 +2101,11 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, } else { // Main context: DSA cache for the trunk layers only - the nextn // layer(s) are never attended by the trunk graph. - llama_kv_cache::layer_filter_cb filter = nullptr; + llama_kv_cache::layer_filter_cb filter_mla = nullptr; if (hparams.n_layer_nextn > 0) { - filter = [&](uint32_t il) { return il < hparams.n_layer(); }; + filter_mla = [&](uint32_t il) { return il < hparams.n_layer(); }; } + llama_kv_cache::layer_filter_cb filter_lid = [&](uint32_t il) { return il < hparams.n_layer() && (arch != LLM_ARCH_GLM_DSA || hparams.is_indexer_full(il)); }; res = new llama_kv_cache_dsa( *this, @@ -2118,7 +2119,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, 1, hparams.n_swa, hparams.swa_type, - filter, + filter_mla, + filter_lid, nullptr); } } break; From 67d5978bb1ba490070ddbd2a7c59c7ec5a60b7cc Mon Sep 17 00:00:00 2001 From: timkhronos Date: Mon, 3 Aug 2026 15:30:08 +0200 Subject: [PATCH 40/98] model: M3: Move MSA into a new memory implementation (#26338) * Move MSA logic from llama-kv-cache into llama-kv-cache-msa * cont : minor * cont : ws fix --------- Co-authored-by: Georgi Gerganov --- src/CMakeLists.txt | 1 + src/llama-graph.cpp | 57 ++++++ src/llama-graph.h | 23 +++ src/llama-hparams.cpp | 10 - src/llama-hparams.h | 5 - src/llama-kv-cache-msa.cpp | 395 +++++++++++++++++++++++++++++++++++++ src/llama-kv-cache-msa.h | 153 ++++++++++++++ src/llama-kv-cache.cpp | 298 ++-------------------------- src/llama-kv-cache.h | 12 +- src/llama-model.cpp | 23 +++ src/models/minimax-m3.cpp | 225 ++++++++++++++------- 11 files changed, 825 insertions(+), 377 deletions(-) create mode 100644 src/llama-kv-cache-msa.cpp create mode 100644 src/llama-kv-cache-msa.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 320784c3a8cc..24f05cc91673 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -25,6 +25,7 @@ add_library(llama llama-kv-cache.cpp llama-kv-cache-iswa.cpp llama-kv-cache-dsa.cpp + llama-kv-cache-msa.cpp llama-kv-cache-dsv4.cpp llama-memory.cpp llama-memory-hybrid.cpp diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 1a35692300c1..fdab7b8dde67 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -8,6 +8,7 @@ #include "llama-kv-cache.h" #include "llama-kv-cache-iswa.h" #include "llama-kv-cache-dsa.h" +#include "llama-kv-cache-msa.h" #include "llama-kv-cache-dsv4.h" #include "llama-memory-hybrid.h" #include "llama-memory-hybrid-iswa.h" @@ -518,6 +519,36 @@ bool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) { return res; } +llm_graph_input_attn_kv_msa::llm_graph_input_attn_kv_msa( + const llama_hparams & hparams, + const llama_cparams & cparams, + const llama_kv_cache_msa_context * mctx) : + llm_graph_input_attn_kv(hparams, cparams, mctx->get_base()), + mctx_msa(mctx) { +} + +void llm_graph_input_attn_kv_msa::set_input(const llama_ubatch * ubatch) { + llm_graph_input_attn_kv::set_input(ubatch); + + mctx_msa->get_idx()->set_input_k_idxs(self_k_idxs_idx, ubatch); +} + +bool llm_graph_input_attn_kv_msa::can_reuse(const llm_graph_params & params) { + mctx_msa = static_cast(params.mctx); + + // the parent class operates on the base cache context + this->mctx = mctx_msa->get_base(); + + bool res = true; + + res &= self_k_idxs ->ne[0] == params.ubatch.n_tokens; + res &= self_k_idxs_idx->ne[0] == params.ubatch.n_tokens; + + res &= can_reuse_kq_mask(self_kq_mask, this->mctx, params.ubatch, params.cparams); + + return res; +} + void llm_graph_input_attn_k_dsa::set_input(const llama_ubatch * ubatch) { mctx->get_mla()->set_input_k_idxs(self_k_idxs_mla, ubatch); @@ -3187,6 +3218,32 @@ llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const { return (llm_graph_input_attn_k_dsa *) res->add_input(std::move(inp)); } +llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa() const { + const auto * mctx_cur = static_cast(mctx); + + auto inp = std::make_unique(hparams, cparams, mctx_cur); + + const auto * mctx_base = mctx_cur->get_base(); + const auto * mctx_idx = mctx_cur->get_idx(); + + { + GGML_ASSERT(hparams.swa_type == LLAMA_SWA_TYPE_NONE && "Use llama_kv_cache_iswa for SWA"); + + inp->self_k_idxs = mctx_base->build_input_k_idxs(ctx0, ubatch); + inp->self_v_idxs = mctx_base->build_input_v_idxs(ctx0, ubatch); + + inp->self_kq_mask = build_attn_inp_kq_mask(ctx0, mctx_base, ubatch, cparams); + inp->self_kq_mask_cnv = inp->self_kq_mask; + } + + inp->self_k_rot = mctx_base->build_input_k_rot(ctx0); + inp->self_v_rot = mctx_base->build_input_v_rot(ctx0); + + inp->self_k_idxs_idx = mctx_idx->build_input_k_idxs(ctx0, ubatch); + + return (llm_graph_input_attn_kv_msa *) res->add_input(std::move(inp)); +} + // TODO: maybe separate the inner implementation into a separate function // like with the non-sliding window equivalent // once sliding-window hybrid caches are a thing. diff --git a/src/llama-graph.h b/src/llama-graph.h index 160e29413552..ff216302dbb1 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -23,6 +23,7 @@ struct llama_memory_context_i; class llama_kv_cache_context; class llama_kv_cache_dsa_context; +class llama_kv_cache_msa_context; class llama_kv_cache_dsv4_raw_context; class llama_kv_cache_dsv4_context; class llama_kv_cache_iswa_context; @@ -425,6 +426,26 @@ class llm_graph_input_attn_k_dsa : public llm_graph_input_i { const llama_kv_cache_dsa_context * mctx; }; +// standard K/V attention input against the base cache, plus destination indices for the indexer key cache +class llm_graph_input_attn_kv_msa : public llm_graph_input_attn_kv { +public: + llm_graph_input_attn_kv_msa( + const llama_hparams & hparams, + const llama_cparams & cparams, + const llama_kv_cache_msa_context * mctx); + ~llm_graph_input_attn_kv_msa() = default; + + void set_input(const llama_ubatch * ubatch) override; + + bool can_reuse(const llm_graph_params & params) override; + + ggml_tensor * get_k_idxs_idx() const { return self_k_idxs_idx; } + + ggml_tensor * self_k_idxs_idx = nullptr; // I64 [n_batch] + + const llama_kv_cache_msa_context * mctx_msa; +}; + class llm_graph_input_attn_kv_iswa : public llm_graph_input_i { public: llm_graph_input_attn_kv_iswa( @@ -1169,6 +1190,8 @@ struct llm_graph_context { llm_graph_input_attn_k_dsa * build_attn_inp_k_dsa() const; + llm_graph_input_attn_kv_msa * build_attn_inp_kv_msa() const; + ggml_tensor * build_attn( llm_graph_input_attn_k_dsa * inp, ggml_tensor * wo, diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 50af97f358c3..846d4c69a626 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -180,16 +180,6 @@ uint32_t llama_hparams::n_embd_v_gqa_max() const { return val; } -uint32_t llama_hparams::n_embd_k_idx(uint32_t il) const { - if (!indexer_kv || indexer_head_size == 0) { - return 0; // arch without a MSA indexer - } - if (il < n_layer_dense_lead) { - return 0; // leading dense layers carry no indexer - } - return indexer_head_size; // 128 -} - uint32_t llama_hparams::n_embd_r() const { if (wkv_head_size != 0) { // for RWKV models diff --git a/src/llama-hparams.h b/src/llama-hparams.h index fc770bf003e6..6e8336c98748 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -230,8 +230,6 @@ struct llama_hparams { // MSA uint32_t indexer_block_size = 0; uint32_t indexer_local_blocks = 0; - // MSA stores its indexer keys in the main KV cache (k_idx tensors); - bool indexer_kv = false; // Indexer is "full" (1) or "shared" (0) // Shared indexers reuse top-k from previous full layer @@ -356,9 +354,6 @@ struct llama_hparams { uint32_t n_embd_k_gqa_max() const; uint32_t n_embd_v_gqa_max() const; - // dimension of the single-head MSA indexer key stream - uint32_t n_embd_k_idx(uint32_t il = 0) const; - // dimension of the rolling state embeddings // corresponds to Mamba's conv_states size or RWKV's token_shift states size uint32_t n_embd_r() const; diff --git a/src/llama-kv-cache-msa.cpp b/src/llama-kv-cache-msa.cpp new file mode 100644 index 000000000000..55ef286cafa7 --- /dev/null +++ b/src/llama-kv-cache-msa.cpp @@ -0,0 +1,395 @@ +#include "llama-kv-cache-msa.h" + +#include "llama-impl.h" +#include "llama-batch.h" +#include "llama-model.h" + +#include +#include +#include + +// llama_kv_cache_msa + +llama_kv_cache_msa::llama_kv_cache_msa( + const llama_model & model, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, + llama_swa_type swa_type, + const layer_filter_cb & filter, + const layer_filter_cb & filter_idx, + const layer_reuse_cb & reuse) : + hparams_idx(model.hparams), + n_stream(unified ? 1 : n_seq_max), n_seq_max(n_seq_max), n_pad(n_pad), + n_swa(n_swa), swa_type(swa_type) { + + LLAMA_LOG_INFO("%s: creating main KV cache, size = %u cells\n", __func__, kv_size); + + kv_base = std::make_unique( + model, model.hparams, type_k, type_v, + v_trans, offload, unified, kv_size, n_seq_max, n_pad, + n_swa, swa_type, nullptr, filter, reuse, nullptr); + + // the MSA indexer uses a single key head per layer + std::fill(hparams_idx.n_head_kv_arr.begin(), hparams_idx.n_head_kv_arr.end(), 1); + hparams_idx.n_embd_head_k_full = model.hparams.indexer_head_size; + // the rope parameters are kept identical to the main cache + + LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size); + + kv_idx = std::make_unique( + model, hparams_idx, type_k, type_v, + v_trans, offload, unified, kv_size, n_seq_max, n_pad, + n_swa, swa_type, nullptr, filter_idx, reuse, nullptr); +} + +void llama_kv_cache_msa::clear(bool data) { + kv_base->clear(data); + kv_idx ->clear(data); +} + +bool llama_kv_cache_msa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { + bool res = true; + + res = res & kv_base->seq_rm(seq_id, p0, p1); + res = res & kv_idx ->seq_rm(seq_id, p0, p1); + + return res; +} + +void llama_kv_cache_msa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { + kv_base->seq_cp(seq_id_src, seq_id_dst, p0, p1); + kv_idx ->seq_cp(seq_id_src, seq_id_dst, p0, p1); +} + +void llama_kv_cache_msa::seq_keep(llama_seq_id seq_id) { + kv_base->seq_keep(seq_id); + kv_idx ->seq_keep(seq_id); +} + +void llama_kv_cache_msa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) { + kv_base->seq_add(seq_id, p0, p1, shift); + kv_idx ->seq_add(seq_id, p0, p1, shift); +} + +void llama_kv_cache_msa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) { + kv_base->seq_div(seq_id, p0, p1, d); + kv_idx ->seq_div(seq_id, p0, p1, d); +} + +llama_pos llama_kv_cache_msa::seq_pos_min(llama_seq_id seq_id) const { + return kv_base->seq_pos_min(seq_id); +} + +llama_pos llama_kv_cache_msa::seq_pos_max(llama_seq_id seq_id) const { + return kv_base->seq_pos_max(seq_id); +} + +std::map llama_kv_cache_msa::memory_breakdown() const { + std::map mb = kv_base->memory_breakdown(); + for (const auto & buft_size : kv_idx->memory_breakdown()) { + mb[buft_size.first] += buft_size.second; + } + return mb; +} + +llama_memory_context_ptr llama_kv_cache_msa::init_batch( + llama_batch_allocr & balloc, + uint32_t n_ubatch, + bool embd_all) { + GGML_UNUSED(embd_all); + + do { + balloc.split_reset(); + + std::vector ubatches; + while (true) { + auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0); + + if (ubatch.n_tokens == 0) { + break; + } + + ubatches.push_back(std::move(ubatch)); + } + + if (balloc.get_n_used() < balloc.get_n_tokens()) { + // failed to find a suitable split + break; + } + + auto sinfos_base = kv_base->prepare(ubatches); + if (sinfos_base.empty()) { + break; + } + + auto sinfos_idx = kv_idx->prepare(ubatches); + if (sinfos_idx.empty()) { + break; + } + + assert(sinfos_base.size() == sinfos_idx.size()); + + return std::make_unique( + this, std::move(sinfos_base), std::move(sinfos_idx), std::move(ubatches)); + } while (false); + + return std::make_unique(LLAMA_MEMORY_STATUS_FAILED_PREPARE); +} + +llama_memory_context_ptr llama_kv_cache_msa::init_full() { + return std::make_unique(this); +} + +llama_memory_context_ptr llama_kv_cache_msa::init_update(llama_context * lctx, bool optimize) { + return std::make_unique(this, lctx, optimize); +} + +bool llama_kv_cache_msa::get_can_shift() const { + return kv_base->get_can_shift() && + kv_idx ->get_can_shift() && + kv_base->get_size() == kv_idx->get_size(); +} + +void llama_kv_cache_msa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const { + kv_base->state_write(io, seq_id, flags); + kv_idx ->state_write(io, seq_id, flags); +} + +void llama_kv_cache_msa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { + kv_base->state_read(io, seq_id, flags); + kv_idx ->state_read(io, seq_id, flags); +} + +llama_kv_cache * llama_kv_cache_msa::get_base() const { + return kv_base.get(); +} + +llama_kv_cache * llama_kv_cache_msa::get_idx() const { + return kv_idx.get(); +} + +// llama_kv_cache_msa_context + +llama_kv_cache_msa_context::llama_kv_cache_msa_context(llama_memory_status status) : + kv(nullptr), status(status) {} + +llama_kv_cache_msa_context::llama_kv_cache_msa_context( + llama_kv_cache_msa * kv) : + kv(kv), + ctx_base(kv->get_base()->init_full()), + ctx_idx (kv->get_idx ()->init_full()), + status(llama_memory_status_combine(ctx_base->get_status(), ctx_idx->get_status())) { +} + +llama_kv_cache_msa_context::llama_kv_cache_msa_context( + llama_kv_cache_msa * kv, + llama_context * lctx, + bool optimize) : + kv(kv), + ctx_base(kv->get_base()->init_update(lctx, optimize)), + ctx_idx (kv->get_idx ()->init_update(lctx, optimize)), + status(llama_memory_status_combine(ctx_base->get_status(), ctx_idx->get_status())) { +} + +llama_kv_cache_msa_context::llama_kv_cache_msa_context( + llama_kv_cache_msa * kv, + slot_info_vec_t sinfos_base, + slot_info_vec_t sinfos_idx, + std::vector ubatches) : + kv(kv), + ubatches(std::move(ubatches)), + // here we copy the ubatches. not sure if this is ideal + ctx_base(new llama_kv_cache_context(kv->get_base(), std::move(sinfos_base), this->ubatches)), + ctx_idx (new llama_kv_cache_context(kv->get_idx (), std::move(sinfos_idx), this->ubatches)), + status(llama_memory_status_combine(ctx_base->get_status(), ctx_idx->get_status())) { +} + +llama_kv_cache_msa_context::~llama_kv_cache_msa_context() = default; + +bool llama_kv_cache_msa_context::next() { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + ctx_base->next(); + ctx_idx ->next(); + + if (++i_next >= ubatches.size()) { + return false; + } + + return true; +} + +bool llama_kv_cache_msa_context::apply() { + assert(!llama_memory_status_is_fail(status)); + + bool res = true; + + res = res & ctx_base->apply(); + res = res & ctx_idx ->apply(); + + return res; +} + +llama_memory_status llama_kv_cache_msa_context::get_status() const { + return status; +} + +const llama_ubatch & llama_kv_cache_msa_context::get_ubatch() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return ubatches[i_next]; +} + +const llama_kv_cache_context * llama_kv_cache_msa_context::get_base() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return static_cast(ctx_base.get()); +} + +const llama_kv_cache_context * llama_kv_cache_msa_context::get_idx() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return static_cast(ctx_idx.get()); +} + +uint32_t llama_kv_cache_msa_context::get_n_pos() const { + // pad the value so that the graph remains constant across batches and can be reused + const uint32_t n_pad_cur = std::max(kv->get_n_pad(), 256u); + + llama_pos pos_max = -1; + + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) kv->get_n_seq_max(); ++seq_id) { + pos_max = std::max(pos_max, kv->seq_pos_max(seq_id)); + } + + return std::max(n_pad_cur, GGML_PAD((uint32_t) (pos_max + 1), n_pad_cur)); +} + +void llama_kv_cache_msa_context::set_input_cell_pos(ggml_tensor * dst, const llama_ubatch * ubatch, int32_t div) const { + GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); + GGML_ASSERT(dst->type == GGML_TYPE_I32); + GGML_ASSERT(div > 0); + + const int64_t n_tokens = ubatch->n_tokens; + const int64_t n_kv = dst->ne[0]; + const int64_t n_stream_ub = dst->ne[1]; + + GGML_ASSERT(n_tokens % n_stream_ub == 0); + const int64_t n_tps = n_tokens/n_stream_ub; + + int32_t * data = (int32_t *) dst->data; + + for (int64_t s = 0; s < n_stream_ub; ++s) { + const llama_seq_id seq_id = ubatch->seq_id[s*n_tps][0]; + + const auto & cells = kv->get_base()->get_cells(seq_id); + + for (int64_t j = 0; j < n_kv; ++j) { + // the value for empty or other-sequence cells is irrelevant as consumers mask them + data[s*n_kv + j] = + cells.is_empty(j) || !cells.seq_has(j, seq_id) + ? 0 + : (int32_t) (cells.pos_get(j)/div); + } + } +} + +void llama_kv_cache_msa_context::set_input_pos_slot(ggml_tensor * dst, const llama_ubatch * ubatch) const { + GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); + GGML_ASSERT(dst->type == GGML_TYPE_I32 || dst->type == GGML_TYPE_F32); + + const int64_t n_tokens = ubatch->n_tokens; + const int64_t n_pos = dst->ne[0]; + const int64_t n_stream_ub = dst->ne[1]; + + GGML_ASSERT(n_tokens % n_stream_ub == 0); + const int64_t n_tps = n_tokens/n_stream_ub; + + for (int64_t s = 0; s < n_stream_ub; ++s) { + const llama_seq_id seq_id = ubatch->seq_id[s*n_tps][0]; + + const auto & cells = kv->get_base()->get_cells(seq_id); + + std::vector map(n_pos, 0); + + for (uint32_t j = 0; j < cells.size(); ++j) { + if (cells.is_empty(j) || !cells.seq_has(j, seq_id)) { + continue; + } + + const llama_pos p0 = cells.pos_get(j); + + if (p0 < 0 || p0 >= n_pos) { + continue; + } + + map[p0] = (int32_t) j; + } + + if (dst->type == GGML_TYPE_I32) { + int32_t * data = (int32_t *) dst->data + s*n_pos; + std::copy(map.begin(), map.end(), data); + } else { + float * data = (float *) dst->data + s*n_pos; + for (int64_t p = 0; p < n_pos; ++p) { + data[p] = (float) map[p]; + } + } + } +} + +void llama_kv_cache_msa_context::set_input_pos_mask(ggml_tensor * dst, const llama_ubatch * ubatch) const { + GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + const int64_t n_tokens = ubatch->n_tokens; + const int64_t n_pos = dst->ne[0]; + + GGML_ASSERT(dst->ne[1] == n_tokens); + + const uint32_t n_swa = kv->get_n_swa(); + const llama_swa_type swa_type = kv->get_swa_type(); + + float * data = (float *) dst->data; + + std::fill(data, data + n_pos*n_tokens, -INFINITY); + + for (int64_t i = 0; i < n_tokens; ++i) { + const llama_seq_id seq_id = ubatch->seq_id[i][0]; + + const auto & cells = kv->get_base()->get_cells(seq_id); + + const llama_pos p1 = ubatch->pos[i]; + + for (uint32_t j = 0; j < cells.size(); ++j) { + if (cells.is_empty(j) || !cells.seq_has(j, seq_id)) { + continue; + } + + const llama_pos p0 = cells.pos_get(j); + + if (p0 < 0 || p0 >= n_pos) { + continue; + } + + // causal mask + if (p0 > p1) { + continue; + } + + // apply SWA if any + if (llama_hparams::is_masked_swa(n_swa, swa_type, p0, p1)) { + continue; + } + + data[i*n_pos + p0] = 0.0f; + } + } +} diff --git a/src/llama-kv-cache-msa.h b/src/llama-kv-cache-msa.h new file mode 100644 index 000000000000..f09b6d32b044 --- /dev/null +++ b/src/llama-kv-cache-msa.h @@ -0,0 +1,153 @@ +#pragma once + +#include "llama-kv-cache.h" + +#include + +// llama_kv_cache_msa + +// uses two instances of llama_kv_cache, one for K/V tensors, and one for the MSA indexer tensors +// both receive identical sequence operations and identical ubatches, so their cell layouts stay in synced. +// the context also exposes per-ubatch pos - cell translation maps populated from llama_kv_cells via +// llama_kv_cache::get_cells(), which the model graph uses to run MSA block selection in position space + +class llama_kv_cache_msa : public llama_memory_i { +public: + llama_kv_cache_msa( + const llama_model & model, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_pad, + uint32_t n_swa, + llama_swa_type swa_type, + const layer_filter_cb & filter, + const layer_filter_cb & filter_idx, + const layer_reuse_cb & reuse); + + ~llama_kv_cache_msa() = default; + + // llama_memory_i + + llama_memory_context_ptr init_batch( + llama_batch_allocr & balloc, + uint32_t n_ubatch, + bool embd_all) override; + + llama_memory_context_ptr init_full() override; + + llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; + + bool get_can_shift() const override; + + void clear(bool data) override; + + bool seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) override; + void seq_cp (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override; + void seq_keep(llama_seq_id seq_id) override; + void seq_add (llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) override; + void seq_div (llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) override; + + llama_pos seq_pos_min(llama_seq_id seq_id) const override; + llama_pos seq_pos_max(llama_seq_id seq_id) const override; + + std::map memory_breakdown() const override; + + // state write/load + + void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override; + void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; + + // llama_kv_cache_msa specific API + + llama_kv_cache * get_base() const; + llama_kv_cache * get_idx () const; + + uint32_t get_n_pad() const { return n_pad; } + uint32_t get_n_seq_max() const { return n_seq_max; } + uint32_t get_n_swa() const { return n_swa; } + llama_swa_type get_swa_type() const { return swa_type; } + +private: + // keep the indexer KV cache hparams instance here as llama_kv_cache stores only a reference + llama_hparams hparams_idx; + + const uint32_t n_stream = 1; + const uint32_t n_seq_max = 1; + const uint32_t n_pad = 1; + + const uint32_t n_swa = 0; + const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE; + + std::unique_ptr kv_base; + std::unique_ptr kv_idx; +}; + +class llama_kv_cache_msa_context : public llama_memory_context_i { +public: + using slot_info_vec_t = llama_kv_cache::slot_info_vec_t; + + // used for errors + llama_kv_cache_msa_context(llama_memory_status status); + + // used to create a full-cache context + llama_kv_cache_msa_context( + llama_kv_cache_msa * kv); + + // used to create an update context + llama_kv_cache_msa_context( + llama_kv_cache_msa * kv, + llama_context * lctx, + bool optimize); + + // used to create a batch processing context from a batch + llama_kv_cache_msa_context( + llama_kv_cache_msa * kv, + slot_info_vec_t sinfos_base, + slot_info_vec_t sinfos_idx, + std::vector ubatches); + + virtual ~llama_kv_cache_msa_context(); + + // llama_memory_context_i + + bool next() override; + bool apply() override; + + llama_memory_status get_status() const override; + const llama_ubatch & get_ubatch() const override; + + // llama_kv_cache_msa_context specific API + + const llama_kv_cache_context * get_base() const; + const llama_kv_cache_context * get_idx () const; + + // max position currently present in the cache plus one, padded MSA blocks are defined over token positions + // so the block-selection tensors are sized by this value rather than by the number of cells + uint32_t get_n_pos() const; + + // position <-> cell translation maps, populated from the base cache cells + // the model graph relates cache contents to token positions only through these per ubatch inputs + // value for empty or other-sequence cells is 0 so consumers must mask them + void set_input_cell_pos(ggml_tensor * dst, const llama_ubatch * ubatch, int32_t div) const; + // positions without a cell map to cell 0, consumers must mask them assumes one sequence per stream + void set_input_pos_slot(ggml_tensor * dst, const llama_ubatch * ubatch) const; + void set_input_pos_mask(ggml_tensor * dst, const llama_ubatch * ubatch) const; + +private: + llama_kv_cache_msa * kv; + + // the index of the next ubatch to process + size_t i_next = 0; + + std::vector ubatches; + + const llama_memory_context_ptr ctx_base; + const llama_memory_context_ptr ctx_idx; + + const llama_memory_status status; +}; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 44cb1668dacf..8678a326d9ee 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -112,7 +112,7 @@ llama_kv_cache::llama_kv_cache( auto it = ctx_map.find(buft); if (it == ctx_map.end()) { ggml_init_params params = { - /*.mem_size =*/ size_t(3u*(1 + n_stream)*n_layer*ggml_tensor_overhead()), //Reserve tensor metadata for up to 3 tensors per layer (K, V, and optional K_idx), plus one view per tensor per stream. + /*.mem_size =*/ size_t(2u*(1 + n_stream)*n_layer*ggml_tensor_overhead()), /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, }; @@ -242,25 +242,9 @@ llama_kv_cache::llama_kv_cache( v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gqa, kv_size, v->nb[1], s*v->nb[2]) : nullptr); } - const uint32_t n_embd_k_idx = hparams.n_embd_k_idx(il); - ggml_tensor * k_idx = n_embd_k_idx > 0 - ? ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_k_idx, kv_size, n_stream) - : nullptr; - if (k_idx) { - ggml_format_name(k_idx, "cache_k_idx_l%d", il); - msa_strict_slots = (n_stream == n_seq_max); - } - - std::vector k_idx_stream; - for (uint32_t s = 0; s < n_stream; ++s) { - k_idx_stream.push_back(k_idx - ? ggml_view_2d(ctx, k_idx, n_embd_k_idx, kv_size, k_idx->nb[1], s*k_idx->nb[2]) - : nullptr); - } - map_layer_ids[il] = layers.size(); - layers.push_back({ il, k, v, k_idx, k_stream, v_stream, k_idx_stream }); + layers.push_back({ il, k, v, k_stream, v_stream, }); } if (reuse) { @@ -309,24 +293,13 @@ llama_kv_cache::llama_kv_cache( } { - const size_t memory_size_k = size_k_bytes(); - const size_t memory_size_v = size_v_bytes(); - const size_t memory_size_k_idx = size_k_idx_bytes(); - const size_t memory_size_total = memory_size_k + memory_size_v + memory_size_k_idx; - - constexpr float mib = 1024.0f * 1024.0f; - - const std::string k_log = format(", K (%s): %7.2f MiB", ggml_type_name(type_k), (float) memory_size_k / mib); - const std::string v_log = format(", V (%s): %7.2f MiB", ggml_type_name(type_v), (float) memory_size_v / mib); - - std::string k_idx_log; - if (memory_size_k_idx > 0) { - k_idx_log = format(", K_idx (%s): %7.2f MiB", ggml_type_name(GGML_TYPE_F32), (float) memory_size_k_idx / mib); - } + const size_t memory_size_k = size_k_bytes(); + const size_t memory_size_v = size_v_bytes(); - LLAMA_LOG_INFO("%s: size = %7.2f MiB (%6u cells, %3d layers, %2u/%u seqs)%s%s%s\n", __func__, - (float) memory_size_total / mib, kv_size, (int) layers.size(), n_seq_max, n_stream, - k_log.c_str(), v_log.c_str(), k_idx_log.c_str()); + LLAMA_LOG_INFO("%s: size = %7.2f MiB (%6u cells, %3d layers, %2u/%u seqs), K (%s): %7.2f MiB, V (%s): %7.2f MiB\n", __func__, + (float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f), kv_size, (int) layers.size(), n_seq_max, n_stream, + ggml_type_name(type_k), (float)memory_size_k / (1024.0f * 1024.0f), + ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f)); } // TODO: refactor [TAG_KV_CACHE_SHARE_CELLS] @@ -419,39 +392,6 @@ bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { p1 = std::numeric_limits::max(); } - // empty range - nothing to remove - if (p0 >= p1) { - return true; - } - - // MSA anchors block selection to absolute cache slots (slot == position). Tail trim and full removal preserve this invariant, but removing a prefix - // or middle range would free slots while later cells survive, desynchronizing the indexer cache. Reject such removals before modifying the cache. - if (msa_strict_slots) { - for (llama_seq_id sid = 0; sid < (llama_seq_id) seq_to_stream.size(); ++sid) { - if (seq_id >= 0 && sid != seq_id) { - continue; - } - - const auto & cells = v_cells[seq_to_stream[sid]]; - - const llama_pos pmin = cells.seq_pos_min(sid); - const llama_pos pmax = cells.seq_pos_max(sid); - - if (pmin < 0) { - continue; // empty sequence - } - - const bool overlaps = p0 <= pmax && p1 > pmin; // the range removes something - const bool leaves_tail = p1 <= pmax; // cells beyond the range survive - - if (overlaps && leaves_tail) { - LLAMA_LOG_WARN("%s: MSA: partial (non-suffix) removal [%d, %d) for seq %d is not supported " - "(block selection is anchored to cache slots) - rejected\n", __func__, p0, p1, sid); - return false; - } - } - } - if (seq_id >= 0) { auto & cells = v_cells[seq_to_stream[seq_id]]; auto & head = v_heads[seq_to_stream[seq_id]]; @@ -906,10 +846,6 @@ bool llama_kv_cache::update(llama_context * lctx, bool do_shift, const stream_co if (layer.v_stream[ssrc]) { ggml_backend_tensor_copy(layer.v_stream[ssrc], layer.v_stream[sdst]); } - if (layer.k_idx_stream[ssrc]) { - GGML_ASSERT(layer.k_idx_stream[sdst]); - ggml_backend_tensor_copy(layer.k_idx_stream[ssrc], layer.k_idx_stream[sdst]); - } } } } @@ -1058,44 +994,6 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch, const auto & cells = v_cells[seq_to_stream[seq_id]]; - if (n_tokens > cells.size()) { - LLAMA_LOG_ERROR("%s: n_tokens = %d > size = %u\n", __func__, n_tokens, cells.size()); - return { }; - } - - // MSA block selection assumes slot == logical position (append-only streams). - if (msa_strict_slots) { - for (uint32_t ii = 0; ii < n_tokens; ++ii) { - const llama_pos pos = ubatch.pos[s*n_tokens + ii]; - - if (pos < 0 || (uint64_t) pos >= cells.size()) { - LLAMA_LOG_WARN("%s: MSA: position %d is outside the cache range [0, %u)\n", - __func__, pos, cells.size()); - return { }; - } - - const uint32_t idx = (uint32_t) pos; - - if (!cells.is_empty(idx)) { - LLAMA_LOG_WARN("%s: MSA: required slot %u is already occupied (stream %u)\n", - __func__, idx, seq_to_stream[seq_id]); - return { }; - } - - // strictly increasing positions, rules out duplicates and, for contiguous requests, is tightened to exact adjacency - if (!res.idxs[s].empty() && (cont ? idx != res.idxs[s].back() + 1 - : idx <= res.idxs[s].back())) { - LLAMA_LOG_WARN("%s: MSA: token positions are not %s within the ubatch\n", - __func__, cont ? "contiguous" : "strictly increasing"); - return { }; - } - - res.idxs[s].push_back(idx); - } - - continue; - } - uint32_t head_cur = v_heads[seq_to_stream[seq_id]]; // if we have enough unused cells before the current head -> @@ -1104,6 +1002,11 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch, head_cur = 0; } + if (n_tokens > cells.size()) { + LLAMA_LOG_ERROR("%s: n_tokens = %d > size = %u\n", __func__, n_tokens, cells.size()); + return { }; + } + uint32_t n_tested = 0; // for continuous slots, we test that all tokens in the ubatch fit, starting from the current head @@ -1210,15 +1113,6 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & const auto idx = sinfo.idxs[s][ii]; - if (msa_strict_slots && (llama_pos) idx != ubatch.pos[i]) { - LLAMA_LOG_ERROR("%s: MSA slot/position invariant violated: " - "writing pos %d into cell %u (stream %u). The indexer cache " - "would desync and block selection would silently corrupt. " - "This is a bug, please report it with reproduction steps.\n", - __func__, ubatch.pos[i], idx, sinfo.strm[s]); - GGML_ABORT("MSA: slot != pos"); - } - if (!cells.is_empty(idx)) { assert(cells.seq_count(idx) == 1); @@ -1262,8 +1156,7 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & LLAMA_LOG_DEBUG("%s: purging positions [%d, %d] of sequence %d from KV cache\n", __func__, cells.seq_pos_min(s), seq_pos_max_rm[s], s); - // under MSA strict slots this path should be unreachable, since strict MSA placement never selects occupied cells - GGML_ASSERT(seq_rm(s, cells.seq_pos_min(s), seq_pos_max_rm[s] + 1)); + seq_rm(s, cells.seq_pos_min(s), seq_pos_max_rm[s] + 1); } } @@ -1283,12 +1176,6 @@ bool llama_kv_cache::get_can_shift() const { if (hparams.n_pos_per_embd() > 1) { return false; } - // shifting would leave k_idx stale - for (const auto & layer : layers) { - if (layer.k_idx) { - return false; - } - } return true; } @@ -1337,6 +1224,12 @@ ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { return layers[ikv].k; } +const llama_kv_cells & llama_kv_cache::get_cells(llama_seq_id seq_id) const { + GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()); + + return v_cells[seq_to_stream[seq_id]]; +} + uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const { uint32_t result = 0; @@ -1405,23 +1298,6 @@ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_k ggml_row_size(v->type, kv_size*n_embd_v_gqa)*sinfo.s0); } -ggml_tensor * llama_kv_cache::get_k_idx(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { - const int32_t ikv = map_layer_ids.at(il); - auto * k_idx = layers[ikv].k_idx; - GGML_ASSERT(k_idx); - - const uint64_t kv_size = get_size(); - const int64_t n_idx = k_idx->ne[0]; // 128 - const uint32_t ns = sinfo.s1 - sinfo.s0 + 1; - - return ggml_view_4d(ctx, k_idx, - n_idx, 1, n_kv, ns, - ggml_row_size(k_idx->type, n_idx), // nb1 (single head) - ggml_row_size(k_idx->type, n_idx), // nb2 (per cell) - ggml_row_size(k_idx->type, n_idx*kv_size), // nb3 (per stream) - ggml_row_size(k_idx->type, n_idx*kv_size)*sinfo.s0); -} - ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const { GGML_UNUSED(sinfo); @@ -1523,28 +1399,6 @@ ggml_tensor * llama_kv_cache::build_input_k_idxs(ggml_context * ctx, const llama return k_idxs; } -ggml_tensor * llama_kv_cache::cpy_k_idx(ggml_context * ctx, ggml_tensor * k_idx_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const { - GGML_UNUSED(sinfo); - const int32_t ikv = map_layer_ids.at(il); - ggml_tensor * k_idx = layers[ikv].k_idx; - GGML_ASSERT(k_idx && "cpy_k_idx on a layer with no indexer cache"); - - const int64_t n_embd_head = k_idx_cur->ne[0]; // 128 - const int64_t n_head = k_idx_cur->ne[1]; // 1 - const int64_t n_tokens = k_idx_cur->ne[2]; - const int64_t n_embd_gqa = n_embd_head*n_head; // 128 - - GGML_ASSERT(ggml_row_size(k_idx_cur->type, n_embd_head) == k_idx_cur->nb[1]); - k_idx_cur = ggml_view_2d(ctx, k_idx_cur, n_embd_gqa, n_tokens, k_idx_cur->nb[2], 0); - - const int64_t n_stream = k_idx->ne[2]; - if (n_stream > 1) { - const int64_t kv_size = get_size(); - k_idx = ggml_reshape_2d(ctx, k_idx, n_embd_gqa, kv_size*n_stream); - } - return ggml_set_rows(ctx, k_idx, k_idx_cur, k_idxs); // same k_idxs as the K store -} - ggml_tensor * llama_kv_cache::build_input_v_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const { const uint32_t n_tokens = ubatch.n_tokens; @@ -1979,18 +1833,6 @@ size_t llama_kv_cache::size_v_bytes() const { return size_v_bytes; } -size_t llama_kv_cache::size_k_idx_bytes() const { - size_t size_k_idx_bytes = 0; - - for (const auto & layer : layers) { - if (layer.k_idx) { - size_k_idx_bytes += ggml_nbytes(layer.k_idx); - } - } - - return size_k_idx_bytes; -} - ggml_tensor * llama_kv_cache::build_rope_shift( const llama_cparams & cparams, ggml_context * ctx, @@ -2303,36 +2145,6 @@ void llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t } } - if (size_k_idx_bytes() > 0) { - const uint32_t has_k_idx_u32 = 1; - io.write(&has_k_idx_u32, sizeof(has_k_idx_u32)); - - for (const auto & layer : layers) { - const uint32_t layer_has_k_idx = layer.k_idx ? 1 : 0; - io.write(&layer_has_k_idx, sizeof(layer_has_k_idx)); - - if (!layer_has_k_idx) { - continue; - } - - GGML_ASSERT(layer.k_idx_stream[cr.strm]); - - const int32_t k_idx_type_i = (int32_t) layer.k_idx->type; - io.write(&k_idx_type_i, sizeof(k_idx_type_i)); - - const uint64_t k_idx_size_row = ggml_row_size(layer.k_idx->type, layer.k_idx->ne[0]); - io.write(&k_idx_size_row, sizeof(k_idx_size_row)); - - for (const auto & range : cr.data) { - const size_t range_size = range.second - range.first; - const size_t buf_size = range_size * k_idx_size_row; - const size_t offset = range.first * k_idx_size_row; - - io.write_tensor(layer.k_idx_stream[cr.strm], offset, buf_size); - } - } - } - if (!v_trans) { for (const auto & layer : layers) { const uint32_t il = layer.il; @@ -2581,68 +2393,6 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32 } } - if (size_k_idx_bytes() > 0) { - uint32_t has_k_idx_u32 = 0; - io.read(&has_k_idx_u32, sizeof(has_k_idx_u32)); - - if (has_k_idx_u32 != 1) { - LLAMA_LOG_ERROR("%s: missing k_idx data in KV cache state\n", __func__); - return false; - } - - for (const auto & layer : layers) { - uint32_t layer_has_k_idx = 0; - io.read(&layer_has_k_idx, sizeof(layer_has_k_idx)); - - const uint32_t expected_layer_has_k_idx = layer.k_idx ? 1 : 0; - - if (layer_has_k_idx != expected_layer_has_k_idx) { - LLAMA_LOG_ERROR( - "%s: mismatched k_idx state for layer: got %u, expected %u\n", - __func__, layer_has_k_idx, expected_layer_has_k_idx); - return false; - } - - if (!layer_has_k_idx) { - continue; - } - - GGML_ASSERT(layer.k_idx_stream[strm]); - - int32_t k_idx_type_i = -1; - io.read(&k_idx_type_i, sizeof(k_idx_type_i)); - - if (k_idx_type_i != (int32_t) layer.k_idx->type) { - LLAMA_LOG_ERROR( - "%s: mismatched k_idx type: got %d, expected %d\n", - __func__, k_idx_type_i, (int32_t) layer.k_idx->type); - return false; - } - - uint64_t k_idx_size_row = 0; - io.read(&k_idx_size_row, sizeof(k_idx_size_row)); - - const uint64_t expected_k_idx_size_row = ggml_row_size(layer.k_idx->type, layer.k_idx->ne[0]); - - if (k_idx_size_row != expected_k_idx_size_row) { - LLAMA_LOG_ERROR( - "%s: mismatched k_idx row size: got %zu, expected %zu\n", - __func__, (size_t) k_idx_size_row, (size_t) expected_k_idx_size_row); - return false; - } - - if (cell_count) { - if (sinfo.is_contiguous()) { - io.read_tensor(layer.k_idx_stream[strm], sinfo.head() * k_idx_size_row, cell_count * k_idx_size_row); - } else { - for (uint32_t i = 0; i < cell_count; ++i) { - io.read_tensor(layer.k_idx_stream[strm], sinfo.idxs[0][i] * k_idx_size_row, k_idx_size_row); - } - } - } - } - } - if (!this->v_trans) { for (const auto & layer : layers) { const uint32_t il = layer.il; @@ -2844,10 +2594,6 @@ ggml_tensor * llama_kv_cache_context::get_v(ggml_context * ctx, int32_t il) cons return kv->get_v(ctx, il, n_kv, sinfos[i_cur]); } -ggml_tensor * llama_kv_cache_context::get_k_idx(ggml_context * ctx, int32_t il) const { - return kv->get_k_idx(ctx, il, n_kv, sinfos[i_cur]); -} - ggml_tensor * llama_kv_cache_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const { return kv->cpy_k(ctx, k_cur, k_idxs, il, sinfos[i_cur]); } @@ -2856,10 +2602,6 @@ ggml_tensor * llama_kv_cache_context::cpy_v(ggml_context * ctx, ggml_tensor * v_ return kv->cpy_v(ctx, v_cur, v_idxs, il, sinfos[i_cur]); } -ggml_tensor * llama_kv_cache_context::cpy_k_idx(ggml_context * ctx, ggml_tensor * k_idx_cur, ggml_tensor * k_idxs, int32_t il) const { - return kv->cpy_k_idx(ctx, k_idx_cur, k_idxs, il, sinfos[i_cur]); -} - ggml_tensor * llama_kv_cache_context::build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const { return kv->build_input_k_idxs(ctx, ubatch); } diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index d5a92f4405b5..6cb6dbd2f984 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -164,6 +164,8 @@ class llama_kv_cache : public llama_memory_i { std::vector get_layer_ids() const; ggml_tensor * get_k_storage(int32_t il) const; + const llama_kv_cells & get_cells(llama_seq_id seq_id) const; + // // graph_build API // @@ -173,12 +175,10 @@ class llama_kv_cache : public llama_memory_i { // get views of the current state of the cache ggml_tensor * get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ggml_tensor * get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; - ggml_tensor * get_k_idx(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; // store k_cur and v_cur in the cache based on the provided head location ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const; ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const; - ggml_tensor * cpy_k_idx(ggml_context * ctx, ggml_tensor * k_idx_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const; // // preparation API @@ -230,11 +230,9 @@ class llama_kv_cache : public llama_memory_i { ggml_tensor * k; ggml_tensor * v; - ggml_tensor * k_idx; // MSA single-head indexer keys, F32 std::vector k_stream; std::vector v_stream; - std::vector k_idx_stream; }; bool v_trans = true; // the value tensor is transposed @@ -263,9 +261,6 @@ class llama_kv_cache : public llama_memory_i { // env: LLAMA_KV_CACHE_DEBUG int debug = 0; - // set when a k_idx (indexer) cache exists and the stream layout supports MSA (single seq, or one stream per seq) - bool msa_strict_slots = false; - // this is the SWA type of the cache - not to be confused with the model SWA type const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE; @@ -298,7 +293,6 @@ class llama_kv_cache : public llama_memory_i { size_t size_k_bytes() const; size_t size_v_bytes() const; - size_t size_k_idx_bytes() const; ggml_tensor * build_rope_shift( const llama_cparams & cparams, @@ -378,7 +372,6 @@ class llama_kv_cache_context : public llama_memory_context_i { // get views of the current state of the cache ggml_tensor * get_k(ggml_context * ctx, int32_t il) const; ggml_tensor * get_v(ggml_context * ctx, int32_t il) const; - ggml_tensor * get_k_idx(ggml_context * ctx, int32_t il) const; // store k_cur and v_cur in the cache based on the provided head location // note: the heads in k_cur and v_cur should be laid out contiguously in memory @@ -388,7 +381,6 @@ class llama_kv_cache_context : public llama_memory_context_i { // - v_idxs [n_tokens] or [n_tokens*n_embd_v_gqa] depending if V cache is transposed ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const; ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il) const; - ggml_tensor * cpy_k_idx(ggml_context * ctx, ggml_tensor * k_idx_cur, ggml_tensor * k_idxs, int32_t il) const; // create destination indices for each head of the current batch for where it would be written in the KV cache // the indices address the global KV cache (not per stream) - this is not relevant for the user of this API, but diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 938d98798cdb..8fff1a432667 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -11,6 +11,7 @@ #include "llama-kv-cache.h" #include "llama-kv-cache-iswa.h" #include "llama-kv-cache-dsa.h" +#include "llama-kv-cache-msa.h" #include "llama-kv-cache-dsv4.h" #include "llama-memory-hybrid.h" #include "llama-memory-hybrid-iswa.h" @@ -2071,6 +2072,28 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, { res = nullptr; } break; + case LLM_ARCH_MINIMAX_M3: + { + // sparse (MSA) layers carry an indexer key cache, but leading dense layers do not + llama_kv_cache::layer_filter_cb filter_idx = + [&](int32_t il) { return (uint32_t) il >= hparams.n_layer_dense_lead; }; + + res = new llama_kv_cache_msa( + *this, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + 1, + hparams.n_swa, + hparams.swa_type, + nullptr, + filter_idx, + nullptr); + } break; case LLM_ARCH_GLM_DSA: case LLM_ARCH_DEEPSEEK32: { diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 0773ad5435c9..8bd6a4298e0d 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -1,5 +1,5 @@ #include "models.h" -#include "llama-kv-cache.h" +#include "llama-kv-cache-msa.h" #include #include #include @@ -7,7 +7,8 @@ // MiniMax-M3: MiniMax-M2 style GQA (per-head QK-norm, partial rotary) with // DeepSeek-V3 leading-dense + routed/shared experts (sigmoid gating, routed scaling), // swigluoai activation, and MiniMax Sparse Attention (MSA). MTP is not in released model weights. -// Notes: Blocks are anchored to absolute KV cache slots. +// MSA blocks are defined over token positions. The graph translates between position space (block +// selection) and cell space (K/V/indexer storage) via per-ubatch pos<->cell maps populated from llama_kv_cells void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -23,7 +24,6 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, hparams.indexer_block_size); ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks); msa_p = { (int) hparams.indexer_block_size, (int) hparams.indexer_top_k, (int) hparams.indexer_local_blocks }; - hparams.indexer_kv = true; switch (hparams.n_layer()) { case 60: type = LLM_TYPE_428B_A23B; break; @@ -86,43 +86,83 @@ std::unique_ptr llama_model_minimax_m3::build_arch_graph(cons return std::make_unique(*this, params); } -// per-query local-force bias for MSA selection -// local window always wins a slot -class llm_graph_input_msa_local : public llm_graph_input_i { +class llm_graph_input_msa : public llm_graph_input_i { public: - llm_graph_input_msa_local(int blk, int local, int64_t nblk) : blk(blk), local(local), nblk(nblk) {} + llm_graph_input_msa(const llama_kv_cache_msa_context * mctx, int blk, int local) : + mctx(mctx), blk(blk), local(local) {} void set_input(const llama_ubatch * ubatch) override { - if (!bias || !ubatch->pos) { - return; - } - const int64_t n_tokens = ubatch->n_tokens; - std::vector data((size_t) nblk * n_tokens, 0.0f); - for (int64_t i = 0; i < n_tokens; ++i) { - const int64_t L = ubatch->pos[i] / blk; - for (int l = 0; l < local && L - l >= 0; ++l) { - if (L - l < nblk) { - data[(size_t) i * nblk + (L - l)] = 1e30f; + if (pos_slot_i) { mctx->set_input_pos_slot(pos_slot_i, ubatch); } + if (pos_slot_f) { mctx->set_input_pos_slot(pos_slot_f, ubatch); } + if (cell_blk) { mctx->set_input_cell_pos(cell_blk, ubatch, blk); } + if (pos_mask) { mctx->set_input_pos_mask(pos_mask, ubatch); } + + // local-force bias over position blocks + if (bias && ubatch->pos) { + const int64_t n_tokens = ubatch->n_tokens; + const int64_t nblk = bias->ne[0]; + std::vector data((size_t) nblk * n_tokens, 0.0f); + for (int64_t i = 0; i < n_tokens; ++i) { + const int64_t L = ubatch->pos[i] / blk; + for (int l = 0; l < local && L - l >= 0; ++l) { + if (L - l < nblk) { + data[(size_t) i * nblk + (L - l)] = 1e30f; + } } } + ggml_backend_tensor_set(bias, data.data(), 0, data.size() * sizeof(float)); } - ggml_backend_tensor_set(bias, data.data(), 0, data.size() * sizeof(float)); } - // valid as long as the bias tensor dims still match the new ubatch/cache window + // valid as long as the tensor dims still match the new ubatch/cache window and the + // ubatch is in the same regime (decode graphs have pos_slot_f, batch graphs cell_blk) bool can_reuse(const llm_graph_params & params) override { - const auto * mctx = static_cast(params.mctx); + const auto * mctx_new = static_cast(params.mctx); + + this->mctx = mctx_new; + + const int64_t n_ps = GGML_PAD((int64_t) mctx_new->get_n_pos(), blk); + const int64_t ns = params.cparams.kv_unified ? 1 : params.ubatch.n_seqs_unq; + + const bool decode = params.ubatch.n_tokens == ns; // one token per stream bool res = true; - res &= bias->ne[1] == params.ubatch.n_tokens; - res &= bias->ne[0] * blk == (int64_t) mctx->get_n_kv(); + + res &= bias->ne[0] * blk == n_ps; + res &= bias->ne[1] == params.ubatch.n_tokens; + + res &= pos_mask->ne[0] == n_ps; + res &= pos_mask->ne[1] == params.ubatch.n_tokens; + + res &= pos_slot_i->ne[0] == n_ps; + res &= pos_slot_i->ne[1] == ns; + + res &= decode == (pos_slot_f != nullptr); + res &= decode == (cell_blk == nullptr); + + if (pos_slot_f) { + res &= pos_slot_f->ne[0] == n_ps; + res &= pos_slot_f->ne[1] == ns; + } + + if (cell_blk) { + res &= cell_blk->ne[0] == (int64_t) mctx_new->get_base()->get_n_kv(); + res &= cell_blk->ne[1] == ns; + } + return res; } - ggml_tensor * bias = nullptr; - int blk; - int local; - int64_t nblk; + ggml_tensor * bias = nullptr; // F32 [nblk, n_tokens] local-force bias (position blocks) + ggml_tensor * pos_mask = nullptr; // F32 [n_ps, n_tokens] 0/-inf visibility, by position + ggml_tensor * pos_slot_i = nullptr; // I32 [n_ps, ns] pos -> cell (get_rows index) + ggml_tensor * pos_slot_f = nullptr; // F32 [n_ps, ns] pos -> cell (gatherable values, decode) + ggml_tensor * cell_blk = nullptr; // I32 [n_kv, ns] cell -> position block (batch) + + const llama_kv_cache_msa_context * mctx; + + int blk; + int local; }; // One FA call for all GQA groups (and at multi-stream decode, all streams) by mapping them onto the FA sequence dim (ne[3]) @@ -173,7 +213,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ inpL = build_inp_embd(model.tok_embd); ggml_tensor * inp_pos = build_inp_pos(); - auto inp_attn = build_attn_inp_kv(); + auto inp_attn = build_attn_inp_kv_msa(); // MSA calls ggml_flash_attn_ext directly and assumes the non-transposed V layout that // llama.cpp only provides when flash attention is enabled. Block selection is anchored @@ -199,34 +239,51 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ } // hoisted per-graph MSA state (shared by every sparse layer) - llm_graph_input_msa_local * msa_loc = nullptr; + llm_graph_input_msa * msa = nullptr; ggml_tensor * msa_kqm = nullptr; - ggml_tensor * msa_mf = nullptr; - int64_t n_kv = 0, nblk = 0, ns = 1, n_tps = 0; + ggml_tensor * msa_mf = nullptr; // F32 copy of the FA mask for the final mask add + int64_t n_kv = 0, n_ps = 0, nblk = 0, ns = 1, n_tps = 0; bool msa_decode = false; // gather (1 token per stream) vs mask const int blk = mm.msa_p.blk; const int64_t Hd = hparams.indexer_n_head; // one indexer head per GQA group if (msa_enabled) { + const auto * mctx_msa = static_cast(mctx); + msa_kqm = inp_attn->get_kq_mask(); n_kv = msa_kqm->ne[0]; n_tps = msa_kqm->ne[1]; // tokens per stream ns = msa_kqm->ne[3]; // streams in this ubatch GGML_ASSERT(msa_kqm->type == GGML_TYPE_F16 && "MSA requires the FA (f16) mask"); GGML_ASSERT(n_tps*ns == n_tokens); - GGML_ASSERT(n_kv % blk == 0 && - "MSA: KV/mask n_kv must be a multiple of indexer.block_size (128); " - "the flash-attention KV padding must be a multiple of the block size. " - "A non-multiple would silently drop the partial tail block."); - nblk = n_kv / blk; + + // the position axis covers every position currently in the cache and is padded to whole blocks + n_ps = GGML_PAD((int64_t) mctx_msa->get_n_pos(), blk); + nblk = n_ps / blk; msa_decode = n_tps == 1; - msa_mf = ggml_cast(ctx0, msa_kqm, GGML_TYPE_F32); + auto inp = std::make_unique(mctx_msa, blk, mm.msa_p.local); + + inp->bias = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, nblk, n_tokens); // stream-grouped tokens + ggml_set_input(inp->bias); + + inp->pos_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_ps, n_tokens); + ggml_set_input(inp->pos_mask); + + inp->pos_slot_i = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_ps, ns); + ggml_set_input(inp->pos_slot_i); + + if (msa_decode) { + inp->pos_slot_f = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_ps, ns); + ggml_set_input(inp->pos_slot_f); + } else { + inp->cell_blk = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_kv, ns); + ggml_set_input(inp->cell_blk); - auto loc = std::make_unique(blk, mm.msa_p.local, nblk); - loc->bias = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, nblk, n_tokens); // stream-grouped tokens - ggml_set_input(loc->bias); - msa_loc = (llm_graph_input_msa_local *) res->add_input(std::move(loc)); + msa_mf = ggml_cast(ctx0, msa_kqm, GGML_TYPE_F32); + } + + msa = (llm_graph_input_msa *) res->add_input(std::move(inp)); } ggml_tensor * inp_out_ids = build_inp_out_ids(); @@ -283,9 +340,11 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ik = ggml_rope_ext(ctx0, ik, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - const auto * mctx_cur = inp_attn->mctx; - ggml_build_forward_expand(gf, mctx_cur->cpy_k_idx(ctx0, ik, inp_attn->get_k_idxs(), il)); - ggml_tensor * ik_kv = mctx_cur->get_k_idx(ctx0, il); + const auto * mctx_msa_l = static_cast(mctx); + const auto * mctx_cur = mctx_msa_l->get_base(); + const auto * mctx_idx = mctx_msa_l->get_idx(); + ggml_build_forward_expand(gf, mctx_idx->cpy_k(ctx0, ik, inp_attn->get_k_idxs_idx(), il)); + ggml_tensor * ik_kv = mctx_idx->get_k(ctx0, il); if (inp_attn->self_k_rot) { Qcur = llama_mul_mat_hadamard(ctx0, Qcur, inp_attn->self_k_rot); @@ -316,42 +375,52 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ if (msa_decode) { // decode: batched over streams top-k + gather, one grouped FA - // scores: per-stream batched matmul over the stream dim (ne[3]). - // the cache views are not contiguous across streams (stride = kv_size, not n_kv) - ggml_tensor * ikv4 = ggml_view_4d(ctx0, ik_kv, n_idx_dim, n_kv, 1, ns, - ik_kv->nb[2], ik_kv->nb[3], ik_kv->nb[3], 0); + // gather the indexer keys through the pos -> cell map + ggml_tensor * ik3 = ggml_view_3d(ctx0, ik_kv, n_idx_dim, n_kv, ns, + ik_kv->nb[2], ik_kv->nb[3], 0); + ggml_tensor * ikp = ggml_get_rows(ctx0, ik3, msa->pos_slot_i); // [n_idx_dim, n_ps, ns] ggml_tensor * iq4 = ggml_reshape_4d(ctx0, iq, n_idx_dim, Hd, 1, ns); - ggml_tensor * sc = ggml_mul_mat(ctx0, ikv4, iq4); + ggml_tensor * sc = ggml_mul_mat(ctx0, + ggml_reshape_4d(ctx0, ikp, n_idx_dim, n_ps, 1, ns), iq4); ggml_mul_mat_set_prec(sc, GGML_PREC_F32); - sc = ggml_add_inplace(ctx0, sc, msa_mf); + // unmapped positions come out -inf, so they can never rank into the top-k + sc = ggml_add_inplace(ctx0, sc, + ggml_reshape_4d(ctx0, msa->pos_mask, n_ps, 1, 1, ns)); ggml_tensor * bs = ggml_pool_2d(ctx0, sc, GGML_OP_POOL_MAX, blk, 1, blk, 1, 0, 0); cb(bs, "msa_bs", il); ggml_tensor * bsf = ggml_add(ctx0, bs, - ggml_reshape_4d(ctx0, msa_loc->bias, nblk, 1, 1, ns)); - ggml_tensor * idx = ggml_top_k(ctx0, bsf, K); + ggml_reshape_4d(ctx0, msa->bias, nblk, 1, 1, ns)); + ggml_tensor * idx = ggml_top_k(ctx0, bsf, K); // position blocks - // token idx: tj[t,k,h,s] = blk*idx[k,h,s] + t (for the mask gather) - // row idx: tr[t,k,h,s] = tj*HKV + h (for the per-stream K/V gather) + // pos idx: tj[t,k,h,s] = blk*idx[k,h,s] + t (positions - mask gather) + // cell idx: cs[t,k,h,s] = pos_slot[tj] (pos -> cell translation) + // row idx: tr[t,k,h,s] = cs*HKV + h (per-stream K/V gather) ggml_tensor * a = ggml_scale(ctx0, ggml_cast(ctx0, idx, GGML_TYPE_F32), (float) blk); a = ggml_reshape_4d(ctx0, a, 1, K, Hd, ns); ggml_tensor * tj = ggml_add(ctx0, ggml_repeat_4d(ctx0, a, blk, K, Hd, ns), ggml_reshape_3d(ctx0, ggml_arange(ctx0, 0.0f, (float) blk, 1.0f), blk, 1, 1)); + + ggml_tensor * tokj = ggml_cast(ctx0, ggml_reshape_2d(ctx0, tj, (int64_t) blk*K*Hd, ns), GGML_TYPE_I32); + + ggml_tensor * cs = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, msa->pos_slot_f, 1, n_ps, ns), tokj); // [1, blk*K*Hd, ns] + cs = ggml_reshape_4d(ctx0, cs, blk, K, Hd, ns); + ggml_tensor * tr = ggml_add(ctx0, - ggml_scale(ctx0, tj, (float) HKV), + ggml_scale(ctx0, cs, (float) HKV), ggml_reshape_3d(ctx0, ggml_arange(ctx0, 0.0f, (float) HKV, 1.0f), 1, 1, Hd)); - ggml_tensor * tokj = ggml_cast(ctx0, ggml_reshape_2d(ctx0, tj, (int64_t) blk*K*Hd, ns), GGML_TYPE_I32); ggml_tensor * tokr = ggml_cast(ctx0, ggml_reshape_2d(ctx0, tr, (int64_t) blk*K*Hd, ns), GGML_TYPE_I32); ggml_tensor * k3 = ggml_view_3d(ctx0, k, D, HKV*n_kv, ns, k->nb[1], k->nb[3], 0); ggml_tensor * v3 = ggml_view_3d(ctx0, v, D, HKV*n_kv, ns, v->nb[1], v->nb[3], 0); - ggml_tensor * m3 = ggml_reshape_3d(ctx0, msa_kqm, 1, n_kv, ns); + ggml_tensor * mp = ggml_reshape_3d(ctx0, msa->pos_mask, 1, n_ps, ns); ggml_tensor * kg = ggml_get_rows(ctx0, k3, tokr); ggml_tensor * vg = ggml_get_rows(ctx0, v3, tokr); - ggml_tensor * mg = ggml_get_rows(ctx0, m3, tokj); + ggml_tensor * mg = ggml_get_rows(ctx0, mp, tokj); // fold (group, stream) onto the FA channel dim const ggml_type kt = ggml_is_quantized(k->type) ? GGML_TYPE_F16 : k->type; @@ -372,12 +441,16 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ iq->nb[1], iq->nb[2], st*n_tps*iq->nb[2]); ggml_tensor * ik_s = ggml_view_2d(ctx0, ik_kv, n_idx_dim, n_kv, ik_kv->nb[2], st*ik_kv->nb[3]); - ggml_tensor * mf_s = ggml_view_3d(ctx0, msa_mf, n_kv, 1, n_tps, - msa_mf->nb[1], msa_mf->nb[1], st*msa_mf->nb[3]); - ggml_tensor * km_s = ggml_view_3d(ctx0, msa_kqm, n_kv, n_tps, 1, - msa_kqm->nb[1], msa_kqm->nb[3], st*msa_kqm->nb[3]); - ggml_tensor * bias_s = ggml_view_3d(ctx0, msa_loc->bias, nblk, 1, n_tps, - msa_loc->bias->nb[1], msa_loc->bias->nb[1], st*n_tps*msa_loc->bias->nb[1]); + ggml_tensor * psl_s = ggml_view_1d(ctx0, msa->pos_slot_i, n_ps, + st*msa->pos_slot_i->nb[1]); + ggml_tensor * pm_s = ggml_view_3d(ctx0, msa->pos_mask, n_ps, 1, n_tps, + msa->pos_mask->nb[1], msa->pos_mask->nb[1], st*n_tps*msa->pos_mask->nb[1]); + ggml_tensor * cb_s = ggml_view_1d(ctx0, msa->cell_blk, n_kv, + st*msa->cell_blk->nb[1]); + ggml_tensor * mf_s = ggml_view_3d(ctx0, msa_mf, n_kv, n_tps, 1, + msa_mf->nb[1], msa_mf->nb[3], st*msa_mf->nb[3]); + ggml_tensor * bias_s = ggml_view_3d(ctx0, msa->bias, nblk, 1, n_tps, + msa->bias->nb[1], msa->bias->nb[1], st*n_tps*msa->bias->nb[1]); ggml_tensor * q_s = ggml_view_3d(ctx0, Qcur, D, n_head, n_tps, Qcur->nb[1], Qcur->nb[2], st*n_tps*Qcur->nb[2]); ggml_tensor * k_s = ggml_view_4d(ctx0, k, D, HKV, n_kv, 1, @@ -385,14 +458,16 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * v_s = ggml_view_4d(ctx0, v, D, HKV, n_kv, 1, v->nb[1], v->nb[2], v->nb[3], st*v->nb[3]); - // block scores: bs = maxpool_blk(idx_q * idx_k^T + causal mask) + // block scores: the indexer keys are gathered through the pos -> cell map first // scores are unscaled, only the top-k ordering matters - ggml_tensor * sc = ggml_mul_mat(ctx0, ik_s, + ggml_tensor * ikp = ggml_get_rows(ctx0, ik_s, psl_s); // [n_idx_dim, n_ps] + ggml_tensor * sc = ggml_mul_mat(ctx0, ikp, ggml_reshape_2d(ctx0, iq_s, n_idx_dim, Hd*n_tps)); // indexer scores run in F32 ggml_mul_mat_set_prec(sc, GGML_PREC_F32); - sc = ggml_reshape_3d(ctx0, sc, n_kv, Hd, n_tps); - sc = ggml_add_inplace(ctx0, sc, mf_s); + sc = ggml_reshape_3d(ctx0, sc, n_ps, Hd, n_tps); + // unmapped positions (holes, padding, empty cells) come out -inf + sc = ggml_add_inplace(ctx0, sc, pm_s); ggml_tensor * bs = ggml_pool_2d(ctx0, sc, GGML_OP_POOL_MAX, blk, 1, blk, 1, 0, 0); cb(bs, "msa_bs", il); @@ -416,14 +491,16 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ bm = ggml_cont(ctx0, ggml_permute(ctx0, bm, 0, 2, 1, 3)); // [nblk, n_tps, Hd] cb(bm, "msa_block_mask", il); - // expand block -> token granularity (j = bk*blk + t), - // then combine with the causal mask in place - ggml_tensor * bmx = ggml_repeat_4d(ctx0, - ggml_reshape_3d(ctx0, bm, 1, nblk, n_tps*Hd), - blk, nblk, n_tps*Hd, 1); + // expand block -> cell granularity through the cell -> position block + // map, then combine with the causal mask. empty cells are masked by the causal mask. + ggml_tensor * bm2 = ggml_cont(ctx0, ggml_transpose(ctx0, + ggml_reshape_2d(ctx0, bm, nblk, n_tps*Hd))); // [n_tps*Hd, nblk] + ggml_tensor * bmc = ggml_get_rows(ctx0, bm2, cb_s); // [n_tps*Hd, n_kv] F32 + ggml_tensor * bmx = ggml_cont(ctx0, ggml_transpose(ctx0, bmc)); bmx = ggml_reshape_3d(ctx0, bmx, n_kv, n_tps, Hd); - ggml_tensor * mask4 = ggml_add_inplace(ctx0, bmx, km_s); - mask4 = ggml_reshape_4d(ctx0, mask4, n_kv, n_tps, 1, Hd); + ggml_tensor * mask4 = ggml_add_inplace(ctx0, bmx, mf_s); + mask4 = ggml_cast(ctx0, + ggml_reshape_4d(ctx0, mask4, n_kv, n_tps, 1, Hd), GGML_TYPE_F16); cb(mask4, "msa_mask4", il); // cache views with groups on ne[3]; From c50b34a1e0330c4781a0846932033689ee1eaa85 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 3 Aug 2026 17:32:01 +0300 Subject: [PATCH 41/98] graph : fix unused input tensors in minimax m3 graph (#26519) --- src/llama-graph.cpp | 16 +++++++++++----- src/llama-graph.h | 2 +- src/models/minimax-m3.cpp | 7 ++++++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index fdab7b8dde67..9dde345df406 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -530,7 +530,9 @@ llm_graph_input_attn_kv_msa::llm_graph_input_attn_kv_msa( void llm_graph_input_attn_kv_msa::set_input(const llama_ubatch * ubatch) { llm_graph_input_attn_kv::set_input(ubatch); - mctx_msa->get_idx()->set_input_k_idxs(self_k_idxs_idx, ubatch); + if (self_k_idxs_idx) { + mctx_msa->get_idx()->set_input_k_idxs(self_k_idxs_idx, ubatch); + } } bool llm_graph_input_attn_kv_msa::can_reuse(const llm_graph_params & params) { @@ -541,8 +543,10 @@ bool llm_graph_input_attn_kv_msa::can_reuse(const llm_graph_params & params) { bool res = true; - res &= self_k_idxs ->ne[0] == params.ubatch.n_tokens; - res &= self_k_idxs_idx->ne[0] == params.ubatch.n_tokens; + res &= self_k_idxs->ne[0] == params.ubatch.n_tokens; + if (self_k_idxs_idx) { + res &= self_k_idxs_idx->ne[0] == params.ubatch.n_tokens; + } res &= can_reuse_kq_mask(self_kq_mask, this->mctx, params.ubatch, params.cparams); @@ -3218,7 +3222,7 @@ llm_graph_input_attn_k_dsa * llm_graph_context::build_attn_inp_k_dsa() const { return (llm_graph_input_attn_k_dsa *) res->add_input(std::move(inp)); } -llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa() const { +llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa(bool msa_enabled) const { const auto * mctx_cur = static_cast(mctx); auto inp = std::make_unique(hparams, cparams, mctx_cur); @@ -3239,7 +3243,9 @@ llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa() const { inp->self_k_rot = mctx_base->build_input_k_rot(ctx0); inp->self_v_rot = mctx_base->build_input_v_rot(ctx0); - inp->self_k_idxs_idx = mctx_idx->build_input_k_idxs(ctx0, ubatch); + if (msa_enabled) { + inp->self_k_idxs_idx = mctx_idx->build_input_k_idxs(ctx0, ubatch); + } return (llm_graph_input_attn_kv_msa *) res->add_input(std::move(inp)); } diff --git a/src/llama-graph.h b/src/llama-graph.h index ff216302dbb1..32d8d395aa45 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -1190,7 +1190,7 @@ struct llm_graph_context { llm_graph_input_attn_k_dsa * build_attn_inp_k_dsa() const; - llm_graph_input_attn_kv_msa * build_attn_inp_kv_msa() const; + llm_graph_input_attn_kv_msa * build_attn_inp_kv_msa(bool msa_enabled) const; ggml_tensor * build_attn( llm_graph_input_attn_k_dsa * inp, diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 8bd6a4298e0d..854d5aed0f82 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -213,7 +213,9 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ inpL = build_inp_embd(model.tok_embd); ggml_tensor * inp_pos = build_inp_pos(); - auto inp_attn = build_attn_inp_kv_msa(); + + // ========================================== + // TODO: avoid such kind of complexity in the model graphs // MSA calls ggml_flash_attn_ext directly and assumes the non-transposed V layout that // llama.cpp only provides when flash attention is enabled. Block selection is anchored @@ -225,6 +227,8 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ const bool streams_ok = cparams.n_seq_max == 1 || !cparams.kv_unified; const bool msa_enabled = fa_on && streams_ok; + auto * inp_attn = build_attn_inp_kv_msa(msa_enabled); + static bool warned_no_fa = false; if (!fa_on && !warned_no_fa) { LLAMA_LOG_WARN("%s: flash attention disabled; MSA requires it -> running DENSE attention " @@ -237,6 +241,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ "-> running DENSE attention. Output may be degraded. Drop --kv-unified to enable MSA.\n", __func__); warned_unified = true; } + // ========================================== // hoisted per-graph MSA state (shared by every sparse layer) llm_graph_input_msa * msa = nullptr; From 39eab74a05d3e68ac822b6dc6cd78c90cb985c19 Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Mon, 3 Aug 2026 07:36:19 -0700 Subject: [PATCH 42/98] opencl: route large q6_K lm_head to the flat GEMV (#26427) * add a direct size condition for `large` weights; the original dimension condition is insufficient -- q6_K lm_head for gemma-4 E2B has [1536, 262144], which is big enough to slowdown gemv_noshuffle but does not satisfy the dimension condition (ne0 >= 2048) --- ggml/src/ggml-opencl/ggml-opencl.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 915c8e7b903f..fc0fce0d780a 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -7065,7 +7065,7 @@ static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) { return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1; } -static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_tensor *tensor) { +static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { // gemv_noshuffle variant perf drops for large M, use flat variant for large M. // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. // q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold. @@ -7083,7 +7083,15 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_tensor *tensor) { if ((tensor->ne[1] % 128 != 0) && tensor->ne[2] == 1 && tensor->ne[3] == 1) { return true; } - return tensor->ne[1] >= 32768 && tensor->ne[0] >= 2048 && tensor->ne[2] == 1 && tensor->ne[3] == 1; + + // The gemv_noshuffle slowdown tracks TOTAL weight size, not ne0 alone; ne0 >= 2048 is a + // proxy for "large weight" that misses a narrow-hidden vocab-scale lm_head. + // Add a direct size escape so such weights also take the flat path, without changing + // which weights ne0 >= 2048 already routes there. + // The size escape is not taken on the A7X since its compiler miscompiles the flat K-quant GEMV + return tensor->ne[1] >= 32768 + && (tensor->ne[0] >= 2048 || (backend_ctx->adreno_gen != ADRENO_GPU_GEN::A7X && ggml_nbytes(tensor) >= (256ull << 20))) + && tensor->ne[2] == 1 && tensor->ne[3] == 1; } static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { @@ -9403,7 +9411,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, cl_kernel kernel; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS kernel = backend_ctx->kernel_convert_block_q6_K; - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(tensor)) { + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { kernel = backend_ctx->kernel_convert_block_q6_K_noshuffle; } #else @@ -9436,7 +9444,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, tensor->extra = extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(tensor)) { + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { cl_int M = tensor->ne[1]; // ne01 cl_int K = tensor->ne[0]; // ne00 @@ -10473,7 +10481,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } - if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(tensor)) { + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { static ggml_cl_buffer buf_trans_ql; static ggml_cl_buffer buf_trans_qh; static ggml_cl_buffer buf_trans_s; @@ -18895,7 +18903,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co } // q6_K x fp32 - if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q6_K(src0)) { + if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q6_K(backend_ctx, src0)) { ggml_cl_mul_mat_q6_K_f32_adreno(backend, src0, src1, dst); return; } From dbadb68eecdfb3ab0e86872d011738fc937f0364 Mon Sep 17 00:00:00 2001 From: AgoraPete Date: Mon, 3 Aug 2026 17:03:14 +0200 Subject: [PATCH 43/98] ggml: use dynamic allocation for split graph inputs (#22789) * ggml: use dynamic allocation for split graph inputs Replace fixed-size GGML_SCHED_MAX_SPLIT_INPUTS arrays with dynamically allocated buffers in the backend scheduler. This fixes crashes when loading wide MoE models (Gemma 4, Qwen MoE, Mixtral, DeepSeek) on multi-backend setups where graph splits exceed 30 input tensors. - split->inputs: dynamic array with grow-on-demand - sched->graph_inputs: dynamic array with grow-on-demand - graph_size calculation now uses actual input count instead of fixed constant * cont : clean-up --------- Co-authored-by: Georgi Gerganov --- ggml/src/ggml-backend.cpp | 63 +++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 7f4e252dca39..f6fb91798ca4 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -765,8 +765,9 @@ struct ggml_backend_sched_split { int backend_id; int i_start; int i_end; - struct ggml_tensor * inputs[GGML_SCHED_MAX_SPLIT_INPUTS]; + struct ggml_tensor ** inputs; int n_inputs; + int inputs_capacity; // graph view of this split struct ggml_cgraph graph; }; @@ -805,8 +806,9 @@ struct ggml_backend_sched { int cur_copy; int next_copy; ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES]; - struct ggml_tensor * graph_inputs[GGML_SCHED_MAX_SPLIT_INPUTS]; + struct ggml_tensor ** graph_inputs; int n_graph_inputs; + int graph_inputs_capacity; struct ggml_context * ctx; @@ -832,6 +834,36 @@ struct ggml_backend_sched { #define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)] #define tensor_copy(tensor, backend_id, copy_id) tensor_id_copy(hash_id(tensor), backend_id, copy_id) +static void ggml_backend_sched_split_inputs_grow(struct ggml_backend_sched_split * split) { + int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS; + if (split->inputs_capacity > 0) { + new_cap = 2*split->inputs_capacity; + GGML_LOG_WARN("%s: increasing split inputs capacity from %d to %d\n", __func__, split->inputs_capacity, new_cap); + } + auto * pnew = (struct ggml_tensor **) realloc((void *) split->inputs, new_cap * sizeof(struct ggml_tensor *)); + if (pnew == NULL) { + GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *)); + GGML_ABORT("failed to grow split inputs container"); + } + split->inputs = pnew; + split->inputs_capacity = new_cap; +} + +static void ggml_backend_sched_graph_inputs_grow(ggml_backend_sched_t sched) { + int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS; + if (sched->graph_inputs_capacity > 0) { + new_cap = 2*sched->graph_inputs_capacity; + GGML_LOG_WARN("%s: increasing graph inputs capacity from %d to %d\n", __func__, sched->graph_inputs_capacity, new_cap); + } + auto * pnew = (struct ggml_tensor **) realloc((void *) sched->graph_inputs, new_cap * sizeof(struct ggml_tensor *)); + if (pnew == NULL) { + GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *)); + GGML_ABORT("failed to grow graph inputs container"); + } + sched->graph_inputs = pnew; + sched->graph_inputs_capacity = new_cap; +} + // returns the priority of the backend, lower id is higher priority static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) { for (int i = 0; i < sched->n_backends; i++) { @@ -1297,7 +1329,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra } // check if the split has too many inputs // FIXME: count the number of inputs instead of only checking when full - if (split->n_inputs == GGML_SCHED_MAX_SPLIT_INPUTS) { + if (split->n_inputs >= split->inputs_capacity) { const size_t id = hash_id(src); int src_backend_id = sched->hv_tensor_backend_ids[id]; bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id); @@ -1313,10 +1345,14 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra split->i_end = i; i_split++; if (i_split >= sched->splits_capacity) { + int old_cap = sched->splits_capacity; sched->splits_capacity *= 2; sched->splits = (ggml_backend_sched_split *) realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split)); GGML_ASSERT(sched->splits != NULL); + for (int k = old_cap; k < sched->splits_capacity; k++) { + memset(&sched->splits[k], 0, sizeof(struct ggml_backend_sched_split)); + } } split = &sched->splits[i_split]; split->backend_id = node_backend_id; @@ -1353,7 +1389,9 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra SET_CAUSE(tensor_copy, "4.cpy"); } int n_graph_inputs = sched->n_graph_inputs++; - GGML_ASSERT(n_graph_inputs < GGML_SCHED_MAX_SPLIT_INPUTS); + if (n_graph_inputs >= sched->graph_inputs_capacity) { + ggml_backend_sched_graph_inputs_grow(sched); + } sched->graph_inputs[n_graph_inputs] = src; } } @@ -1373,7 +1411,9 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra SET_CAUSE(tensor_copy, "4.cpy"); } int n_inputs = split->n_inputs++; - GGML_ASSERT(n_inputs < GGML_SCHED_MAX_SPLIT_INPUTS); + if (n_inputs >= split->inputs_capacity) { + ggml_backend_sched_split_inputs_grow(split); + } split->inputs[n_inputs] = src; } node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy); @@ -1399,7 +1439,11 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra sched->prev_leaf_backend_ids = tmp; } - int graph_size = std::max(graph->n_nodes, graph->n_leafs) + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sched->n_copies; + int total_inputs = sched->n_graph_inputs; + for (int i = 0; i < sched->n_splits; i++) { + total_inputs += sched->splits[i].n_inputs; + } + int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies; // remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC] sched->debug_prev_graph_size = sched->debug_graph_size; @@ -1782,6 +1826,9 @@ ggml_backend_sched_t ggml_backend_sched_new( sched->splits = (ggml_backend_sched_split *) calloc(initial_splits_capacity, sizeof(sched->splits[0])); sched->splits_capacity = initial_splits_capacity; + sched->graph_inputs_capacity = GGML_SCHED_MAX_SPLIT_INPUTS; + sched->graph_inputs = (struct ggml_tensor **) calloc(sched->graph_inputs_capacity, sizeof(struct ggml_tensor *)); + for (int b = 0; b < n_backends; b++) { sched->backends[b] = backends[b]; sched->bufts[b] = bufts ? bufts[b] : ggml_backend_get_default_buffer_type(backends[b]); @@ -1814,7 +1861,11 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { ggml_gallocr_free(sched->galloc); ggml_free(sched->ctx); ggml_hash_set_free(&sched->hash_set); + for (int i = 0; i < sched->splits_capacity; i++) { + free(sched->splits[i].inputs); + } free(sched->splits); + free(sched->graph_inputs); free(sched->hv_tensor_backend_ids); free(sched->hv_tensor_copies); free(sched->node_backend_ids); From e8e06f78e253a98a739b8ae4c6b661b357249ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Mon, 3 Aug 2026 17:40:53 +0200 Subject: [PATCH 44/98] vocab : validate default special token ids (#26506) --- src/llama-vocab.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 443cd4640858..10032a8c6442 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2532,6 +2532,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { const std::string & key = kv(std::get<0>(it)); int32_t & id = std::get<1>(it); + if (id >= 0 && static_cast(id) >= id_to_token.size()) { + LLAMA_LOG_WARN("%s: default special token '%s' = %d out of vocab range, disabling\n", + __func__, key.c_str(), id); + id = LLAMA_TOKEN_NULL; + } + uint32_t new_id; if (!ml.get_key(std::get<0>(it), new_id, false)) { continue; From 99111b19ce482f081e92ec6c6cdbe6a4c815c515 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Mon, 3 Aug 2026 18:51:02 +0200 Subject: [PATCH 45/98] server: add get_info tool (#26522) * server: add get_info tool * fix --rpc in docs * server: harden get_info probe result handling Report the OS as unknown when the probe process fails to spawn or times out, so the diagnostic text from run() is never returned as an OS name. Strip the probe output on both ends, which also drops the blank line that ver prints before the version on Windows. Name the output and timeout limits, and report an unreadable working directory as unknown instead of an empty string. * server: simplify get_info result handling Drop the named limits and the working directory error branch, keeping the probe result handling to a single expression. --------- Co-authored-by: Pascal --- common/arg.cpp | 4 +-- common/common.h | 2 ++ examples/gen-docs/gen-docs.cpp | 2 ++ tools/server/README.md | 4 ++- tools/server/server-tools.cpp | 51 ++++++++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 3 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 305938fcb2b4..b75f4f05f039 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2582,7 +2582,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.mtmd_batch_max_tokens = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS")); - if (llama_supports_rpc()) { + if (params.is_gen_docs || llama_supports_rpc()) { add_opt(common_arg( {"--rpc"}, "SERVERS", "comma-separated list of RPC servers (host:port)", @@ -3331,7 +3331,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--tools"}, "TOOL1,TOOL2,...", "experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n" "specify \"all\" to enable all tools\n" - "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n" + "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n" "note: for security reasons, this will limit --cors-origins to localhost by default", [](common_params & params, const std::string & value) { params.server_tools = parse_csv_row(value); diff --git a/common/common.h b/common/common.h index 919c0ea103a4..25dac86d8a7d 100644 --- a/common/common.h +++ b/common/common.h @@ -739,6 +739,8 @@ struct common_params { llama_progress_callback load_progress_callback = NULL; void * load_progress_callback_user_data = NULL; bool no_alloc = false; // Don't allocate model buffers + + bool is_gen_docs = false; // whether we are running inside llama-gen-docs }; // call once at the start of a program if it uses libcommon diff --git a/examples/gen-docs/gen-docs.cpp b/examples/gen-docs/gen-docs.cpp index baf61bf27b54..114416719c7f 100644 --- a/examples/gen-docs/gen-docs.cpp +++ b/examples/gen-docs/gen-docs.cpp @@ -70,6 +70,8 @@ static void write_table(std::ostringstream & ss, std::vector & opt static void write_help(std::ostringstream & ss, const md_file & md) { common_params params; + params.is_gen_docs = true; + auto ctx_arg = common_params_parser_init(params, md.ex); std::vector common_options; diff --git a/tools/server/README.md b/tools/server/README.md index f45c018972d2..a0956f9e65af 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -198,7 +198,9 @@ For the full list of features, please refer to [server's changelog](https://gith | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG) | | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG_FILE) | | `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_UI_MCP_PROXY) | -| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | +| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | +| `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_CONFIG) | +| `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_JSON) | | `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_AGENT) | | `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)
(env: LLAMA_ARG_UI) | | `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)
(env: LLAMA_ARG_EMBEDDINGS) | diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index 4a6c5ed4420a..984bb478ea23 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -1090,6 +1090,56 @@ struct server_tool_get_datetime : server_tool { } }; +// +// get_info: returns runtime info (OS name/version and cwd) +// + +struct server_tool_get_info : server_tool { + server_tool_get_info() { + name = "get_info"; + display_name = "Get Runtime Info"; + permission_write = false; + } + + json get_definition() const override { + return { + {"type", "function"}, + {"function", { + {"name", name}, + {"description", "Returns runtime info: the OS name/version and the current working directory"}, + {"parameters", { + {"type", "object"}, + {"properties", json::object()}, + }}, + }}, + }; + } + + json invoke(json params, server_tool::stream *) const override { + auto io = make_tools_io(params); + +#ifdef _WIN32 + auto res = io->run({"cmd", "/c", "ver"}, 4096, 5); +#else + auto res = io->run({"uname", "-a"}, 4096, 5); +#endif + // "ver" prints a blank line before the version, so the output is stripped on both ends; + // a failed spawn or a timeout leaves a diagnostic in res.output, which is not an OS name + std::string os_info = res.exit_code == 0 && !res.timed_out ? string_strip(res.output) : "unknown"; + + std::string cwd = json_value(params, "cwd", std::string()); + if (cwd.empty()) { + std::error_code ec; + cwd = fs::current_path(ec).string(); + } + + return { + {"os", os_info}, + {"cwd", cwd}, + }; + } +}; + struct server_tool_stream_result : server_task_result { std::string chunk; bool done = false; @@ -1199,6 +1249,7 @@ static std::vector> build_tools() { tools.push_back(std::make_unique()); tools.push_back(std::make_unique()); tools.push_back(std::make_unique()); + tools.push_back(std::make_unique()); return tools; } From ee0445c99cffbe8d920b05cad28cb055d7049c0a Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 3 Aug 2026 18:58:15 +0200 Subject: [PATCH 46/98] tests: add model resolution test on synthetic repo listings (#26172) * tests: add model resolution test on synthetic repo listings Include download.cpp and arg.cpp inside a namespace with hf_cache monkey patched to serve hardcoded listings, so the resolution and the model handler assembly are tested end-to-end through the real CLI parsing, without modifying the tested code and without network access. Covers the primary, shard, mmproj, sidecar and preset resolution on layouts mimicking real vendor conventions, replays every case on permutations of the listing to assert determinism, and asserts the final wired paths for the spec type auto-selection, the -md precedence and the fallback suppression. * tests: keep model resolution checks active and let the handler resolve Replace assert with a REQUIRE macro alive in Release builds, key the fake hf_cache by repo id so the real handler init resolves every plan itself, check the exact shard sets, restrict the permutation exception to the order dependent picks, and cover dflash and eagle3. * tests: fix model resolution build on fatal warnings CI and Windows The namespaced copy of the sources leaves many static functions unused in this TU, exempt it from the unused warnings. Pre-include the windows headers so arg.cpp does not pull them inside the namespace. Declare the renamed copies of the download.h functions, verbatim from the header and renamed in sync by the macros, so missing declaration and missing prototype warnings are satisfied on every toolchain. * tests: fix winsock inclusion order for the model resolution test WIN32_LEAN_AND_MEAN and winsock2.h before windows.h, so http.h does not redefine the socket types afterwards. * tests: link cpp-httplib to the model resolution test The test compiles its own copy of download.cpp, which calls httplib directly, and the private link of llama-common does not propagate the symbols under lld-link. * common_http_client * common: finish the http client wrapper Add the virtual Head, Get and Post methods and the passthrough setters to the common_http_client skeleton, move follow_location into the constructor, expose the underlying client for the ranged pull path, and rename the missed common_http_client_init call sites. * tests: rewrite model resolution on the http client stub Replace the namespace inclusion of the sources by a plain TU: the common_http_client factory returns a stub serving hardcoded HF API responses, so the real hf_cache parsing, resolution and CLI handler run against synthetic listings in an isolated cache directory. Failures print the named case, the reordering and the actual versus expected values, the assembly cases use the full command line as context, and the empty result cases are checked once to keep the logs short. * tests: fix the model resolution on Windows and the builds without TLS Assert the exact expected paths composed like the cache does instead of suffix matching on forward slashes, set the environment portably, and serve the stub through an http endpoint so the builds rejecting the https scheme still reach it. Pause the log so the negative cases can be replayed on every reordering. * tests: make the model resolution failures self explanatory Resume the paused log before the failure report so the CI shows why the tested code bailed, and format the stub oids portably. * common: hold the http client factory behind exported functions The factory was an inline variable, and the Windows shared builds export functions but not data, so the executable and the DLL each had their own instance: the stub installed by the test was invisible to the library, which reached for the real endpoint and resolved nothing. Route the creation through functions compiled into the library and format the stub oids portably. * common: add the http client factory source missed in the previous commit * common: typedef the http client factory callback Address review from @ngxson * tests: serve the model resolution repos over the loopback Replace the client stub by a real httplib server bound to the loopback, so no C++ object crosses the module boundary anymore and the library exercises its own client and transport end to end. The debug shared build on Windows crashed inside the stubbed path. * common: add portable common_get_env and common_set_env helpers Address review from @ngxson * common: drop the http client factory left without a caller The loopback server made the stub substitution unnecessary, the client init builds the real client directly again. * common: read the model endpoint through the env helpers * nit: drop the stub leftovers from the model resolution test * common: align common_set_env and isolate the test cache per run The POSIX branch now behaves like _putenv_s, so the helper has a single contract on every platform, and common_get_env already reads an unset and an empty variable alike. The model resolution test keys its cache directory on the loopback port, where two concurrent runs on the same machine used to share one directory and the initial cleanup of either wiped the other. * tests: move the model resolution server into main * tests: support the DSpark sidecar resolution * common: revert the http client to the plain httplib client address review from @ngxson --------- Co-authored-by: Xuan Son Nguyen --- common/common.cpp | 41 ++- common/common.h | 9 + tests/CMakeLists.txt | 3 + tests/test-model-resolution.cpp | 506 ++++++++++++++++++++++++++++++++ 4 files changed, 547 insertions(+), 12 deletions(-) create mode 100644 tests/test-model-resolution.cpp diff --git a/common/common.cpp b/common/common.cpp index c941fd505ab3..d9ce5755161e 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -998,6 +998,23 @@ bool fs_is_directory(const std::string & path) { return std::filesystem::exists(dir) && std::filesystem::is_directory(dir); } +std::string common_get_env(const std::string & name) { + const char * value = std::getenv(name.c_str()); + return value == nullptr ? "" : value; +} + +void common_set_env(const std::string & name, const std::string & value) { +#if defined(_WIN32) + _putenv_s(name.c_str(), value.c_str()); +#else + if (value.empty()) { + unsetenv(name.c_str()); + } else { + setenv(name.c_str(), value.c_str(), 1); + } +#endif +} + std::string fs_get_cache_directory() { std::string cache_directory = ""; auto ensure_trailing_slash = [](std::string p) { @@ -1463,18 +1480,18 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode common_init_result::~common_init_result() = default; std::string common_get_model_endpoint() { - const char * model_endpoint_env = getenv("MODEL_ENDPOINT"); - // We still respect the use of environment-variable "HF_ENDPOINT" for backward-compatibility. - const char * hf_endpoint_env = getenv("HF_ENDPOINT"); - const char * endpoint_env = model_endpoint_env ? model_endpoint_env : hf_endpoint_env; - std::string model_endpoint = "https://huggingface.co/"; - if (endpoint_env) { - model_endpoint = endpoint_env; - if (model_endpoint.back() != '/') { - model_endpoint += '/'; - } - } - return model_endpoint; + std::string endpoint = common_get_env("MODEL_ENDPOINT"); + if (endpoint.empty()) { + // the HF_ENDPOINT variable is respected for backward compatibility + endpoint = common_get_env("HF_ENDPOINT"); + } + if (endpoint.empty()) { + return "https://huggingface.co/"; + } + if (endpoint.back() != '/') { + endpoint += '/'; + } + return endpoint; } char * common_get_model_or_exit(int argc, char * argv[]) { diff --git a/common/common.h b/common/common.h index 25dac86d8a7d..78d0877566b6 100644 --- a/common/common.h +++ b/common/common.h @@ -865,6 +865,15 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat bool glob_match(const std::string & pattern, const std::string & str); +// +// Environment utils +// + +// portable environment access, an unset variable reads as an empty string +// and setting an empty value unsets the variable +std::string common_get_env(const std::string & name); +void common_set_env(const std::string & name, const std::string & value); + // // Filesystem utils // diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 881e55c75a1d..419e1eba4c2c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -258,6 +258,9 @@ llama_build_and_test(test-thread-safety.cpp ARGS -m "${MODEL_DEST}" -ngl 99 -p " set_tests_properties(test-thread-safety PROPERTIES FIXTURES_REQUIRED test-download-model) llama_build_and_test(test-arg-parser.cpp) +llama_build_and_test(test-model-resolution.cpp) +# the test serves its repos from an httplib server, and the library links it privately +target_link_libraries(test-model-resolution PRIVATE cpp-httplib) if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC) # TODO: repair known memory leaks diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp new file mode 100644 index 000000000000..a96e40bb2bf8 --- /dev/null +++ b/tests/test-model-resolution.cpp @@ -0,0 +1,506 @@ +// tests the HF model resolution and the model handler assembly end-to-end on +// synthetic repo listings: a local httplib server bound to the loopback +// serves hardcoded HF API responses, so the real client, hf_cache, resolution +// and CLI parsing run against them without external network access + +#include "arg.h" +#include "common.h" +#include "download.h" +#include "http.h" +#include "log.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// the case and reordering being checked, printed with every failure +static std::string g_context; + +// independent of NDEBUG, so the checks stay alive in Release builds +#define REQUIRE(x) do { \ + if (!(x)) { \ + fprintf(stderr, "%s:%d: [%s] REQUIRE(%s) failed\n", \ + __FILE__, __LINE__, g_context.c_str(), #x); \ + std::abort(); \ + } \ +} while (0) + +#define REQUIRE_EQ(actual, expected) do { \ + if (!((actual) == (expected))) { \ + fprintf(stderr, "%s:%d: [%s] REQUIRE_EQ(%s, %s) failed\n actual: '%s'\n expected: '%s'\n", \ + __FILE__, __LINE__, g_context.c_str(), #actual, #expected, \ + std::string(actual).c_str(), std::string(expected).c_str()); \ + std::abort(); \ + } \ +} while (0) + +// +// synthetic repos keyed by repo id, served over the loopback by a real +// httplib server, so the tested code runs its own client and transport +// + +static std::map> g_repos; + +static const char * COMMIT = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + +// the server lives in main, so its destructor runs before the static teardown +// tears down the winsock state httplib brings in +static void serve_repos(httplib::Server & server) { + server.Get(R"(/api/models/(.+)/refs)", [](const httplib::Request & req, httplib::Response & res) { + if (g_repos.count(req.matches[1])) { + res.set_content(nlohmann::json{{"branches", {{{"name", "main"}, {"targetCommit", COMMIT}}}}}.dump(), + "application/json"); + } else { + res.status = 404; + } + }); + server.Get(R"(/api/models/(.+)/tree/.+)", [](const httplib::Request & req, httplib::Response & res) { + if (!g_repos.count(req.matches[1])) { + res.status = 404; + return; + } + auto files = nlohmann::json::array(); + size_t i = 0; + for (const auto & p : g_repos[req.matches[1]]) { + char oid[41]; + snprintf(oid, sizeof(oid), "%040lx", (unsigned long) ++i); + files.push_back({{"type", "file"}, {"path", p}, {"size", 1}, {"oid", oid}}); + } + res.set_content(files.dump(), "application/json"); + }); +} + +static common_params_model model_ref(const std::string & hf_repo, const std::string & hf_file = "") { + common_params_model m; + m.hf_repo = hf_repo; + m.hf_file = hf_file; + return m; +} + +// the model cache is isolated under a temporary directory named after the +// loopback port, so concurrent runs on a shared machine keep their own, and +// the local path the handler wires for a file is snapshots// +static std::filesystem::path cache_dir; + +static std::string cached(std::string repo_id, const std::string & path) { + string_replace_all(repo_id, "/", "--"); + return (cache_dir / ("models--" + repo_id) / "snapshots" / COMMIT / path).string(); +} + +// +// fixtures mimicking real repo layouts +// + +// flat layout in the style of ggml-org/gemma-4-31B-it-GGUF +static const std::vector flat = { + "README.md", + "model-BF16.gguf", + "model-Q4_K_M.gguf", + "model-Q8_0.gguf", + "mmproj-model-BF16.gguf", + "mmproj-model-Q8_0.gguf", + "mtp-model-BF16.gguf", + "mtp-model-Q4_0.gguf", + "mtp-model-Q8_0.gguf", + "dflash-model-BF16.gguf", + "dflash-model-Q8_0.gguf", +}; + +// quants in subdirectories with sharded files and root sidecars, +// in the style of stepfun-ai/Step-3.7-Flash-GGUF +static const std::vector subdir = { + "mmproj-model-f16.gguf", + "model-mtp-BF16.gguf", + "model-mtp-Q8_0.gguf", + "Q3_K_M/model-Q3_K_M-00001-of-00003.gguf", + "Q3_K_M/model-Q3_K_M-00002-of-00003.gguf", + "Q3_K_M/model-Q3_K_M-00003-of-00003.gguf", + "Q8_0/model-Q8_0-00001-of-00002.gguf", + "Q8_0/model-Q8_0-00002-of-00002.gguf", +}; + +// sidecar quants exist where the full model quant does not, +// in the style of ggml-org/Qwen3.6-27B-GGUF +static const std::vector hole = { + "model-BF16.gguf", + "model-Q4_K_M.gguf", + "model-Q8_0.gguf", + "mtp-model-BF16.gguf", + "mtp-model-Q4_0.gguf", + "mtp-model-Q8_0.gguf", + "dflash-model-BF16.gguf", + "dflash-model-Q8_0.gguf", +}; + +// unsloth-style naming with UD quants and a suffix MTP file +static const std::vector unsloth = { + "model-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "model-MTP-BF16.gguf", +}; + +// bartowski-style vendor prefix and mradermacher-style dot quant +static const std::vector vendors = { + "TheDrummer_Model-24B-v4.1-Q8_0.gguf", + "BlackSheep-24B.Q8_0.gguf", +}; + +// every speculative sidecar type at the same quant +static const std::vector quad = { + "model-Q8_0.gguf", + "mtp-model-Q8_0.gguf", + "dflash-model-Q8_0.gguf", + "eagle3-model-Q8_0.gguf", + "dspark-model-Q8_0.gguf", +}; + +static const std::vector dflash_only = { + "model-Q8_0.gguf", + "dflash-model-Q8_0.gguf", +}; + +static const std::vector eagle3_only = { + "model-Q8_0.gguf", + "eagle3-model-Q8_0.gguf", +}; + +// a single full quant with dspark sidecars at other quants, +// in the style of ggml-org/DeepSeek-V4-Flash-0731-GGUF +static const std::vector spark = { + "README.md", + "model-MXFP4.gguf", + "dspark-model-BF16.gguf", + "dspark-model-MXFP4.gguf", +}; + +// dspark outranks dflash in the type auto-selection +static const std::vector dspark_dflash = { + "model-Q8_0.gguf", + "dflash-model-Q8_0.gguf", + "dspark-model-Q8_0.gguf", +}; + +// +// table-driven plan resolution through the real entry point, +// each case replayed on multiple deterministic reorderings of the listing, +// except the cases whose pick legitimately depends on the listing order +// + +struct plan_case { + const char * name; + const std::vector & files; + const char * hf_repo; + const char * hf_file; + bool sidecars; // request mmproj + mtp + dflash + eagle3 + dspark + bool order_dependent; // the expected pick depends on the listing order + const char * primary; + std::vector model_files; + const char * mmproj; + const char * mtp; + const char * dflash; + const char * eagle3; + const char * dspark; +}; + +static const plan_case plan_cases[] = { + // exact tag picks the matching primary, sidecars follow the tag + {"flat exact tag", flat, "test/repo:Q8_0", "", true, false, + "model-Q8_0.gguf", {"model-Q8_0.gguf"}, + "mmproj-model-Q8_0.gguf", "mtp-model-Q8_0.gguf", "dflash-model-Q8_0.gguf", "", ""}, + + // no tag falls back to the default quant preference + {"flat default", flat, "test/repo", "", false, false, + "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"}, + "", "", "", "", ""}, + + // no tag and no default match falls back to the first model in the listing + {"unsloth fallback", unsloth, "test/repo", "", true, true, + "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"}, + "mmproj-BF16.gguf", "", "", "", ""}, + + // explicit hf_file picks that exact file + {"flat hf_file", flat, "test/repo", "model-BF16.gguf", false, false, + "model-BF16.gguf", {"model-BF16.gguf"}, + "", "", "", "", ""}, + + // missing hf_file resolves nothing + {"flat missing hf_file", flat, "test/repo", "nope.gguf", false, false, + "", {}, + "", "", "", "", ""}, + + // a sharded primary brings all its parts, a subdir primary finds the root sidecar + {"subdir shards", subdir, "test/repo:Q3_K_M", "", true, false, + "Q3_K_M/model-Q3_K_M-00001-of-00003.gguf", + {"Q3_K_M/model-Q3_K_M-00001-of-00003.gguf", + "Q3_K_M/model-Q3_K_M-00002-of-00003.gguf", + "Q3_K_M/model-Q3_K_M-00003-of-00003.gguf"}, + "mmproj-model-f16.gguf", "model-mtp-Q8_0.gguf", "", "", ""}, + + // a tag with no matching full model still resolves the requested sidecars + {"hole tag sidecar", hole, "test/repo:Q4_0", "", true, false, + "", {}, + "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", "", ""}, + + // the same tag without a requested sidecar resolves nothing + {"hole tag alone", hole, "test/repo:Q4_0", "", false, false, + "", {}, + "", "", "", "", ""}, + + // no tag anchors the sidecars on the primary quant + {"hole default anchor", hole, "test/repo", "", true, false, + "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"}, + "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", "", ""}, + + // the mtp- keyword is case sensitive, a suffix -MTP file is not discovered + {"unsloth suffix mtp", unsloth, "test/repo:Q8_K_XL", "", true, false, + "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"}, + "mmproj-BF16.gguf", "", "", "", ""}, + + // vendor prefixes and the dot quant convention both match the tag, + // first match wins between two files at the same quant + {"vendor prefix", vendors, "test/repo:Q8_0", "", false, true, + "TheDrummer_Model-24B-v4.1-Q8_0.gguf", {"TheDrummer_Model-24B-v4.1-Q8_0.gguf"}, + "", "", "", "", ""}, + + // every sidecar type resolves at the tag + {"quad exact tag", quad, "test/repo:Q8_0", "", true, false, + "model-Q8_0.gguf", {"model-Q8_0.gguf"}, + "", "mtp-model-Q8_0.gguf", "dflash-model-Q8_0.gguf", "eagle3-model-Q8_0.gguf", "dspark-model-Q8_0.gguf"}, + + // no tag anchors the dspark sidecar on the only full quant + {"spark default anchor", spark, "test/repo", "", true, false, + "model-MXFP4.gguf", {"model-MXFP4.gguf"}, + "", "", "", "", "dspark-model-MXFP4.gguf"}, + + // a tag with no matching full model still resolves the exact dspark sidecar + {"spark tag sidecar", spark, "test/repo:BF16", "", true, false, + "", {}, + "", "", "", "", "dspark-model-BF16.gguf"}, +}; + +static void check_plan(const plan_case & c) { + common_download_opts opts; + opts.download_mmproj = c.sidecars; + opts.download_mtp = c.sidecars; + opts.download_dflash = c.sidecars; + opts.download_eagle3 = c.sidecars; + opts.download_dspark = c.sidecars; + + auto plan = common_download_get_hf_plan(model_ref(c.hf_repo, c.hf_file), opts); + + REQUIRE_EQ(plan.primary.path, c.primary); + REQUIRE_EQ(plan.mmproj.path, c.mmproj); + REQUIRE_EQ(plan.mtp.path, c.mtp); + REQUIRE_EQ(plan.dflash.path, c.dflash); + REQUIRE_EQ(plan.eagle3.path, c.eagle3); + REQUIRE_EQ(plan.dspark.path, c.dspark); + + // exact shard set, order insensitive; the primary must be the first split + std::vector actual; + for (const auto & f : plan.model_files) { + actual.push_back(f.path); + } + std::sort(actual.begin(), actual.end()); + auto expected = c.model_files; + std::sort(expected.begin(), expected.end()); + REQUIRE(actual == expected); + if (!expected.empty()) { + REQUIRE(plan.primary.path == expected.front()); + } +} + +static void test_plan_resolution() { + printf("test-model-resolution: plan resolution on %zu cases\n", sizeof(plan_cases) / sizeof(plan_cases[0])); + + for (const auto & c : plan_cases) { + printf(" %s\n", c.name); + // invariant: the resolution is insensitive to the listing order + for (size_t rot = 0; rot < c.files.size(); ++rot) { + if (c.order_dependent && rot > 0) { + continue; + } + g_context = std::string(c.name) + ", reordering " + std::to_string(rot); + auto files = c.files; + std::rotate(files.begin(), files.begin() + rot, files.end()); + if (rot % 2 == 1) { + std::reverse(files.begin(), files.end()); + } + g_repos["test/repo"] = files; + check_plan(c); + } + } + g_repos.clear(); +} + +// +// end-to-end assembly: real CLI parsing, real handler init resolving over the +// loopback, downloads skipped by flipping offline before apply +// + +static void assemble(std::vector argv, common_params & params) { + std::vector cargv; + g_context.clear(); + for (auto & a : argv) { + g_context += g_context.empty() ? a : " " + a; + cargv.push_back(a.data()); + } + bool ok = common_params_parse((int) cargv.size(), cargv.data(), params, LLAMA_EXAMPLE_SERVER); + REQUIRE(ok); + + auto handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER); + + // skip the network execution, on_done still wires the params + params.offline = true; + common_models_handler_apply(handler, params); +} + +static void test_task_assembly() { + printf("test-model-resolution: end-to-end assembly\n"); + + g_repos["test/main"] = flat; + g_repos["test/hole"] = hole; + g_repos["test/quad"] = quad; + g_repos["test/dflash"] = dflash_only; + g_repos["test/eagle3"] = eagle3_only; + g_repos["test/spark"] = spark; + g_repos["test/pair"] = dspark_dflash; + g_repos["test/small"] = {"draft-model-Q4_K_M.gguf"}; + g_repos["test/preset"] = {"preset.ini", "model-Q8_0.gguf"}; + + { + // plain -hf wires the model and its mmproj, nothing speculative + common_params params; + assemble({"server", "-hf", "test/main:Q8_0"}, params); + REQUIRE_EQ(params.model.path, cached("test/main", "model-Q8_0.gguf")); + REQUIRE_EQ(params.mmproj.path, cached("test/main", "mmproj-model-Q8_0.gguf")); + REQUIRE(params.speculative.draft.mparams.path.empty()); + } + { + // --no-mmproj disables the mmproj discovery + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "--no-mmproj"}, params); + REQUIRE(params.mmproj.path.empty()); + } + { + // an explicit --mmproj wins over the discovery + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "--mmproj", "/local/mmproj.gguf"}, params); + REQUIRE(params.mmproj.path == "/local/mmproj.gguf"); + } + { + // -hf with a spec type wires the sidecar of the main repo as fallback draft + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "--spec-type", "draft-mtp"}, params); + REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/main", "mtp-model-Q8_0.gguf")); + } + { + // -hfd with a spec type wires the draft repo sidecar at its tag, + // not its full model, and suppresses the main repo fallback + common_params params; + assemble({"server", "-hf", "test/hole:Q8_0", "-hfd", "test/hole:Q4_0", "--spec-type", "draft-mtp"}, params); + REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/hole", "mtp-model-Q4_0.gguf")); + } + { + // an explicit -md file wins over the sidecar resolution + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/main", "-md", "mtp-model-BF16.gguf", "--spec-type", "draft-mtp"}, params); + REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/main", "mtp-model-BF16.gguf")); + } + { + // -hfd without a spec type auto-selects the type, mtp first when all ship + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/quad:Q8_0"}, params); + REQUIRE(params.speculative.types == std::vector{COMMON_SPECULATIVE_TYPE_DRAFT_MTP}); + REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/quad", "mtp-model-Q8_0.gguf")); + } + { + // auto-selection with only a dflash sidecar + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/dflash:Q8_0"}, params); + REQUIRE(params.speculative.types == std::vector{COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH}); + REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/dflash", "dflash-model-Q8_0.gguf")); + } + { + // auto-selection with only an eagle3 sidecar + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/eagle3:Q8_0"}, params); + REQUIRE(params.speculative.types == std::vector{COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3}); + REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/eagle3", "eagle3-model-Q8_0.gguf")); + } + { + // auto-selection prefers dspark over dflash when both ship + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/pair:Q8_0"}, params); + REQUIRE(params.speculative.types == std::vector{COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK}); + REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/pair", "dspark-model-Q8_0.gguf")); + } + { + // -hf with the dspark spec type wires the sidecar of the main repo, + // anchored on the only full quant + common_params params; + assemble({"server", "-hf", "test/spark", "--spec-type", "draft-dspark"}, params); + REQUIRE_EQ(params.model.path, cached("test/spark", "model-MXFP4.gguf")); + REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/spark", "dspark-model-MXFP4.gguf")); + } + { + // -hfd on a repo without sidecars keeps resolving a full model as draft + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/small"}, params); + REQUIRE(params.speculative.types == std::vector{COMMON_SPECULATIVE_TYPE_NONE}); + REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/small", "draft-model-Q4_K_M.gguf")); + } + { + // a preset repo wires the preset and clears the model for router mode + common_params params; + assemble({"server", "-hf", "test/preset"}, params); + REQUIRE_EQ(params.models_preset, cached("test/preset", "preset.ini")); + REQUIRE(params.model.path.empty()); + REQUIRE(params.model.hf_repo.empty()); + } + + g_repos.clear(); +} + +int main(void) { + // unbuffered, so a crash cannot swallow the reports already printed + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); + + // the negative cases legitimately log errors on every reordering, + // keep the output down to the reports + common_log_pause(common_log_main()); + + // the loopback endpoint also keeps the client init from rejecting + // https on the builds without TLS support + httplib::Server server; + serve_repos(server); + int port = server.bind_to_any_port("127.0.0.1"); + + // isolate the cache, its location is read once so it is set + // before anything else + cache_dir = std::filesystem::temp_directory_path() / + ("test-model-resolution-cache-" + std::to_string(port)); + std::filesystem::remove_all(cache_dir); + common_set_env("LLAMA_CACHE", cache_dir.string()); + + std::thread server_thread([&server] { server.listen_after_bind(); }); + server.wait_until_ready(); + common_set_env("MODEL_ENDPOINT", "http://127.0.0.1:" + std::to_string(port) + "/"); + + test_plan_resolution(); + test_task_assembly(); + + server.stop(); + server_thread.join(); + + std::filesystem::remove_all(cache_dir); + printf("test-model-resolution: all tests OK\n"); + return 0; +} From 57c092139a175b991331976882907dbbed17889d Mon Sep 17 00:00:00 2001 From: jacekpoplawski <67507230+jacekpoplawski@users.noreply.github.com> Date: Mon, 3 Aug 2026 20:27:52 +0200 Subject: [PATCH 47/98] model : support MTP in GLM-4.7-Flash (#24868) --- conversion/glm.py | 60 +++++++ gguf-py/gguf/constants.py | 7 + src/models/deepseek2.cpp | 333 +++++++++++++++++++++++++++++++++++--- src/models/models.h | 4 + 4 files changed, 379 insertions(+), 25 deletions(-) diff --git a/conversion/glm.py b/conversion/glm.py index cc34cddbf843..e28f54574e07 100644 --- a/conversion/glm.py +++ b/conversion/glm.py @@ -206,10 +206,70 @@ def prepare_tensors(self): @ModelBase.register("Glm4MoeLiteForCausalLM") class Glm4MoeLiteModel(DeepseekV2Model): model_arch = gguf.MODEL_ARCH.DEEPSEEK2 + skip_mtp = False + supports_mtp_export = True + _n_main_layers: int | None = None def set_vocab(self): return self._set_vocab_glm() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + num_hidden_layers = self.hparams["num_hidden_layers"] + self.num_nextn_predict_layers = self.hparams.get("num_nextn_predict_layers", 0) + self.skip_mtp = self.no_mtp or self.num_nextn_predict_layers == 0 + + if self.skip_mtp: + self.block_count = num_hidden_layers + else: + self.block_count = num_hidden_layers + self.num_nextn_predict_layers + + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + if self.skip_mtp: + return + + self.gguf_writer.add_nextn_predict_layers(self.num_nextn_predict_layers) + + def index_tensors(self, remote_hf_model_id: str | None = None): + type(self)._n_main_layers = self.hparams["num_hidden_layers"] + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + + @classmethod + def filter_tensors(cls, item): + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem + + if cls._n_main_layers is not None: + match = re.match(r"model\.layers\.(\d+)\.", name) + is_mtp = match is not None and int(match.group(1)) >= cls._n_main_layers + if is_mtp and cls.no_mtp: + return None + if cls.mtp_only and not is_mtp and name not in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + ): + return None + + return name, gen + + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) + + if not self.mtp_only or not from_dir: + return + + output_type: str = self.ftype.name.partition("_")[2] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, + self.metadata.version, size_label=None, output_type=output_type, model_type=None) + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + @ModelBase.register("GlmMoeDsaForCausalLM") class GlmMoeDsaModel(DeepseekV2Model): diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 7df984432d43..c9ec92bd8ed4 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -3221,6 +3221,13 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN_SHEXP, MODEL_TENSOR.FFN_UP_SHEXP, MODEL_TENSOR.FFN_EXP_PROBS_B, + # NextN/MTP tensors + MODEL_TENSOR.NEXTN_EH_PROJ, + MODEL_TENSOR.NEXTN_EMBED_TOKENS, + MODEL_TENSOR.NEXTN_ENORM, + MODEL_TENSOR.NEXTN_HNORM, + MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, + MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], MODEL_ARCH.DEEPSEEK2OCR: [ MODEL_TENSOR.TOKEN_EMBD, diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp index a9e8bc514036..ba90c0d0776b 100644 --- a/src/models/deepseek2.cpp +++ b/src/models/deepseek2.cpp @@ -37,6 +37,11 @@ void llama_model_deepseek2::load_arch_hparams(llama_model_loader & ml) { hparams.rope_yarn_log_mul /= 0.1f; } + // NextN/MTP + ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false); + GGML_ASSERT(hparams.n_layer_nextn == 0 || + hparams.n_layer() + hparams.n_layer_nextn == hparams.n_layer_all); + // (optional) temperature tuning - used by mistral-large ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE, hparams.f_attn_temp_scale, false); ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.n_attn_temp_floor_scale, false); // FIXME why not use temperature_length? @@ -52,10 +57,20 @@ void llama_model_deepseek2::load_arch_hparams(llama_model_loader & ml) { } } -void llama_model_deepseek2::load_arch_tensors(llama_model_loader &) { +void llama_model_deepseek2::load_arch_tensors(llama_model_loader & ml) { LLAMA_LOAD_LOCALS; const int64_t n_expert_shared = hparams.n_expert_shared; + const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); + const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; + const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); + const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } + const bool is_mla = hparams.is_mla(); // note: these are the actual head sizes you get when treating as MHA or after "decompression" using wv_b for MLA @@ -81,44 +96,45 @@ void llama_model_deepseek2::load_arch_tensors(llama_model_loader &) { output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); } - for (int i = 0; i < n_layer; ++i) { + for (int i = 0; i < n_layer_all; ++i) { auto & layer = layers[i]; + const int flags = i < n_layer ? trunk_flags : mtp_flags; - layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags); if (q_lora_rank > 0) { - layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0); + layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, flags); } - layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0); + layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, flags); if (q_lora_rank > 0) { - layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0); - layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head_k_mla}, 0); + layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, flags); + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head_k_mla}, flags); } else { - layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_embd_head_k_mla}, 0); + layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_embd_head_k_mla}, flags); } - layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, 0); + layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, flags); // note: only old legacy GGUF files will have the unsplit wkv_b tensor in if (is_mla) { - layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head}, 0); - layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, 0); + layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head}, flags); + layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, flags); } else { - layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, "weight", i), {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v_mla)}, 0); + layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, "weight", i), {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v_mla)}, flags); } - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v_mla, n_embd}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v_mla, n_embd}, flags); - layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags); if (i < (int) hparams.n_layer_dense_lead) { - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, flags); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, flags); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, flags); } else { - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, flags); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED | flags); if (n_expert == 0) { throw std::runtime_error("n_expert must be > 0"); @@ -128,21 +144,281 @@ void llama_model_deepseek2::load_arch_tensors(llama_model_loader &) { } // MoE branch - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); - create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, flags); + create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, flags); // Shared expert branch - layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); - layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd}, 0); - layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd}, flags); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); + } + + // NextN/MTP tensors + if (i >= n_layer) { + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, mtp_flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, mtp_flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), { n_embd }, mtp_flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED | flags); } } } std::unique_ptr llama_model_deepseek2::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique(*this, params); + } return std::make_unique(*this, params); } +llama_model_deepseek2::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + GGML_ASSERT(hparams.n_layer_nextn > 0 && "GLM4 MTP requires n_layer_nextn > 0"); + GGML_ASSERT(hparams.n_layer_nextn == 1 && "GLM4 MTP currently only supports a single MTP block"); + GGML_ASSERT(hparams.is_mla() && "GLM4 MTP requires MLA"); + GGML_ASSERT(hparams.f_attn_temp_scale == 0.0f && "GLM4 MTP does not support attention temperature scaling"); + + // The appended MTP block is stored immediately after the main decoder layers. + const int il = hparams.n_layer(); + const auto & layer = model.layers[il]; + + GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj"); + GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm"); + GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); + + GGML_ASSERT((uint32_t) il >= hparams.n_layer_dense_lead && "GLM4 MTP block expected to use MoE FFN"); + + const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; + const int64_t kv_lora_rank = hparams.n_lora_kv; + + GGML_ASSERT(n_embd_head_qk_nope >= 1); + GGML_ASSERT(hparams.n_lora_q > 0); + GGML_ASSERT(layer.wq_a); + GGML_ASSERT(layer.attn_q_a_norm); + GGML_ASSERT(layer.wq_b); + GGML_ASSERT(layer.wkv_a_mqa); + GGML_ASSERT(layer.attn_kv_a_norm); + GGML_ASSERT(layer.wk_b); + + const bool has_split_exps = + layer.ffn_up_exps != nullptr && + layer.ffn_gate_exps != nullptr; + + const bool has_fused_exps = layer.ffn_gate_up_exps != nullptr; + + GGML_ASSERT(has_split_exps || has_fused_exps); + GGML_ASSERT(layer.ffn_norm); + GGML_ASSERT(layer.ffn_gate_inp); + GGML_ASSERT(layer.ffn_down_exps); + GGML_ASSERT(layer.ffn_gate_shexp); + GGML_ASSERT(layer.ffn_down_shexp); + GGML_ASSERT(layer.ffn_up_shexp); + + auto inp = std::make_unique(hparams.n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); + ggml_set_input(inp->embd); + + ggml_tensor * tok_embd; + if (ubatch.token) { + ggml_tensor * tok_embd_w = layer.nextn.embed_tokens + ? layer.nextn.embed_tokens + : model.tok_embd; + + tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens); + } else { + tok_embd = inp->embd; + } + cb(tok_embd, "mtp_tok_embd", il); + + inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); + ggml_set_input(inp->h); + ggml_set_name(inp->h, "mtp_h_input"); + + ggml_tensor * h_embd = inp->h; + + res->add_input(std::move(inp)); + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + auto * inp_attn_k = build_attn_inp_k(); + + ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + + ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il); + cb(e_norm, "mtp_enorm", il); + + ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, 0); + cb(concat, "mtp_concat", il); + + ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s); + cb(cur, "mtp_eh_proj", il); + + ggml_tensor * inpSA = cur; + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_norm", il); + + ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_a, cur); + cb(q, "mtp_q_a", il); + + q = build_norm(q, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(q, "mtp_q_a_norm", il); + + q = ggml_mul_mat(ctx0, layer.wq_b, q); + cb(q, "mtp_q_b", il); + + ggml_tensor * q_nope = + ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, + ggml_row_size(q->type, n_embd_head_k_mla), + ggml_row_size(q->type, n_embd_head_k_mla) * n_head, 0); + cb(q_nope, "mtp_q_nope", il); + + ggml_tensor * q_pe = + ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, + ggml_row_size(q->type, n_embd_head_k_mla), + ggml_row_size(q->type, n_embd_head_k_mla) * n_head, + ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "mtp_q_pe", il); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur); + cb(kv_cmpr_pe, "mtp_kv_cmpr_pe", il); + + ggml_tensor * kv_cmpr = + ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + cb(kv_cmpr, "mtp_kv_cmpr", il); + + ggml_tensor * k_pe = + ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + cb(k_pe, "mtp_k_pe", il); + + kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + cb(kv_cmpr, "mtp_kv_cmpr_norm", il); + + GGML_ASSERT(ext_factor >= 0.0f); + + const float attn_factor_org = + attn_factor * (1.0f + 0.1f * logf(1.0f / freq_scale)); + + const float mscale = + attn_factor_org * (1.0f + 0.1f * hparams.rope_yarn_log_mul * logf(1.0f / freq_scale)); + + const float kq_scale = + 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k_mla)); + + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(q_pe, "mtp_q_pe_rope", il); + + k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(k_pe, "mtp_k_pe_rope", il); + + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + cb(q_nope, "mtp_q_nope_perm", il); + + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope); + cb(q_nope_absorbed, "mtp_q_nope_absorbed", il); + + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + cb(q_nope_absorbed, "mtp_q_nope_absorbed_perm", il); + + ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + cb(Qcur, "mtp_Qcur", il); + + kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, hparams.n_lora_kv, 1, n_tokens); + cb(kv_cmpr, "mtp_kv_cmpr_reshape", il); + + ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); + cb(Kcur, "mtp_Kcur", il); + + ggml_tensor * Vcur = kv_cmpr; + cb(Vcur, "mtp_Vcur", il); + + cur = build_attn(inp_attn_k, + layer.wo, nullptr, layer.wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale, il); + cb(cur, "mtp_attn_out", il); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "mtp_ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_ffn_norm", il); + + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, + layer.ffn_gate_up_exps); + cb(moe_out, "mtp_ffn_moe_out", il); + + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, nullptr, nullptr, + layer.ffn_gate_shexp, nullptr, nullptr, + layer.ffn_down_shexp, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "mtp_ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "mtp_ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "mtp_post_ffn", il); + + ggml_tensor * head_norm_w = layer.nextn.shared_head_norm + ? layer.nextn.shared_head_norm + : model.output_norm; + GGML_ASSERT(head_norm_w && "GLM4 MTP: missing both nextn.shared_head_norm and output_norm"); + + cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1); + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + cb(cur, "mtp_shared_head_norm", -1); + + ggml_tensor * head_w = layer.nextn.shared_head_head + ? layer.nextn.shared_head_head + : model.output; + + ggml_tensor * head_s = layer.nextn.shared_head_head + ? layer.nextn.shared_head_head_s + : model.output_s; + + GGML_ASSERT(head_w && "GLM4 MTP: missing LM head (nextn.shared_head_head or model.output)"); + + cur = build_lora_mm(head_w, cur, head_s); + cb(cur, "result_output", -1); + + res->t_logits = cur; + ggml_build_forward_expand(gf, cur); +} + llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { // lite variants include DeepSeek-V2-Lite, GigaChat3-10B-A1.8B @@ -365,7 +641,7 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); } } - if (il == n_layer - 1 && inp_out_ids) { + if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); } @@ -425,6 +701,13 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + cb(cur, "result_norm", -1); res->t_embd = cur; diff --git a/src/models/models.h b/src/models/models.h index 930cc3184233..5f206621d579 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1084,6 +1084,10 @@ struct llama_model_deepseek2 : public llama_model_base { graph(const llama_model & model, const llm_graph_params & params); }; + struct graph_mtp : public llm_graph_context { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; From fe2adf0e722f30f5295fdec8a0f1dc788f7498bc Mon Sep 17 00:00:00 2001 From: "Alessandro de Oliveira Faria (A.K.A.CABELO)" Date: Mon, 3 Aug 2026 15:31:15 -0300 Subject: [PATCH 48/98] vendor : update BoringSSL to 0.20260803.0 (#26523) --- vendor/cpp-httplib/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/cpp-httplib/CMakeLists.txt b/vendor/cpp-httplib/CMakeLists.txt index bf674583ff9c..49c419094a01 100644 --- a/vendor/cpp-httplib/CMakeLists.txt +++ b/vendor/cpp-httplib/CMakeLists.txt @@ -41,7 +41,7 @@ if (LLAMA_BUILD_BORINGSSL) set(FIPS OFF CACHE BOOL "Enable FIPS (BoringSSL)") set(BORINGSSL_GIT "https://boringssl.googlesource.com/boringssl" CACHE STRING "BoringSSL git repository") - set(BORINGSSL_VERSION "0.20260730.0" CACHE STRING "BoringSSL version") + set(BORINGSSL_VERSION "0.20260803.0" CACHE STRING "BoringSSL version") message(STATUS "Fetching BoringSSL version ${BORINGSSL_VERSION}") From 94bc47f2807805ffdc1c5fbe5dce5cd2afdf3a97 Mon Sep 17 00:00:00 2001 From: "Alessandro de Oliveira Faria (A.K.A.CABELO)" Date: Mon, 3 Aug 2026 19:30:42 -0300 Subject: [PATCH 49/98] vendor : update cpp-httplib to 0.52.0 (#26485) --- scripts/sync_vendor.py | 2 +- vendor/cpp-httplib/httplib.cpp | 583 +++++++++++++++++++++++---------- vendor/cpp-httplib/httplib.h | 331 ++++++++++++++++++- 3 files changed, 732 insertions(+), 184 deletions(-) diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 679e557a3215..9faa6a307c13 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -5,7 +5,7 @@ import sys import subprocess -HTTPLIB_VERSION = "refs/tags/v0.51.0" +HTTPLIB_VERSION = "refs/tags/v0.52.0" vendor = { "https://github.com/nlohmann/json/releases/latest/download/json.hpp": "vendor/nlohmann/json.hpp", diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp index 15517963874a..30e2de896220 100644 --- a/vendor/cpp-httplib/httplib.cpp +++ b/vendor/cpp-httplib/httplib.cpp @@ -1412,6 +1412,46 @@ bool stream_line_reader::getline() { #endif for (size_t i = 0;; i++) { + // Fast path: whatever the stream has already buffered can be scanned for + // the terminator in one pass. Asking for a byte at a time costs a virtual + // call, a bounds check and a one-byte copy per character of the request. + size_t buffered_size = 0; + if (auto buffered = strm_.buffered_data(buffered_size)) { + auto take = buffered_size; + auto terminated = false; + + for (size_t at = 0; at < buffered_size;) { + auto nl = static_cast( + memchr(buffered + at, '\n', buffered_size - at)); + if (!nl) { break; } + auto pos = static_cast(nl - buffered); +#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR + take = pos + 1; + terminated = true; + break; +#else + // A bare LF does not end the line; keep looking for CRLF. The CR may + // be the last byte of an earlier chunk, hence prev_byte. + if ((pos > 0 ? buffered[pos - 1] : prev_byte) == '\r') { + take = pos + 1; + terminated = true; + break; + } + at = pos + 1; +#endif + } + + if (size() + take > CPPHTTPLIB_MAX_LINE_LENGTH) { return false; } +#ifndef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR + prev_byte = buffered[take - 1]; +#endif + append(buffered, take); + strm_.consume_buffered(take); + i += take; + if (terminated) { return true; } + continue; + } + if (size() >= CPPHTTPLIB_MAX_LINE_LENGTH) { // Treat exceptionally long lines as an error to // prevent infinite loops/memory exhaustion @@ -1443,16 +1483,26 @@ bool stream_line_reader::getline() { return true; } -void stream_line_reader::append(char c) { - if (fixed_buffer_used_size_ < fixed_buffer_size_ - 1) { - fixed_buffer_[fixed_buffer_used_size_++] = c; +void stream_line_reader::append(char c) { append(&c, 1); } + +void stream_line_reader::append(const char *data, size_t size) { + // Once the line has outgrown the fixed buffer everything must keep going to + // the growable one, even if a later chunk would have fit. Without the + // emptiness check a short append after a long one would land in the fixed + // buffer, which ptr() and size() no longer look at, and be lost. + if (growable_buffer_.empty() && + fixed_buffer_used_size_ + size < fixed_buffer_size_) { + memcpy(fixed_buffer_ + fixed_buffer_used_size_, data, size); + fixed_buffer_used_size_ += size; fixed_buffer_[fixed_buffer_used_size_] = '\0'; } else { + // Unlike the per-character overload, this can be the very first append of + // the line, so the fixed buffer may hold nothing and carry no terminator + // yet. assign() takes an explicit length and does not need one. if (growable_buffer_.empty()) { - assert(fixed_buffer_[fixed_buffer_used_size_] == '\0'); growable_buffer_.assign(fixed_buffer_, fixed_buffer_used_size_); } - growable_buffer_ += c; + growable_buffer_.append(data, size); } } @@ -1525,6 +1575,14 @@ bool mmap::open(const char *path) { is_open_empty_file = true; return false; } + + if (addr_ == MAP_FAILED) { + // Clear the sentinel before `close()`, since `is_open()` only checks + // `addr_` against nullptr and `munmap()` must not be called with it. + addr_ = nullptr; + close(); + return false; + } #endif return true; @@ -1702,8 +1760,17 @@ class SocketStream final : public Stream { socket_t socket() const override; time_t duration() const override; void set_read_timeout(time_t sec, time_t usec = 0) override; + const char *buffered_data(size_t &size) const override; + void consume_buffered(size_t size) override; + + // The caller has just seen this socket become readable. Lets the next read + // skip its own readiness wait, which would otherwise ask the kernel a + // question that was answered a moment ago. Consumed by that read. + void set_readable_hint() { readable_hint_ = true; } private: + bool ensure_readable(); + socket_t sock_; time_t read_timeout_sec_; time_t read_timeout_usec_; @@ -1715,6 +1782,7 @@ class SocketStream final : public Stream { std::vector read_buff_; size_t read_buff_off_ = 0; size_t read_buff_content_size_ = 0; + bool readable_hint_ = false; static const size_t read_buff_size_ = 1024l * 4; }; @@ -1782,6 +1850,9 @@ process_server_socket(const std::atomic &svr_sock, socket_t sock, [&](bool close_connection, bool &connection_closed) { SocketStream strm(sock, read_timeout_sec, read_timeout_usec, write_timeout_sec, write_timeout_usec); + // process_server_socket_core() only gets here once keep_alive() has + // seen the socket go readable. + strm.set_readable_hint(); return callback(strm, close_connection, connection_closed); }); } @@ -3071,19 +3142,49 @@ bool zstd_decompressor::decompress(const char *data, size_t data_length, } #endif +bool contains_case_ignore(const std::string &s, const char *token) { + auto token_end = token + std::strlen(token); + return std::search(s.begin(), s.end(), token, token_end, [](char a, char b) { + return case_ignore::to_lower(a) == case_ignore::to_lower(b); + }) != s.end(); +} + +// Content codings are case-insensitive (RFC 9110 8.4.1). Matching them +// case-sensitively would make a response labeled e.g. "GZIP" look like an +// unknown coding, and its payload would be handed back still compressed. +bool is_zlib_encoding(const std::string &encoding) { + return case_ignore::equal(encoding, "gzip") || + case_ignore::equal(encoding, "deflate"); +} + +bool is_brotli_encoding(const std::string &encoding) { + return contains_case_ignore(encoding, "br"); +} + +bool is_zstd_encoding(const std::string &encoding) { + return contains_case_ignore(encoding, "zstd"); +} + +// Returns true if the content coding is one cpp-httplib is able to decompress +// when the corresponding support is compiled in. +bool is_known_content_encoding(const std::string &encoding) { + return is_zlib_encoding(encoding) || is_brotli_encoding(encoding) || + is_zstd_encoding(encoding); +} + std::unique_ptr create_decompressor(const std::string &encoding) { std::unique_ptr decompressor; - if (encoding == "gzip" || encoding == "deflate") { + if (is_zlib_encoding(encoding)) { #ifdef CPPHTTPLIB_ZLIB_SUPPORT decompressor = detail::make_unique(); #endif - } else if (encoding.find("br") != std::string::npos) { + } else if (is_brotli_encoding(encoding)) { #ifdef CPPHTTPLIB_BROTLI_SUPPORT decompressor = detail::make_unique(); #endif - } else if (encoding == "zstd" || encoding.find("zstd") != std::string::npos) { + } else if (is_zstd_encoding(encoding)) { #ifdef CPPHTTPLIB_ZSTD_SUPPORT decompressor = detail::make_unique(); #endif @@ -3145,8 +3246,7 @@ const char *get_header_value(const Headers &headers, size_t get_header_value_count(const Headers &headers, const std::string &key) { - auto r = headers.equal_range(key); - return static_cast(std::distance(r.first, r.second)); + return headers.count(key); } template @@ -3370,44 +3470,33 @@ ReadContentResult read_content_chunked(Stream &strm, T &x, bool is_chunked_transfer_encoding(const Headers &headers) { // RFC 9112 6.1: a message is framed with the chunked coding when "chunked" // is the final transfer coding. A single field value may list several - // codings ("gzip, chunked"), and the list may be split across multiple - // Transfer-Encoding header lines (RFC 9110 5.3). Match the last coding token - // case-insensitively rather than comparing the whole value against "chunked". + // codings ("gzip, chunked"), and RFC 9110 5.3 lets that list be split across + // several Transfer-Encoding lines, which combine into one comma-separated + // list in the order the lines were received. Headers preserves that order, + // so the final coding is the last token of the last line. Match it + // case-insensitively rather than comparing the whole value against + // "chunked". // // Security: reading a chunked message as unframed leaves its body in the // socket, where a keep-alive connection parses it as a smuggled request. - // Headers is an unordered_multimap whose iteration order for duplicate keys - // is not portable, so when there is more than one Transfer-Encoding line we - // cannot tell which coding is truly final. In that ambiguous case we fail - // safe by treating the message as chunked (a mis-parse just closes the - // connection, whereas the opposite error enables smuggling). + // Server::process_request() answers 400 and closes when the final coding is + // not chunked, so a request whose framing cannot be determined never + // reaches the "no body" path. auto rng = headers.equal_range("Transfer-Encoding"); + if (rng.first == rng.second) { return false; } - size_t line_count = 0; - bool chunked_present = false; - bool last_line_ends_with_chunked = false; + // Cleared per line, so a trailing line carrying no coding at all leaves the + // combined list ending in nothing rather than inheriting the line before it. + std::string last_coding; for (auto it = rng.first; it != rng.second; ++it) { - line_count++; const auto &value = it->second; - - std::string last_coding; - bool line_has_chunked = false; + last_coding.clear(); split(value.data(), value.data() + value.size(), ',', - [&](const char *b, const char *e) { - last_coding.assign(b, e); - if (case_ignore::equal(last_coding, "chunked")) { - line_has_chunked = true; - } - }); - - if (line_has_chunked) { chunked_present = true; } - last_line_ends_with_chunked = case_ignore::equal(last_coding, "chunked"); + [&](const char *b, const char *e) { last_coding.assign(b, e); }); } - if (line_count == 0) { return false; } - if (line_count == 1) { return last_line_ends_with_chunked; } - return chunked_present; + return case_ignore::equal(last_coding, "chunked"); } template @@ -3420,9 +3509,12 @@ bool prepare_content_receiver(T &x, int &status, std::unique_ptr decompressor; if (!encoding.empty()) { + // A coding we know about but were not built with is an error. An + // unrecognized coding (including "identity") is left alone and the + // payload is passed through as-is, since some servers misuse the header, + // e.g. by sending a character set such as "Content-Encoding: UTF-8". decompressor = detail::create_decompressor(encoding); - if (!decompressor) { - // Unsupported encoding or no support compiled in + if (!decompressor && detail::is_known_content_encoding(encoding)) { status = StatusCode::UnsupportedMediaType_415; return false; } @@ -3845,6 +3937,19 @@ std::string params_to_query_str(const Params ¶ms) { return query; } +// Splits one "key=value" span of a query string at its first '='. A span with +// no '=' at all lands entirely in key, leaving val empty, which is how a bare +// "?flag" keeps its name. +void divide_query_pair(const char *b, const char *e, std::string &key, + std::string &val) { + divide(b, static_cast(e - b), '=', + [&](const char *lhs_data, std::size_t lhs_size, const char *rhs_data, + std::size_t rhs_size) { + key.assign(lhs_data, lhs_size); + val.assign(rhs_data, rhs_size); + }); +} + void parse_query_text(const char *data, std::size_t size, Params ¶ms) { std::set cache; @@ -3855,12 +3960,7 @@ void parse_query_text(const char *data, std::size_t size, std::string key; std::string val; - divide(b, static_cast(e - b), '=', - [&](const char *lhs_data, std::size_t lhs_size, const char *rhs_data, - std::size_t rhs_size) { - key.assign(lhs_data, lhs_size); - val.assign(rhs_data, rhs_size); - }); + divide_query_pair(b, e, key, val); if (!key.empty()) { params.emplace(decode_query_component(key), decode_query_component(val)); @@ -3874,20 +3974,18 @@ void parse_query_text(const std::string &s, Params ¶ms) { // Normalize a query string by decoding and re-encoding each key/value pair // while preserving the original parameter order. This avoids double-encoding -// and ensures consistent encoding without reordering (unlike Params which -// uses std::multimap and sorts keys). +// and ensures consistent encoding. It works on the raw string rather than +// parsing into Params and re-serializing, because that round trip cannot +// reproduce the input: params_to_query_str() always emits '=', so a bare +// "flag" would come back as "flag=", and parse_query_text() drops exactly +// duplicated pairs. std::string normalize_query_string(const std::string &query) { std::string result; split(query.data(), query.data() + query.size(), '&', [&](const char *b, const char *e) { std::string key; std::string val; - divide(b, static_cast(e - b), '=', - [&](const char *lhs_data, std::size_t lhs_size, - const char *rhs_data, std::size_t rhs_size) { - key.assign(lhs_data, lhs_size); - val.assign(rhs_data, rhs_size); - }); + divide_query_pair(b, e, key, val); if (!key.empty()) { auto dec_key = decode_query_component(key); @@ -3904,6 +4002,43 @@ std::string normalize_query_string(const std::string &query) { return result; } +// Build the request target that goes on the wire from a caller-supplied path. +// Shared by the buffered send path and the streaming API so that both put the +// same bytes in the request line for the same input. +std::string encode_request_target(const std::string &target, + bool path_encode) { + // `substr(0, npos)` yields the whole string, which is what the no-query + // case needs. + auto query_pos = target.find('?'); + auto path_part = target.substr(0, query_pos); + std::string query_part; + if (query_pos != std::string::npos) { + query_part = target.substr(query_pos + 1); + } + + auto result = path_encode ? encode_path(path_part) : std::move(path_part); + + if (!query_part.empty()) { + // When path encoding is disabled the caller has supplied an already-encoded + // target and expects the exact bytes to be sent on the wire, so skip + // normalization for the query too. Normalizing would decode-then-re-encode + // it and corrupt pre-encoded binary payloads (e.g. turning `%20` into `+`, + // which a strict RFC 3986 server decodes back as `+`, not a space). + if (path_encode) { + auto normalized = normalize_query_string(query_part); + if (!normalized.empty()) { + result += '?'; + result += normalized; + } + } else { + result += '?'; + result += query_part; + } + } + + return result; +} + bool parse_multipart_boundary(const std::string &content_type, std::string &boundary) { std::map params; @@ -4969,21 +5104,8 @@ bool is_field_valid(const std::string &name, const std::string &value) { } // namespace fields -bool perform_websocket_handshake(Stream &strm, const std::string &host, - int port, bool is_ssl, - const std::string &path, - const Headers &headers, +bool perform_websocket_handshake(Stream &strm, Request &req, std::string &selected_subprotocol) { - // Validate path and host - if (!fields::is_field_value(path) || !fields::is_field_value(host)) { - return false; - } - - // Validate user-provided headers - for (const auto &h : headers) { - if (!fields::is_field_valid(h.first, h.second)) { return false; } - } - // Generate random Sec-WebSocket-Key thread_local std::mt19937 rng(std::random_device{}()); std::string key_bytes(16, '\0'); @@ -4993,19 +5115,30 @@ bool perform_websocket_handshake(Stream &strm, const std::string &host, } auto client_key = base64_encode(key_bytes); - // Build upgrade request - std::string req_str = "GET " + path + " HTTP/1.1\r\n"; - req_str += "Host: " + make_host_and_port_string(host, port, is_ssl) + "\r\n"; - req_str += "Upgrade: websocket\r\n"; - req_str += "Connection: Upgrade\r\n"; - req_str += "Sec-WebSocket-Key: " + client_key + "\r\n"; - req_str += "Sec-WebSocket-Version: 13\r\n"; - for (const auto &h : headers) { - req_str += h.first + ": " + h.second + "\r\n"; + req.headers.erase("Upgrade"); + req.headers.erase("Connection"); + req.headers.erase("Sec-WebSocket-Key"); + req.headers.erase("Sec-WebSocket-Version"); + req.headers.emplace("Upgrade", "websocket"); + req.headers.emplace("Connection", "Upgrade"); + req.headers.emplace("Sec-WebSocket-Key", client_key); + req.headers.emplace("Sec-WebSocket-Version", "13"); + + // Build the request in memory first, like ClientImpl::write_request does. + // Writing straight to the socket would leak a request line onto the wire + // before check_and_write_headers gets a chance to reject an invalid header, + // and would emit one small write per header. + BufferStream bstrm; + + if (write_request_line(bstrm, req.method, req.path) < 0) { return false; } + + auto error = Error::Success; + if (!check_and_write_headers(bstrm, req.headers, write_headers, error)) { + return false; } - req_str += "\r\n"; - if (strm.write(req_str.data(), req_str.size()) < 0) { return false; } + const auto &data = bstrm.get_buffer(); + if (!write_data(strm, data.data(), data.size())) { return false; } // Verify 101 response and Sec-WebSocket-Accept header auto expected_accept = websocket_accept_key(client_key); @@ -5013,6 +5146,39 @@ bool perform_websocket_handshake(Stream &strm, const std::string &host, selected_subprotocol); } +bool is_ip_address(const std::string &host) { + struct in_addr addr4; + struct in6_addr addr6; + return inet_pton(AF_INET, host.c_str(), &addr4) == 1 || + inet_pton(AF_INET6, host.c_str(), &addr6) == 1; +} + +// Resolve where a client should connect for `host`, honoring a user-supplied +// hostname-to-address map. `host` itself is never rewritten, so it keeps +// supplying the Host header and SNI; only the connection target changes. +// +// A mapped IP literal goes to `ip`, which keeps create_socket's AI_NUMERICHOST +// path. Anything else goes to `connect_host`, which create_socket resolves as +// a name, or uses as the socket path when the address family is AF_UNIX. An +// absent or empty mapping leaves `host` as the connection target; without the +// empty check the value would reach getaddrinfo as a null node and silently +// resolve to loopback. +void apply_addr_map(const std::map &addr_map, + const std::string &host, std::string &connect_host, + std::string &ip) { + connect_host = host; + ip.clear(); + + auto it = addr_map.find(host); + if (it == addr_map.end() || it->second.empty()) { return; } + + if (is_ip_address(it->second)) { + ip = it->second; + } else { + connect_host = it->second; + } +} + } // namespace detail /* @@ -5044,7 +5210,12 @@ class SSLSocketStream final : public Stream { time_t duration() const override; void set_read_timeout(time_t sec, time_t usec = 0) override; + // See SocketStream::set_readable_hint(). + void set_readable_hint() { readable_hint_ = true; } + private: + bool ensure_readable(); + socket_t sock_; tls::session_t session_; time_t read_timeout_sec_; @@ -5053,6 +5224,7 @@ class SSLSocketStream final : public Stream { time_t write_timeout_usec_; time_t max_timeout_msec_; const std::chrono::time_point start_time_; + bool readable_hint_ = false; }; #ifdef CPPHTTPLIB_OPENSSL_SUPPORT @@ -5196,13 +5368,6 @@ std::string SHA_512(const std::string &s) { } #endif -bool is_ip_address(const std::string &host) { - struct in_addr addr4; - struct in6_addr addr6; - return inet_pton(AF_INET, host.c_str(), &addr4) == 1 || - inet_pton(AF_INET6, host.c_str(), &addr6) == 1; -} - template bool process_server_socket_ssl( const std::atomic &svr_sock, tls::session_t session, @@ -5214,6 +5379,8 @@ bool process_server_socket_ssl( [&](bool close_connection, bool &connection_closed) { SSLSocketStream strm(sock, session, read_timeout_sec, read_timeout_usec, write_timeout_sec, write_timeout_usec); + // See the non-TLS path in process_server_socket(). + strm.set_readable_hint(); return callback(strm, close_connection, connection_closed); }); } @@ -5665,6 +5832,7 @@ std::string to_string(const Error error) { case Error::UnsupportedAddressFamily: return "Unsupported address family"; case Error::HTTPParsing: return "HTTP parsing failed"; case Error::InvalidRangeHeader: return "Invalid Range header"; + case Error::UnsupportedContentEncoding: return "Unsupported Content-Encoding"; default: break; } @@ -6046,8 +6214,7 @@ std::string Request::get_trailer_value(const std::string &key, } size_t Request::get_trailer_value_count(const std::string &key) const { - auto r = trailers.equal_range(key); - return static_cast(std::distance(r.first, r.second)); + return trailers.count(key); } bool Request::has_param(const std::string &key) const { @@ -6071,8 +6238,7 @@ Request::get_param_values(const std::string &key) const { } size_t Request::get_param_value_count(const std::string &key) const { - auto r = params.equal_range(key); - return static_cast(std::distance(r.first, r.second)); + return params.count(key); } bool Request::is_multipart_form_data() const { @@ -6105,8 +6271,7 @@ bool MultipartFormData::has_field(const std::string &key) const { } size_t MultipartFormData::get_field_count(const std::string &key) const { - auto r = fields.equal_range(key); - return static_cast(std::distance(r.first, r.second)); + return fields.count(key); } FormData MultipartFormData::get_file(const std::string &key, @@ -6129,8 +6294,7 @@ bool MultipartFormData::has_file(const std::string &key) const { } size_t MultipartFormData::get_file_count(const std::string &key) const { - auto r = files.equal_range(key); - return static_cast(std::distance(r.first, r.second)); + return files.count(key); } // Multipart FormData writer implementation @@ -6209,8 +6373,7 @@ std::string Response::get_trailer_value(const std::string &key, } size_t Response::get_trailer_value_count(const std::string &key) const { - auto r = trailers.equal_range(key); - return static_cast(std::distance(r.first, r.second)); + return trailers.count(key); } void Response::set_redirect(const std::string &url, int stat) { @@ -6306,8 +6469,7 @@ std::string Result::get_request_header_value(const std::string &key, size_t Result::get_request_header_value_count(const std::string &key) const { - auto r = request_headers_.equal_range(key); - return static_cast(std::distance(r.first, r.second)); + return request_headers_.count(key); } // Stream implementation @@ -6595,6 +6757,24 @@ bool SocketStream::wait_writable() const { return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0; } +bool SocketStream::ensure_readable() { + if (readable_hint_) { + readable_hint_ = false; + return true; + } + return wait_readable(); +} + +const char *SocketStream::buffered_data(size_t &size) const { + size = read_buff_content_size_ - read_buff_off_; + return size ? read_buff_.data() + read_buff_off_ : nullptr; +} + +void SocketStream::consume_buffered(size_t size) { + assert(size <= read_buff_content_size_ - read_buff_off_); + read_buff_off_ += size; +} + bool SocketStream::is_peer_alive() const { return detail::is_socket_alive(sock_); } @@ -6621,7 +6801,7 @@ ssize_t SocketStream::read(char *ptr, size_t size) { } } - if (!wait_readable()) { + if (!ensure_readable()) { error_ = Error::Timeout; return -1; } @@ -7099,6 +7279,14 @@ bool SSLSocketStream::wait_writable() const { !tls::is_peer_closed(session_, sock_); } +bool SSLSocketStream::ensure_readable() { + if (readable_hint_) { + readable_hint_ = false; + return true; + } + return wait_readable(); +} + bool SSLSocketStream::is_peer_alive() const { return !tls::is_peer_closed(session_, sock_); } @@ -7111,7 +7299,7 @@ ssize_t SSLSocketStream::read(char *ptr, size_t size) { error_ = Error::ConnectionClosed; } return ret; - } else if (wait_readable()) { + } else if (ensure_readable()) { tls::TlsError err; auto ret = tls::read(session_, ptr, size, err); if (ret < 0) { @@ -7533,9 +7721,11 @@ void Server::wait_until_ready() const { } void Server::stop() noexcept { - if (is_running_) { - assert(svr_sock_ != INVALID_SOCKET); - std::atomic sock(svr_sock_.exchange(INVALID_SOCKET)); + // Release the listening socket whether or not the accept loop is running: + // bind_to_port() without listen_after_bind() still owns the descriptor. The + // exchange is what makes this safe to call concurrently with the accept loop. + socket_t sock = svr_sock_.exchange(INVALID_SOCKET); + if (sock != INVALID_SOCKET) { detail::shutdown_socket(sock); detail::close_socket(sock); } @@ -7697,7 +7887,15 @@ Server::write_content_with_provider(Stream &strm, const Request &req, }; if (res.content_length_ > 0) { - if (req.ranges.empty()) { + // Only a 206 response is served as a partial representation, matching the + // condition `apply_ranges()` used to decide the Content-Length and the + // multipart boundary. Since `detail::range_error()` validates `req.ranges` + // only for a 2xx status, slicing under any other status would write a body + // that disagrees with the header already sent, from an unchecked offset. + auto is_partial = + !req.ranges.empty() && res.status == StatusCode::PartialContent_206; + + if (!is_partial) { return detail::write_content(strm, res.content_provider_, 0, res.content_length_, is_shutting_down); } else if (req.ranges.size() == 1) { @@ -8096,7 +8294,14 @@ int Server::bind_internal(const std::string &host, int port, } bool Server::listen_internal() { - if (is_decommissioned) { return false; } + // A stop() between bind and listen leaves nothing to accept on. Report + // failure instead of returning success without ever serving, and mark the + // server decommissioned the way any failed listen does so that a concurrent + // wait_until_ready() wakes up instead of spinning forever. + if (is_decommissioned || svr_sock_ == INVALID_SOCKET) { + is_decommissioned = true; + return false; + } auto ret = true; is_running_ = true; @@ -8492,11 +8697,17 @@ Server::process_request(Stream &strm, const std::string &remote_addr, return write_response(strm, close_connection, req, res); } - // RFC 9112 §6.3: Reject requests with both a non-zero Content-Length and - // any Transfer-Encoding to prevent request smuggling. Content-Length: 0 is - // tolerated for compatibility with existing clients. - if (req.get_header_value_u64("Content-Length") > 0 && - req.has_header("Transfer-Encoding")) { + // RFC 9112 §6.3: Reject requests whose framing is ambiguous, which would + // otherwise let an intermediary and this parser disagree on where the body + // ends and enable request smuggling. Two cases: a non-zero Content-Length + // alongside any Transfer-Encoding (Content-Length: 0 is tolerated for + // compatibility with existing clients), and a Transfer-Encoding whose final + // coding is not chunked, which leaves the body length undeterminable. The + // latter must not fall through to the "no body" path, or the body bytes are + // parsed as the next request on a persistent connection. + if (req.has_header("Transfer-Encoding") && + (req.get_header_value_u64("Content-Length") > 0 || + !detail::is_chunked_transfer_encoding(req.headers))) { connection_closed = true; res.status = StatusCode::BadRequest_400; return write_response(strm, close_connection, req, res); @@ -8908,13 +9119,13 @@ socket_t ClientImpl::create_client_socket(Error &error) const { write_timeout_sec_, write_timeout_usec_, interface_, error); } - // Check is custom IP specified for host_ + // Check is custom IP or hostname specified for host_ + std::string connect_host; std::string ip; - auto it = addr_map_.find(host_); - if (it != addr_map_.end()) { ip = it->second; } + detail::apply_addr_map(addr_map_, host_, connect_host, ip); return detail::create_client_socket( - host_, ip, port_, address_family_, tcp_nodelay_, ipv6_v6only_, + connect_host, ip, port_, address_family_, tcp_nodelay_, ipv6_v6only_, socket_options_, connection_timeout_sec_, connection_timeout_usec_, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, write_timeout_usec_, interface_, error); @@ -9142,11 +9353,13 @@ void ClientImpl::prepare_default_headers(Request &r, bool for_stream, if (!r.has_header(header.first)) { r.headers.insert(header); } } + // RFC 9110 5.3 recommends sending control data such as Host first, so + // prepend it rather than appending it after the caller's own fields. if (!r.has_header("Host")) { if (address_family_ == AF_UNIX) { - r.headers.emplace("Host", "localhost"); + r.headers.emplace_front("Host", "localhost"); } else { - r.headers.emplace( + r.headers.emplace_front( "Host", detail::make_host_and_port_string(host_, port_, is_ssl())); } } @@ -9197,7 +9410,12 @@ ClientImpl::open_stream(const std::string &method, const std::string &path, handle.response = detail::make_unique(); handle.error = Error::Success; - auto query_path = params.empty() ? path : append_query_params(path, params); + // Encode the target exactly like the buffered send path does, so that the + // same `path` produces the same request line through either API. + auto raw_query_path = + params.empty() ? path : append_query_params(path, params); + auto query_path = detail::encode_request_target(raw_query_path, path_encode_); + handle.connection_ = detail::make_unique(); { @@ -9311,7 +9529,20 @@ ClientImpl::open_stream(const std::string &method, const std::string &path, auto content_encoding = handle.response->get_header_value("Content-Encoding"); if (!content_encoding.empty()) { + // Same policy as prepare_content_receiver(): reject a coding we know about + // but were not built with, pass an unrecognized one through as-is. handle.decompressor_ = detail::create_decompressor(content_encoding); + if (!handle.decompressor_) { + if (detail::is_known_content_encoding(content_encoding)) { + handle.error = Error::UnsupportedContentEncoding; + handle.response.reset(); + return handle; + } + } else if (!handle.decompressor_->is_valid()) { + handle.error = Error::Compression; + handle.response.reset(); + return handle; + } } return handle; @@ -9842,52 +10073,26 @@ bool ClientImpl::write_request(Stream &strm, Request &req, { detail::BufferStream bstrm; - // Extract path and query from req.path - std::string path_part, query_part; + // Extract the query from req.path. The encoding itself is delegated to + // `encode_request_target`; the raw query is still needed here to decide + // between populating `req.params` from it and falling back to building a + // query out of caller-supplied `req.params`. auto query_pos = req.path.find('?'); - if (query_pos != std::string::npos) { - path_part = req.path.substr(0, query_pos); - query_part = req.path.substr(query_pos + 1); - } else { - path_part = req.path; - query_part = ""; - } + auto query_part = query_pos == std::string::npos + ? std::string() + : req.path.substr(query_pos + 1); - // Encode path part. If the original `req.path` already contained a - // query component, preserve its raw query string (including parameter - // order) instead of reparsing and reassembling it which may reorder - // parameters due to container ordering (e.g. `Params` uses - // `std::multimap`). When there is no query in `req.path`, fall back to - // building a query from `req.params` so existing callers that pass - // `Params` continue to work. auto path_with_query = - path_encode_ ? detail::encode_path(path_part) : path_part; + detail::encode_request_target(req.path, path_encode_); if (!query_part.empty()) { - // Normalize the query string (decode then re-encode) while preserving - // the original parameter order. When path encoding is disabled the - // caller has supplied an already-encoded target and expects the exact - // bytes to be sent on the wire, so skip normalization for the query - // too. Normalizing here would decode-then-re-encode the query and - // corrupt pre-encoded binary payloads (e.g. turning `%20` into `+`, - // which a strict RFC 3986 server decodes back as `+`, not a space). - if (path_encode_) { - auto normalized = detail::normalize_query_string(query_part); - if (!normalized.empty()) { path_with_query += '?' + normalized; } - } else { - path_with_query += '?' + query_part; - } - - // Still populate req.params for handlers/users who read them. + // The query already came in through `req.path`; still populate + // `req.params` for handlers/users who read them. detail::parse_query_text(query_part, req.params); - } else { - // No query in path; parse any query_part (empty) and append params - // from `req.params` when present (preserves prior behavior for - // callers who provide Params separately). - detail::parse_query_text(query_part, req.params); - if (!req.params.empty()) { - path_with_query = append_query_params(path_with_query, req.params); - } + } else if (!req.params.empty()) { + // No query in `req.path`; build one from `req.params` so existing + // callers that pass `Params` separately continue to work. + path_with_query = append_query_params(path_with_query, req.params); } // Write request line and headers @@ -10298,14 +10503,26 @@ bool ClientImpl::process_request(Stream &strm, Request &req, } if (res.status != StatusCode::NotModified_304) { - int dummy_status; + auto content_status = 0; auto max_length = (!has_payload_max_length_ && req.content_receiver) ? (std::numeric_limits::max)() : payload_max_length_; - if (!detail::read_content(strm, res, max_length, dummy_status, + if (!detail::read_content(strm, res, max_length, content_status, std::move(progress), std::move(out), decompress_)) { - if (error != Error::Canceled) { error = Error::Read; } + if (error != Error::Canceled) { + // Tell the caller apart from a plain read failure when the body could + // not be decoded because of its Content-Encoding. + switch (content_status) { + case StatusCode::UnsupportedMediaType_415: + error = Error::UnsupportedContentEncoding; + break; + case StatusCode::InternalServerError_500: + error = Error::Compression; + break; + default: error = Error::Read; break; + } + } output_error_log(error, &req); return false; } @@ -16769,18 +16986,42 @@ bool WebSocketClient::create_stream(std::unique_ptr &strm) { return true; } +void WebSocketClient::prepare_default_headers(Request &req) { +#ifdef CPPHTTPLIB_SSL_ENABLED + auto is_ssl = is_ssl_; +#else + auto is_ssl = false; +#endif + + if (!req.has_header("Host")) { + if (address_family_ == AF_UNIX) { + req.headers.emplace("Host", "localhost"); + } else { + req.headers.emplace( + "Host", detail::make_host_and_port_string(host_, port_, is_ssl)); + } + } + +#ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENT + if (!req.has_header("User-Agent")) { + auto agent = std::string("cpp-httplib/") + CPPHTTPLIB_VERSION; + req.set_header("User-Agent", agent); + } +#endif +} + bool WebSocketClient::connect() { if (!is_valid_) { return false; } shutdown_and_close(); - // Check is custom IP specified for host_ + // Check is custom IP or hostname specified for host_ + std::string connect_host; std::string ip; - auto it = addr_map_.find(host_); - if (it != addr_map_.end()) { ip = it->second; } + detail::apply_addr_map(addr_map_, host_, connect_host, ip); Error error; sock_ = detail::create_client_socket( - host_, ip, port_, address_family_, tcp_nodelay_, ipv6_v6only_, + connect_host, ip, port_, address_family_, tcp_nodelay_, ipv6_v6only_, socket_options_, connection_timeout_sec_, connection_timeout_usec_, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_, write_timeout_usec_, interface_, error); @@ -16793,23 +17034,19 @@ bool WebSocketClient::connect() { return false; } -#ifdef CPPHTTPLIB_SSL_ENABLED - auto is_ssl = is_ssl_; -#else - auto is_ssl = false; -#endif + Request req; + req.method = "GET"; + req.path = path_; + req.headers = headers_; + prepare_default_headers(req); std::string selected_subprotocol; - if (!detail::perform_websocket_handshake(*strm, host_, port_, is_ssl, path_, - headers_, selected_subprotocol)) { + if (!detail::perform_websocket_handshake(*strm, req, selected_subprotocol)) { shutdown_and_close(); return false; } subprotocol_ = std::move(selected_subprotocol); - Request req; - req.method = "GET"; - req.path = path_; ws_ = std::unique_ptr(new WebSocket(std::move(strm), req, false, websocket_ping_interval_sec_, websocket_max_missed_pongs_)); diff --git a/vendor/cpp-httplib/httplib.h b/vendor/cpp-httplib/httplib.h index 6df357d706c9..94605defa31f 100644 --- a/vendor/cpp-httplib/httplib.h +++ b/vendor/cpp-httplib/httplib.h @@ -8,8 +8,8 @@ #ifndef CPPHTTPLIB_HTTPLIB_H #define CPPHTTPLIB_HTTPLIB_H -#define CPPHTTPLIB_VERSION "0.51.0" -#define CPPHTTPLIB_VERSION_NUM "0x003300" +#define CPPHTTPLIB_VERSION "0.52.0" +#define CPPHTTPLIB_VERSION_NUM "0x003400" #ifdef _WIN32 #if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00 @@ -182,7 +182,7 @@ #endif #ifndef CPPHTTPLIB_LISTEN_BACKLOG -#define CPPHTTPLIB_LISTEN_BACKLOG 5 +#define CPPHTTPLIB_LISTEN_BACKLOG 128 #endif #ifndef CPPHTTPLIB_MAX_LINE_LENGTH @@ -321,6 +321,7 @@ using socket_t = int; #include #include #include +#include #include #include #include @@ -333,9 +334,11 @@ using socket_t = int; #include #include #include +#include #include #include #include +#include // On macOS with a TLS backend, enable Keychain root certificates by default // unless the user explicitly opts out. Not enabled on iOS/tvOS/watchOS since @@ -968,11 +971,291 @@ enum StatusCode { NetworkAuthenticationRequired_511 = 511, }; +namespace detail { + +// A multimap that keeps its entries in the order they were inserted. +// +// HTTP needs that order in two places. RFC 9110 5.3 makes the order of header +// fields sharing a field name significant and forbids a proxy from reordering +// them, and a query string's parameters are meaningful in the order the caller +// wrote them. Neither standard container expresses it: std::unordered_multimap +// gives no ordering guarantee at all for equivalent keys (libstdc++ yields +// reverse insertion order, libc++ insertion order), and std::multimap sorts by +// key, which would drop control data such as Host behind whatever else the +// message carries and alphabetise a query string. +// +// Entries are therefore kept in a flat vector, in order. Lookup is a linear +// scan, which beats hashing for the handful of entries a message carries +// (headers are capped at CPPHTTPLIB_HEADER_MAX_COUNT). +// +// KeyEqual compares keys; it is what makes Headers case-insensitive and +// Params, whose parameter names are case-sensitive, not. +template class insertion_ordered_multimap { +public: + using key_type = std::string; + using mapped_type = Mapped; + using value_type = std::pair; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using reference = value_type &; + using const_reference = const value_type &; + +private: + static size_type npos() { return static_cast(-1); } + + static bool keys_equal(const std::string &a, const std::string &b) { + return KeyEqual()(a, b); + } + + // Iterating yields every entry in insertion order, but equal_range() and + // find() have to walk only the entries sharing one key, which are not + // adjacent. Both are the same iterator type: key_idx_ selects between the + // two traversals, and since equality compares only the position, an iterator + // restricted to one key still compares equal to end(). + template class iterator_t { + public: + using iterator_category = std::bidirectional_iterator_tag; + using value_type = insertion_ordered_multimap::value_type; + using difference_type = insertion_ordered_multimap::difference_type; + using pointer = V *; + using reference = V &; + + iterator_t() : data_(nullptr), idx_(0), size_(0), key_idx_(npos()) {} + + template ::value, + int>::type = 0> + iterator_t(const iterator_t &rhs) + : data_(rhs.data_), idx_(rhs.idx_), size_(rhs.size_), + key_idx_(rhs.key_idx_) {} + + reference operator*() const { return data_[idx_]; } + pointer operator->() const { return data_ + idx_; } + + iterator_t &operator++() { + // Saturating, so that advancing past the last entry of a key (which + // get_multimap_value() does when asked for an out-of-range id) stays at + // end() instead of running off the container. + if (idx_ >= size_) { return *this; } + ++idx_; + if (key_idx_ != npos()) { + while (idx_ < size_ && !matches(idx_)) { + ++idx_; + } + } + return *this; + } + + iterator_t operator++(int) { + auto tmp = *this; + ++*this; + return tmp; + } + + iterator_t &operator--() { + if (idx_ == 0) { return *this; } + --idx_; + if (key_idx_ != npos()) { + while (idx_ > 0 && !matches(idx_)) { + --idx_; + } + } + return *this; + } + + iterator_t operator--(int) { + auto tmp = *this; + --*this; + return tmp; + } + + template bool operator==(const iterator_t &rhs) const { + return idx_ == rhs.idx_; + } + + template bool operator!=(const iterator_t &rhs) const { + return idx_ != rhs.idx_; + } + + private: + friend class insertion_ordered_multimap; + template friend class iterator_t; + + iterator_t(V *data, size_type idx, size_type size, size_type key_idx) + : data_(data), idx_(idx), size_(size), key_idx_(key_idx) {} + + bool matches(size_type i) const { + return keys_equal(data_[i].first, data_[key_idx_].first); + } + + V *data_; + size_type idx_; + size_type size_; + size_type key_idx_; + }; + +public: + using iterator = iterator_t; + using const_iterator = iterator_t; + + insertion_ordered_multimap() = default; + insertion_ordered_multimap(std::initializer_list il) + : entries_(il) {} + template + insertion_ordered_multimap(InputIt first, InputIt last) + : entries_(first, last) {} + + iterator begin() { return make_iter(0, npos()); } + iterator end() { return make_iter(entries_.size(), npos()); } + const_iterator begin() const { return make_citer(0, npos()); } + const_iterator end() const { return make_citer(entries_.size(), npos()); } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + + bool empty() const { return entries_.empty(); } + size_type size() const { return entries_.size(); } + void clear() { entries_.clear(); } + void swap(insertion_ordered_multimap &rhs) { entries_.swap(rhs.entries_); } + + iterator insert(const value_type &val) { + entries_.push_back(val); + return make_iter(entries_.size() - 1, npos()); + } + + iterator insert(value_type &&val) { + entries_.push_back(std::move(val)); + return make_iter(entries_.size() - 1, npos()); + } + + template iterator emplace(Args &&...args) { + entries_.emplace_back(std::forward(args)...); + return make_iter(entries_.size() - 1, npos()); + } + + // For entries that have to lead the message, such as the Host header field + // (RFC 9110 5.3 recommends sending control data first). + template iterator emplace_front(Args &&...args) { + entries_.emplace(entries_.begin(), std::forward(args)...); + return make_iter(0, npos()); + } + + iterator find(const std::string &key) { + auto i = index_of(key); + return i == npos() ? end() : make_iter(i, i); + } + + const_iterator find(const std::string &key) const { + auto i = index_of(key); + return i == npos() ? end() : make_citer(i, i); + } + + size_type count(const std::string &key) const { + size_type n = 0; + for (const auto &entry : entries_) { + if (keys_equal(entry.first, key)) { n++; } + } + return n; + } + + std::pair equal_range(const std::string &key) { + auto i = index_of(key); + return i == npos() ? std::make_pair(end(), end()) + : std::make_pair(make_iter(i, i), end()); + } + + std::pair + equal_range(const std::string &key) const { + auto i = index_of(key); + return i == npos() ? std::make_pair(end(), end()) + : std::make_pair(make_citer(i, i), end()); + } + + size_type erase(const std::string &key) { + auto before = entries_.size(); + entries_.erase(std::remove_if(entries_.begin(), entries_.end(), + [&](const value_type &entry) { + return keys_equal(entry.first, key); + }), + entries_.end()); + return before - entries_.size(); + } + + iterator erase(const_iterator pos) { + entries_.erase(entries_.begin() + static_cast(pos.idx_)); + return make_iter(pos.idx_, npos()); + } + + // Erases what iterating [first, last) would actually visit, so erasing an + // equal_range() removes only the entries with that key, not everything + // positioned between them. + iterator erase(const_iterator first, const_iterator last) { + auto from = first.idx_; + auto to = last.idx_; + if (from >= to) { return make_iter(from, npos()); } + + auto begin_it = entries_.begin(); + auto from_it = begin_it + static_cast(from); + auto to_it = begin_it + static_cast(to); + + if (first.key_idx_ == npos()) { + entries_.erase(from_it, to_it); + } else { + auto key = entries_[first.key_idx_].first; + auto keep = from_it; + for (auto it = from_it; it != to_it; ++it) { + if (!keys_equal(it->first, key)) { + if (keep != it) { *keep = std::move(*it); } + ++keep; + } + } + if (keep != to_it) { + keep = std::move(to_it, entries_.end(), keep); + } else { + keep = entries_.end(); + } + entries_.erase(keep, entries_.end()); + } + return make_iter(from, npos()); + } + + friend bool operator==(const insertion_ordered_multimap &lhs, + const insertion_ordered_multimap &rhs) { + return lhs.entries_ == rhs.entries_; + } + + friend bool operator!=(const insertion_ordered_multimap &lhs, + const insertion_ordered_multimap &rhs) { + return !(lhs == rhs); + } + +private: + size_type index_of(const std::string &key) const { + for (size_type i = 0; i < entries_.size(); i++) { + if (keys_equal(entries_[i].first, key)) { return i; } + } + return npos(); + } + + iterator make_iter(size_type idx, size_type key_idx) { + return iterator(entries_.data(), idx, entries_.size(), key_idx); + } + + const_iterator make_citer(size_type idx, size_type key_idx) const { + return const_iterator(entries_.data(), idx, entries_.size(), key_idx); + } + + std::vector entries_; +}; + +} // namespace detail + using Headers = - std::unordered_multimap; + detail::insertion_ordered_multimap; -using Params = std::multimap; +// Query parameter names are case-sensitive, unlike header field names. +using Params = + detail::insertion_ordered_multimap>; using Match = std::smatch; using DownloadProgress = std::function; @@ -1079,9 +1362,16 @@ struct FormField { std::string content; Headers headers; }; -using FormFields = std::multimap; +// RFC 7578 5.2: a form processor "SHOULD send back results in order" and +// "Intermediaries MUST NOT reorder the results", so a handler walking these +// should see the parts as they were sent. A std::multimap sorts by field name +// and loses that. Field names are case-sensitive, hence std::equal_to rather +// than the case-insensitive predicate Headers uses. +using FormFields = + detail::insertion_ordered_multimap>; -using FormFiles = std::multimap; +using FormFiles = + detail::insertion_ordered_multimap>; struct MultipartFormData { FormFields fields; // Text fields from multipart @@ -1514,6 +1804,7 @@ enum class Error { UnsupportedAddressFamily, HTTPParsing, InvalidRangeHeader, + UnsupportedContentEncoding, // For internal use only SSLPeerCouldBeClosed_, @@ -1545,6 +1836,18 @@ class Stream { (void)usec; } + // Bytes already pulled off the socket and sitting in this stream's own + // buffer. Exposing them lets a line reader scan for a terminator in one + // pass instead of asking for a byte at a time. A stream that does no + // buffering of its own reports none, and readers fall back to read(). + virtual const char *buffered_data(size_t &size) const { + size = 0; + return nullptr; + } + + // Discards `size` bytes previously returned by buffered_data(). + virtual void consume_buffered(size_t size) { (void)size; } + ssize_t write(const char *ptr); ssize_t write(const std::string &s); @@ -2452,7 +2755,8 @@ class ClientImpl { std::thread::id socket_requests_are_from_thread_ = std::thread::id(); bool socket_should_be_closed_when_request_is_done_ = false; - // Hostname-IP map + // Hostname to connection target map. The value is an IP literal or another + // hostname; only the connection target changes, never the identity. std::map addr_map_; // Default headers @@ -3154,6 +3458,10 @@ struct FileStat { std::string make_host_and_port_string(const std::string &host, int port, bool is_ssl); +template +bool check_and_write_headers(Stream &strm, Headers &headers, T header_writer, + Error &error); + std::string trim_copy(const std::string &s); void divide( @@ -3364,6 +3672,7 @@ class stream_line_reader { private: void append(char c); + void append(const char *data, size_t size); Stream &strm_; char *fixed_buffer_; @@ -3992,6 +4301,7 @@ class WebSocketClient { private: void shutdown_and_close(); bool create_stream(std::unique_ptr &strm); + void prepare_default_headers(Request &req); std::string host_; int port_; @@ -4016,7 +4326,8 @@ class WebSocketClient { time_t connection_timeout_usec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND; std::string interface_; - // Hostname-IP map + // Hostname to connection target map. The value is an IP literal or another + // hostname; only the connection target changes, never the identity. std::map addr_map_; #ifdef CPPHTTPLIB_SSL_ENABLED From 0ef6e55edb306fcbcf73e6f1f41923cccb9cf7f8 Mon Sep 17 00:00:00 2001 From: Thiago Padilha Date: Mon, 3 Aug 2026 19:59:11 -0300 Subject: [PATCH 50/98] chat : add new template for DeepSeek V4 Flash 0731 (#26398) * common/chat: update DeepSeek V4 templates Align the DeepSeek V4 templates with the official encoders while keeping parser behavior out of this change. - Default drop_thinking for DeepSeek V4 history so prior thinking is omitted unless preserve_reasoning is requested or tools are present. - Add structured output response-format instructions to the V4 templates and pass the schema into template rendering. - Add a separate Flash 0731 template for the updated high and max reasoning effort mapping. - Cover reasoning effort, drop_thinking, structured output prompts, preserved reasoning, continuations, and empty tool arguments in template rendering tests. Official references: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/encoding/encoding_dsv4.py https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731/blob/main/encoding/encoding_dsv4.py Assisted-by: Codex * Fix deepseek v4 0731 template selection * remove unneeded lower normalization * Fix DSML parser to consume the tool call separator * address aldehir requests * address aldehir comment --- common/chat.cpp | 33 +++- common/jinja/caps.cpp | 1 + conversion/deepseek.py | 5 +- .../deepseek-ai-DeepSeek-V4-Flash-0731.jinja | 140 ++++++++++++++ .../templates/deepseek-ai-DeepSeek-V4.jinja | 24 ++- tests/test-chat.cpp | 172 ++++++++++++++++++ 6 files changed, 363 insertions(+), 12 deletions(-) create mode 100644 models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja diff --git a/common/chat.cpp b/common/chat.cpp index f7431a6f13f0..d2ff2a1be2d0 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -2114,6 +2114,11 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + std::optional additional_context; + if (is_v4 && has_response_format) { + additional_context = json{ { "response_format", inputs.json_schema } }; + } + const std::string DSML = "｜DSML｜"; const std::string THINK_START = ""; const std::string THINK_END = ""; @@ -2125,9 +2130,12 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha const std::string PARAM_START = "<" + DSML + "parameter"; const std::string PARAM_END = ""; const std::string GEN_PROMPT = "<｜Assistant｜>"; + const std::string TC_SEPARATOR = "\n\n"; - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages); + data.prompt = common_chat_template_direct_apply_impl( + tmpl, inputs, adjusted_messages, std::nullopt, additional_context); + data.generation_prompt = common_chat_template_generation_prompt_impl( + tmpl, inputs, adjusted_messages, std::nullopt, additional_context); data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; data.supports_thinking = true; data.thinking_start_tag = THINK_START; @@ -2141,9 +2149,16 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha if (inputs.has_continuation()) { const auto & msg = inputs.continue_msg; - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); + if (is_v4 && msg.reasoning_content.empty()) { + data.generation_prompt = GEN_PROMPT + THINK_END; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += msg.render_content(); + } + } else { + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } } data.prompt += data.generation_prompt; @@ -2242,7 +2257,9 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha if (extract_reasoning && inputs.enable_thinking) { reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); - reasoning_with_tc = THINK_START + p.reasoning(p.until_one_of({ FC_START, THINK_END })) + obligatory_tool_calls; + reasoning_with_tc = THINK_START + + p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) + + p.space() + obligatory_tool_calls; allow_reasoning_with_tc = true; } else if (extract_reasoning) { // Thinking disabled but reasoning extraction requested: the generation prompt @@ -2265,7 +2282,9 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha return generation_prompt + reasoning + p.content(p.rest()) + end; } - auto content_before_tools = p.negate(p.literal(THINK_START)) + p.content(p.until(FC_START)); + auto content_before_tools = p.negate(p.literal(THINK_START)) + + p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) + + p.space(); return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end : generation_prompt + reasoning + content_before_tools + tool_calls + end; }); diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index 2b9e27ba1c3a..cdd7ccfa26ee 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -482,6 +482,7 @@ caps caps_get(jinja::program & prog) { }); }, [&](context & ctx) { + ctx.set_val("enable_thinking", mk_val(true)); caps_apply_preserve_reasoning(ctx, true); }, nullptr, // tools_fn diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 1c9b325d5e6a..0518fcecc130 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -535,7 +535,10 @@ def __init__(self, *args, **kwargs): logger.info("Skipping %d DeepSeek-V4 MTP tensor(s) for conversion v0", type(self)._skipped_mtp_tensors) # add a default chat template; if the model has a built-in template, it will be overridden later - template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.jinja" + model_id_hint = self.remote_hf_model_id or self.dir_model.name + is_0731 = "0731" in model_id_hint + template_name = "deepseek-ai-DeepSeek-V4-Flash-0731.jinja" if is_0731 else "deepseek-ai-DeepSeek-V4.jinja" + template_path = Path(__file__).parent.parent / "models" / "templates" / template_name if template_path.is_file(): with open(template_path, "r", encoding="utf-8") as f: self.gguf_writer.add_chat_template(f.read()) diff --git a/models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja b/models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja new file mode 100644 index 000000000000..ab689a57c9cf --- /dev/null +++ b/models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja @@ -0,0 +1,140 @@ +{%- if not add_generation_prompt is defined -%} + {%- set add_generation_prompt = false -%} +{%- endif -%} +{%- if not thinking is defined -%} + {%- if enable_thinking is defined -%} + {%- set thinking = enable_thinking -%} + {%- else -%} + {%- set thinking = false -%} + {%- endif -%} +{%- endif -%} +{%- if not drop_thinking is defined -%} + {%- set drop_thinking = true -%} +{%- endif -%} +{%- set dsml_token = '｜DSML｜' -%} +{%- set thinking_start_token = '' -%} +{%- set thinking_end_token = '' -%} +{%- set reasoning_effort_high = 'Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n' -%} +{%- set reasoning_effort_max = 'Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\nYou MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\nDo not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n' -%} +{%- set response_format_template = '## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n' -%} +{%- set has_tools = false -%} +{%- set tools_header = '## Tools\n\nYou have access to a set of tools to help answer the user\'s question. You can invoke tools by writing a "<' + dsml_token + 'tool_calls>" block like the following:\n\n<' + dsml_token + 'tool_calls>\n<' + dsml_token + 'invoke name="$TOOL_NAME">\n<' + dsml_token + 'parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE\n...\n\n<' + dsml_token + 'invoke name="$TOOL_NAME2">\n...\n\n\n\nString parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.\n\nIf thinking_mode is enabled (triggered by ' + thinking_start_token + '), you MUST output your complete reasoning inside ' + thinking_start_token + '...' + thinking_end_token + ' BEFORE any tool calls or final response.\n\nOtherwise, output directly after ' + thinking_end_token + ' with tool calls or final response.\n\n### Available Tool Schemas\n\n' -%} +{%- set tools_footer = '\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n' -%} +{%- set ns = namespace(system_prompt='', is_first_sp=true, has_tool_calls=false) -%} +{%- for message in messages -%} + {%- if message['role'] == 'system' -%} + {%- if ns.is_first_sp -%} + {%- set ns.system_prompt = ns.system_prompt + (message['content'] or '') -%} + {%- set ns.is_first_sp = false -%} + {%- else -%} + {%- set ns.system_prompt = ns.system_prompt + '\n\n' + (message['content'] or '') -%} + {%- endif -%} + {%- endif -%} +{%- endfor -%} +{%- if tools is defined and tools -%} + {%- set has_tools = true -%} + {%- set ts = namespace(schemas='') -%} + {%- for tool in tools -%} + {%- if tool['type'] == 'function' -%} + {%- set ts.schemas = ts.schemas + (tool['function'] | tojson) + '\n' -%} + {%- endif -%} + {%- endfor -%} + {%- if ns.system_prompt -%} + {%- set ns.system_prompt = ns.system_prompt + '\n\n' + tools_header + ts.schemas + tools_footer -%} + {%- else -%} + {%- set ns.system_prompt = tools_header + ts.schemas + tools_footer -%} + {%- endif -%} +{%- endif -%} +{%- if response_format is defined -%} + {%- if ns.system_prompt -%} + {%- set ns.system_prompt = ns.system_prompt + '\n\n' -%} + {%- endif -%} + {%- set ns.system_prompt = ns.system_prompt + response_format_template + (response_format | tojson) -%} +{%- endif -%} +{{- bos_token -}} +{%- if messages and thinking and reasoning_effort is defined and reasoning_effort == 'high' -%} + {{- reasoning_effort_high -}} +{%- elif messages and thinking and reasoning_effort is defined and reasoning_effort == 'max' -%} + {{- reasoning_effort_max -}} +{%- endif -%} +{{- ns.system_prompt -}} +{%- set last_user_idx = namespace(value=-1) -%} +{%- for message in messages -%} + {%- if message['role'] == 'user' or message['role'] == 'developer' or message['role'] == 'tool' -%} + {%- set last_user_idx.value = loop.index0 -%} + {%- endif -%} +{%- endfor -%} +{%- set state = namespace(in_user=false) -%} +{%- for message in messages -%} + {%- if message['role'] == 'tool' -%} + {%- set ns.has_tool_calls = true -%} + {%- endif -%} +{%- endfor -%} +{%- for message in messages -%} + {%- if message['role'] == 'user' or message['role'] == 'developer' -%} + {%- if state.in_user -%} + {{- '\n\n' -}} + {%- else -%} + {{- '<｜User｜>' -}} + {%- set state.in_user = true -%} + {%- endif -%} + {{- message['content'] or '' -}} + {%- elif message['role'] == 'tool' -%} + {%- if state.in_user -%} + {{- '\n\n' -}} + {%- else -%} + {{- '<｜User｜>' -}} + {%- set state.in_user = true -%} + {%- endif -%} + {{- '' + (message['content'] or '') + '' -}} + {%- elif message['role'] == 'assistant' -%} + {%- set state.in_user = false -%} + {{- '<｜Assistant｜>' -}} + {%- set is_after_last_user = loop.index0 > last_user_idx.value -%} + {%- set keep_reasoning = thinking and ((not drop_thinking) or has_tools or is_after_last_user or ns.has_tool_calls) -%} + {%- if keep_reasoning -%} + {{- thinking_start_token -}} + {%- if message['reasoning_content'] is defined and message['reasoning_content'] -%} + {{- message['reasoning_content'] -}} + {%- endif -%} + {{- thinking_end_token -}} + {%- else -%} + {{- thinking_end_token -}} + {%- endif -%} + {%- if message['content'] is defined and message['content'] -%} + {{- message['content'] -}} + {%- endif -%} + {%- if message['tool_calls'] -%} + {{- '\n\n<' + dsml_token + 'tool_calls>\n' -}} + {%- for tool in message['tool_calls'] -%} + {%- set func = tool['function'] -%} + {{- '<' + dsml_token + 'invoke name="' + func['name'] + '">\n' -}} + {%- set args = func['arguments'] -%} + {%- if args is string -%} + {%- set args = args | from_json -%} + {%- endif -%} + {%- for key, val in args.items() -%} + {%- if val is string -%} + {{- '<' + dsml_token + 'parameter name="' + key + '" string="true">' + val + '\n' -}} + {%- else -%} + {{- '<' + dsml_token + 'parameter name="' + key + '" string="false">' + (val | tojson) + '\n' -}} + {%- endif -%} + {%- endfor -%} + {%- if not args -%} + {{- '\n' -}} + {%- endif -%} + {{- '\n' -}} + {%- endfor -%} + {{- '' -}} + {%- endif -%} + {{- '<｜end▁of▁sentence｜>' -}} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{- '<｜Assistant｜>' -}} + {%- if thinking -%} + {{- thinking_start_token -}} + {%- else -%} + {{- thinking_end_token -}} + {%- endif -%} +{%- endif -%} diff --git a/models/templates/deepseek-ai-DeepSeek-V4.jinja b/models/templates/deepseek-ai-DeepSeek-V4.jinja index ad1ff8ce23aa..c7819149ae6e 100644 --- a/models/templates/deepseek-ai-DeepSeek-V4.jinja +++ b/models/templates/deepseek-ai-DeepSeek-V4.jinja @@ -9,11 +9,14 @@ {%- endif -%} {%- endif -%} {%- if not drop_thinking is defined -%} - {%- set drop_thinking = false -%} + {%- set drop_thinking = true -%} {%- endif -%} {%- set dsml_token = '｜DSML｜' -%} {%- set thinking_start_token = '' -%} {%- set thinking_end_token = '' -%} +{%- set reasoning_effort_max = 'Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n' -%} +{%- set response_format_template = '## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n' -%} +{%- set has_tools = false -%} {%- set tools_header = '## Tools\n\nYou have access to a set of tools to help answer the user\'s question. You can invoke tools by writing a "<' + dsml_token + 'tool_calls>" block like the following:\n\n<' + dsml_token + 'tool_calls>\n<' + dsml_token + 'invoke name="$TOOL_NAME">\n<' + dsml_token + 'parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE\n...\n\n<' + dsml_token + 'invoke name="$TOOL_NAME2">\n...\n\n\n\nString parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.\n\nIf thinking_mode is enabled (triggered by ' + thinking_start_token + '), you MUST output your complete reasoning inside ' + thinking_start_token + '...' + thinking_end_token + ' BEFORE any tool calls or final response.\n\nOtherwise, output directly after ' + thinking_end_token + ' with tool calls or final response.\n\n### Available Tool Schemas\n\n' -%} {%- set tools_footer = '\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n' -%} {%- set ns = namespace(system_prompt='', is_first_sp=true, has_tool_calls=false) -%} @@ -28,6 +31,7 @@ {%- endif -%} {%- endfor -%} {%- if tools is defined and tools -%} + {%- set has_tools = true -%} {%- set ts = namespace(schemas='') -%} {%- for tool in tools -%} {%- if tool['type'] == 'function' -%} @@ -40,7 +44,16 @@ {%- set ns.system_prompt = tools_header + ts.schemas + tools_footer -%} {%- endif -%} {%- endif -%} +{%- if response_format is defined -%} + {%- if ns.system_prompt -%} + {%- set ns.system_prompt = ns.system_prompt + '\n\n' -%} + {%- endif -%} + {%- set ns.system_prompt = ns.system_prompt + response_format_template + (response_format | tojson) -%} +{%- endif -%} {{- bos_token -}} +{%- if messages and thinking and reasoning_effort is defined and reasoning_effort == 'max' -%} + {{- reasoning_effort_max -}} +{%- endif -%} {{- ns.system_prompt -}} {%- set last_user_idx = namespace(value=-1) -%} {%- for message in messages -%} @@ -75,8 +88,8 @@ {%- set state.in_user = false -%} {{- '<｜Assistant｜>' -}} {%- set is_after_last_user = loop.index0 > last_user_idx.value -%} - {%- set retain_reasoning = (not drop_thinking) or (is_after_last_user or ns.has_tool_calls) -%} - {%- if retain_reasoning and thinking -%} + {%- set keep_reasoning = thinking and ((not drop_thinking) or has_tools or is_after_last_user or ns.has_tool_calls) -%} + {%- if keep_reasoning -%} {{- thinking_start_token -}} {%- if message['reasoning_content'] is defined and message['reasoning_content'] -%} {{- message['reasoning_content'] -}} @@ -104,6 +117,9 @@ {{- '<' + dsml_token + 'parameter name="' + key + '" string="false">' + (val | tojson) + '\n' -}} {%- endif -%} {%- endfor -%} + {%- if not args -%} + {{- '\n' -}} + {%- endif -%} {{- '\n' -}} {%- endfor -%} {{- '' -}} @@ -118,4 +134,4 @@ {%- else -%} {{- thinking_end_token -}} {%- endif -%} -{%- endif -%} \ No newline at end of file +{%- endif -%} diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 8d571d369e55..f54a58f9b67c 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -3987,6 +3987,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_tool_calls({ { "special_function", R"({"arg1": 1})", {} }, }) + .expect_reconstruction() .run(); // Tool call with negative number @@ -4212,6 +4213,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_tool_calls({ { "special_function", R"({"arg1": 1})", {} }, }) + .expect_reconstruction() .run(); // Tool call with multiple params (mixed types) @@ -4268,6 +4270,24 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .run(); } + { + // The DSML separator belongs to the tool call block, not assistant content. + auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja", detailed_debug); + tst.test( + "\n\n" + "<｜DSML｜tool_calls>\n" + "<｜DSML｜invoke name=\"special_function\">\n" + "<｜DSML｜parameter name=\"arg1\" string=\"false\">1\n" + "\n" + "") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) + .tools({ special_function_tool }) + .expect(message_assist_call) + .expect_reconstruction() + .run(); + } + // GLM-4.6 tests - format: function_name\n...\n...\n { auto tst = peg_tester("models/templates/GLM-4.6.jinja", detailed_debug); @@ -6359,6 +6379,7 @@ static void test_template_generation_prompt() { std::vector messages; bool add_generation_prompt = true; common_chat_continuation continue_final_message = COMMON_CHAT_CONTINUATION_NONE; + bool enable_thinking = true; }; auto basic = [&]() { @@ -6390,6 +6411,7 @@ static void test_template_generation_prompt() { inputs.messages = opts.messages; inputs.add_generation_prompt = opts.add_generation_prompt; inputs.continue_final_message = opts.continue_final_message; + inputs.enable_thinking = opts.enable_thinking; auto params = common_chat_templates_apply(tmpls.get(), inputs); @@ -6488,6 +6510,156 @@ static void test_template_generation_prompt() { check(tmpls, continuation_reasoning(), "<｜Assistant｜>I'm"); } + const std::string deepseek_v4_reasoning_effort_max = "Reasoning Effort: Absolute maximum"; + const std::string deepseek_v4_flash_0731_reasoning_effort_max = "Reasoning Effort: Beyond maximum"; + + { + auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4.jinja"); + check(tmpls, basic(), "<｜Assistant｜>"); + check(tmpls, continuation_content(), "<｜Assistant｜>I'm thinkingHello, "); + check(tmpls, continuation_reasoning(), "<｜Assistant｜>I'm"); + + auto continuation_content_no_thinking = continuation_content(); + continuation_content_no_thinking.messages = { system_msg, message_user, simple_assist_msg("Hello, ") }; + continuation_content_no_thinking.enable_thinking = false; + check(tmpls, continuation_content_no_thinking, "<｜Assistant｜>Hello, "); + + common_chat_templates_inputs max_inputs; + max_inputs.messages = { system_msg, message_user }; + max_inputs.chat_template_kwargs["reasoning_effort"] = R"("max")"; + auto max_params = common_chat_templates_apply(tmpls.get(), max_inputs); + assert_contains(max_params.prompt, deepseek_v4_reasoning_effort_max); + + auto high_inputs = max_inputs; + high_inputs.chat_template_kwargs["reasoning_effort"] = R"("high")"; + auto high_params = common_chat_templates_apply(tmpls.get(), high_inputs); + assert_not_contains(high_params.prompt, deepseek_v4_reasoning_effort_max); + + auto low_inputs = max_inputs; + low_inputs.chat_template_kwargs["reasoning_effort"] = R"("low")"; + auto low_params = common_chat_templates_apply(tmpls.get(), low_inputs); + assert_not_contains(low_params.prompt, deepseek_v4_reasoning_effort_max); + + common_chat_templates_inputs default_effort_inputs; + default_effort_inputs.messages = { system_msg, message_user }; + auto default_effort_params = common_chat_templates_apply(tmpls.get(), default_effort_inputs); + assert_not_contains(default_effort_params.prompt, deepseek_v4_reasoning_effort_max); + + auto non_thinking_max_inputs = max_inputs; + non_thinking_max_inputs.enable_thinking = false; + auto non_thinking_max_params = common_chat_templates_apply(tmpls.get(), non_thinking_max_inputs); + assert_not_contains(non_thinking_max_params.prompt, deepseek_v4_reasoning_effort_max); + + common_chat_templates_inputs response_format_inputs; + response_format_inputs.messages = { system_msg, message_user }; + response_format_inputs.tools = { get_time_tool }; + response_format_inputs.json_schema = + R"({"type":"object","properties":{"answer":{"type":"string"}}})"; + auto response_format_params = common_chat_templates_apply(tmpls.get(), response_format_inputs); + const auto tools_pos = response_format_params.prompt.find("## Tools"); + const auto response_format_pos = response_format_params.prompt.find( + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n"); + if (tools_pos == std::string::npos || response_format_pos == std::string::npos || tools_pos > response_format_pos) { + LOG_ERR("Expected response format after tools\nActual: %s\n", response_format_params.prompt.c_str()); + common_log_flush(common_log_main()); + throw std::runtime_error("Test failed"); + } + assert_contains(response_format_params.prompt, R"("answer": {"type": "string"})"); + + response_format_inputs.json_schema = "{}"; + auto json_object_params = common_chat_templates_apply(tmpls.get(), response_format_inputs); + assert_contains(json_object_params.prompt, + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{}"); + + common_chat_msg assistant_history; + assistant_history.role = "assistant"; + assistant_history.content = "Previous answer"; + assistant_history.reasoning_content = "Previous reasoning"; + + common_chat_msg user_followup; + user_followup.role = "user"; + user_followup.content = "Follow up"; + + common_chat_templates_inputs default_history_inputs; + default_history_inputs.messages = { message_user, assistant_history, user_followup }; + auto default_history_params = common_chat_templates_apply(tmpls.get(), default_history_inputs); + assert_contains(default_history_params.prompt, "<｜Assistant｜>Previous answer"); + + auto drop_thinking_inputs = default_history_inputs; + drop_thinking_inputs.chat_template_kwargs["drop_thinking"] = "false"; + auto drop_thinking_params = common_chat_templates_apply(tmpls.get(), drop_thinking_inputs); + assert_contains(drop_thinking_params.prompt, "<｜Assistant｜>Previous reasoningPrevious answer"); + + auto preserve_reasoning_inputs = default_history_inputs; + preserve_reasoning_inputs.chat_template_kwargs["preserve_reasoning"] = "true"; + auto preserve_reasoning_params = common_chat_templates_apply(tmpls.get(), preserve_reasoning_inputs); + assert_contains(preserve_reasoning_params.prompt, "<｜Assistant｜>Previous reasoningPrevious answer"); + assert_equals(true, common_chat_templates_get_caps(tmpls.get()).at("supports_preserve_reasoning")); + + auto no_preserve_reasoning_inputs = default_history_inputs; + no_preserve_reasoning_inputs.chat_template_kwargs["preserve_reasoning"] = "false"; + auto no_preserve_reasoning_params = common_chat_templates_apply(tmpls.get(), no_preserve_reasoning_inputs); + assert_contains(no_preserve_reasoning_params.prompt, "<｜Assistant｜>Previous answer"); + + common_chat_msg empty_tool_call = simple_assist_msg("", "", "empty_args", "{}"); + common_chat_templates_inputs empty_tool_inputs; + empty_tool_inputs.messages = { message_user, empty_tool_call }; + empty_tool_inputs.tools = { empty_args_tool }; + auto empty_tool_params = common_chat_templates_apply(tmpls.get(), empty_tool_inputs); + assert_contains(empty_tool_params.prompt, + "<｜DSML｜invoke name=\"empty_args\">\n\n"); + } + + { + auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja"); + check(tmpls, basic(), "<｜Assistant｜>"); + check(tmpls, continuation_content(), "<｜Assistant｜>I'm thinkingHello, "); + check(tmpls, continuation_reasoning(), "<｜Assistant｜>I'm"); + + auto continuation_content_no_thinking = continuation_content(); + continuation_content_no_thinking.messages = { system_msg, message_user, simple_assist_msg("Hello, ") }; + continuation_content_no_thinking.enable_thinking = false; + check(tmpls, continuation_content_no_thinking, "<｜Assistant｜>Hello, "); + + common_chat_templates_inputs high_inputs; + high_inputs.messages = { system_msg, message_user }; + high_inputs.chat_template_kwargs["reasoning_effort"] = R"("high")"; + auto high_params = common_chat_templates_apply(tmpls.get(), high_inputs); + assert_contains(high_params.prompt, deepseek_v4_reasoning_effort_max); + + auto max_inputs = high_inputs; + max_inputs.chat_template_kwargs["reasoning_effort"] = R"("max")"; + auto max_params = common_chat_templates_apply(tmpls.get(), max_inputs); + assert_contains(max_params.prompt, deepseek_v4_flash_0731_reasoning_effort_max); + + auto low_inputs = high_inputs; + low_inputs.chat_template_kwargs["reasoning_effort"] = R"("low")"; + auto low_params = common_chat_templates_apply(tmpls.get(), low_inputs); + assert_not_contains(low_params.prompt, deepseek_v4_reasoning_effort_max); + assert_not_contains(low_params.prompt, deepseek_v4_flash_0731_reasoning_effort_max); + + common_chat_templates_inputs default_effort_inputs; + default_effort_inputs.messages = { system_msg, message_user }; + auto default_effort_params = common_chat_templates_apply(tmpls.get(), default_effort_inputs); + assert_not_contains(default_effort_params.prompt, deepseek_v4_reasoning_effort_max); + assert_not_contains(default_effort_params.prompt, deepseek_v4_flash_0731_reasoning_effort_max); + + auto non_thinking_max_inputs = max_inputs; + non_thinking_max_inputs.enable_thinking = false; + auto non_thinking_max_params = common_chat_templates_apply(tmpls.get(), non_thinking_max_inputs); + assert_not_contains(non_thinking_max_params.prompt, deepseek_v4_flash_0731_reasoning_effort_max); + + common_chat_templates_inputs response_format_inputs; + response_format_inputs.messages = { system_msg, message_user }; + response_format_inputs.tools = { get_time_tool }; + response_format_inputs.json_schema = + R"({"type":"object","properties":{"answer":{"type":"string"}}})"; + auto response_format_params = common_chat_templates_apply(tmpls.get(), response_format_inputs); + assert_contains(response_format_params.prompt, + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n"); + assert_contains(response_format_params.prompt, R"("answer": {"type": "string"})"); + } + { auto tmpls = read_templates("models/templates/openbmb-MiniCPM5-1B.jinja"); check(tmpls, basic(), "<|im_start|>assistant\n\n"); From 66fa168a5617292651d93cb528db32537ecaa062 Mon Sep 17 00:00:00 2001 From: Ozymandias_EBON <112784549+johnkarlhill@users.noreply.github.com> Date: Mon, 3 Aug 2026 21:07:23 -0500 Subject: [PATCH 51/98] =?UTF-8?q?Extended=20SYCL=20oneDNN=20SDPA=20to=20no?= =?UTF-8?q?n-FP16=20KV=20caches=20(Q4=5F0=E2=80=93Q8=5F0=20and=20FP32)=20(?= =?UTF-8?q?#25874)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * sycl: extend oneDNN SDPA to Q4_0-Q8_0 and F32 KV caches Extends the oneDNN SDPA path (PR #25222) to handle non-F16 KV caches by dequantizing or converting K/V to dense FP16 on-device before feeding them into the SDPA graph. The fused systolic kernel then runs identically to the native FP16 path. Supported KV types: - Q4_0, Q4_1, Q5_0, Q5_1, Q8_0: to_fp16_sycl / to_fp16_nc_sycl - F32: cont_to_f16_sycl - BF16 and IQ types are excluded (no conversion kernel available) Gate: non-F16 requires K >= 1024 and Q >= 32 (prefill only). F16 KV runs at any length (existing behavior). Also includes the stream sync fix (stream->wait_and_throw() unconditional, PR #25741 by @malsbat) and removal of V_is_K_view aliasing (K and V are always dequantized to separate buffers). Co-Authored-By: Claude * docs: drop GGML_SYCL_FA_DEBUG from SYCL.md (not shipped in this PR) Co-Authored-By: Claude --------- Co-authored-by: Claude --- ggml/src/ggml-sycl/fattn-onednn.cpp | 134 +++++++++++++++++++++++++--- ggml/src/ggml-sycl/fattn.cpp | 25 ++++-- 2 files changed, 142 insertions(+), 17 deletions(-) diff --git a/ggml/src/ggml-sycl/fattn-onednn.cpp b/ggml/src/ggml-sycl/fattn-onednn.cpp index 8465e12248f4..fd17a25d5edd 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.cpp +++ b/ggml/src/ggml-sycl/fattn-onednn.cpp @@ -2,11 +2,13 @@ #include #include #include +#include #include #include #include "fattn-onednn.hpp" #include "fattn-tile.hpp" +#include "convert.hpp" // set minimum query length to treat as prefill (32) #define GGML_SYCL_FA_ONEDNN_MIN_Q 32 @@ -33,10 +35,30 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) { const ggml_tensor * mask = dst->src[3]; const ggml_tensor * sinks = dst->src[4]; - // gate for f16 KV only for now - // need to implement quantized KV + // F16 KV: native SDPA at any KV length. + // Non-F16: dequant to F16 then SDPA at prefill lengths. Only the + // standard quantized KV cache types (Q4_0-Q8_0) and F32 are accepted + // because their to_fp16_sycl conversion is verified. BF16 and IQ* + // are excluded: BF16 needs a strided conversion kernel that does not + // exist yet; IQ types are model-weight-only quants with no dequant + // registration and are never used as KV caches. if (K->type != GGML_TYPE_F16 || V->type != GGML_TYPE_F16) { - return false; + auto kt = K->type, vt = V->type; + bool k_ok = kt == GGML_TYPE_F32 || kt == GGML_TYPE_Q4_0 || kt == GGML_TYPE_Q4_1 || + kt == GGML_TYPE_Q5_0 || kt == GGML_TYPE_Q5_1 || kt == GGML_TYPE_Q8_0; + bool v_ok = vt == GGML_TYPE_F32 || vt == GGML_TYPE_Q4_0 || vt == GGML_TYPE_Q4_1 || + vt == GGML_TYPE_Q5_0 || vt == GGML_TYPE_Q5_1 || vt == GGML_TYPE_Q8_0; + if (!k_ok || !v_ok) { + return false; + } + if (Q->ne[1] < 32 || K->ne[1] < 1024) { + return false; + } + for (const ggml_tensor * t : {K, V}) { + if (t->type == GGML_TYPE_F16 && t->nb[1] % (t->ne[0] * 2) != 0) { + return false; + } + } } // Optional KV-length ceiling (GGML_SYCL_FA_ONEDNN_MAX_KV, 0 = unlimited). Escape hatch: // very long sequences make the fused SDPA slow enough to risk the xe driver watchdog on @@ -205,13 +227,101 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso dnnl::engine eng = ctx.engine_dnnl(stream); dnnl::stream strm = ctx.stream_dnnl(stream); - // cont/cast inputs to contiguous f16 (head-major) -- the layout the fast systolic path wants. - ggml_sycl_pool_alloc Qf(ctx.pool(), (size_t) H * q * d); - ggml_sycl_pool_alloc Kf(ctx.pool(), (size_t) Hkv * seq * d); - ggml_sycl_pool_alloc Vf(ctx.pool(), (size_t) Hkv * seq * d); - cont_to_f16_sycl ((const char *) Q->data, Qf.get(), d, q, H, mb, Q->nb[1], Q->nb[2], Q->nb[3], stream); - cont_to_f16_sycl((const char *) K->data, Kf.get(), d, seq, Hkv, mb, K->nb[1], K->nb[2], K->nb[3], stream); - cont_to_f16_sycl((const char *) V->data, Vf.get(), d, seq, Hkv, mb, V->nb[1], V->nb[2], V->nb[3], stream); + // Q: always f32 -- copy to dense f16. + ggml_sycl_pool_alloc Qf(ctx.pool(), (size_t) H * q * d); + cont_to_f16_sycl((const char *) Q->data, Qf.get(), d, q, H, mb, Q->nb[1], Q->nb[2], Q->nb[3], stream); + + // K/V: use pool-alloc for both F16 and dequant paths. + sycl::half * K_ptr = nullptr; + sycl::half * V_ptr = nullptr; + std::optional> Kf_pool; + std::optional> Vf_pool; + + if (K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16) { + Kf_pool.emplace(ctx.pool(), (size_t) Hkv * seq * d); + Vf_pool.emplace(ctx.pool(), (size_t) Hkv * seq * d); + cont_to_f16_sycl((const char *) K->data, Kf_pool->get(), d, seq, Hkv, mb, K->nb[1], K->nb[2], K->nb[3], stream); + cont_to_f16_sycl((const char *) V->data, Vf_pool->get(), d, seq, Hkv, mb, V->nb[1], V->nb[2], V->nb[3], stream); + K_ptr = Kf_pool->get(); + V_ptr = Vf_pool->get(); + } else if (ggml_is_quantized(K->type)) { + // Quantized K/V: dequant to dense F16 using pool, same lifetime as F16 path. + Kf_pool.emplace(ctx.pool(), ggml_nelements(K)); + K_ptr = Kf_pool->get(); + { + const char * K_data = (const char *)K->data; + const bool k_non_dense = ((int64_t)K->ne[1] * K->nb[1] != K->nb[2]) && K->ne[2] > 1; + const bool k_gemma = k_non_dense && + ((int64_t)K->nb[2] < (int64_t)K->ne[1] * (int64_t)K->nb[1]); + if (ggml_is_contiguously_allocated(K) && !k_non_dense) { + to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(K->type, dst); + to_fp16(K_data, K_ptr, ggml_nelements(K), stream); + } else { + const size_t bs = ggml_blck_size(K->type); + const size_t ts = ggml_type_size(K->type); + to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(K->type); + int64_t s01, s02, s03; + if (k_gemma) { + const int64_t blk_per_row = (int64_t)K->ne[0] / bs; + s01 = (int64_t)Hkv * blk_per_row; + s02 = blk_per_row; + s03 = (int64_t)K->ne[1] * s01; + } else { + s01 = (int64_t)K->nb[1] / ts; + s02 = (int64_t)K->nb[2] / ts; + s03 = (int64_t)K->nb[3] / ts; + } + to_fp16(K_data, K_ptr, + K->ne[0], K->ne[1], K->ne[2], K->ne[3], + s01, s02, s03, stream); + } + } + // Quantized V: always dequant separately. Even when K and V share + // the same underlying allocation (V is a view of K with the same + // data pointer), their logical values differ because the quantized + // elements at different positions/offsets represent different K/V + // data. Master's F16 path also never aliases K and V. + Vf_pool.emplace(ctx.pool(), ggml_nelements(V)); + V_ptr = Vf_pool->get(); + { + const char * V_data = (const char *)V->data; + const bool v_non_dense = ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]) && V->ne[2] > 1; + const bool v_gemma = v_non_dense && + ((int64_t)V->nb[2] < (int64_t)V->ne[1] * (int64_t)V->nb[1]); + if (ggml_is_contiguously_allocated(V) && !v_non_dense) { + to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(V->type, dst); + to_fp16(V_data, V_ptr, ggml_nelements(V), stream); + } else { + const size_t bs = ggml_blck_size(V->type); + const size_t ts = ggml_type_size(V->type); + to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(V->type); + int64_t s01, s02, s03; + if (v_gemma) { + const int64_t blk_per_row = (int64_t)V->ne[0] / bs; + s01 = (int64_t)V->ne[2] * blk_per_row; + s02 = blk_per_row; + s03 = (int64_t)V->ne[1] * s01; + } else { + s01 = (int64_t)V->nb[1] / ts; + s02 = (int64_t)V->nb[2] / ts; + s03 = (int64_t)V->nb[3] / ts; + } + to_fp16(V_data, V_ptr, + V->ne[0], V->ne[1], V->ne[2], V->ne[3], + s01, s02, s03, stream); + } + } + } else { + // F32: strided copy to dense F16 via cont_to_f16_sycl. + Kf_pool.emplace(ctx.pool(), ggml_nelements(K)); + K_ptr = Kf_pool->get(); + cont_to_f16_sycl((const char *) K->data, K_ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3], + K->nb[1], K->nb[2], K->nb[3], stream); + Vf_pool.emplace(ctx.pool(), ggml_nelements(V)); + V_ptr = Vf_pool->get(); + cont_to_f16_sycl((const char *) V->data, V_ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3], + V->nb[1], V->nb[2], V->nb[3], stream); + } // divide-by-(1/scale) reproduces ggml's score *= kq_scale on the proven probe graph. // @@ -244,8 +354,8 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso auto id2ptr = [&](size_t r) -> void * { if (r == E.id_q) return Qf.get(); - if (r == E.id_k) return Kf.get(); - if (r == E.id_v) return Vf.get(); + if (r == E.id_k) return K_ptr; + if (r == E.id_v) return V_ptr; if (r == E.id_scale) return scale_dev; if (r == E.id_mask) return (void *) mask->data; return nullptr; diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp index f8efa53acdeb..a85eb721f6af 100644 --- a/ggml/src/ggml-sycl/fattn.cpp +++ b/ggml/src/ggml-sycl/fattn.cpp @@ -97,7 +97,7 @@ static void ggml_sycl_flash_attn_ext_vec(ggml_backend_sycl_context & ctx, ggml_t enum best_fattn_kernel { BEST_FATTN_KERNEL_NONE = 0, BEST_FATTN_KERNEL_VEC = 100, - BEST_FATTN_KERNEL_ONEDNN = 150, // added enum for onednn==150 + BEST_FATTN_KERNEL_ONEDNN = 150, // oneDNN SDPA: native F16 (PR #25222) BEST_FATTN_KERNEL_TILE = 200, BEST_FATTN_KERNEL_MKL = 300, }; @@ -130,6 +130,14 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; + // XMX-accelerated path: oneDNN SDPA (native F16 and dequant+non-F16). + // ONEDNN requires min 32 query tokens — short-circuit decode to avoid + // calling _supported() on every decode FA call. + if (Q->ne[1] >= 32 + && ggml_sycl_flash_attn_ext_onednn_supported(dst)) { + return BEST_FATTN_KERNEL_ONEDNN; + } + // MKL path: XMX-accelerated GEMM for prompt processing (all KV cache types). // The MKL kernel converts non-F16 K/V to F16 via to_fp16_sycl before GEMM, // so quantized, F16, BF16, and F32 caches all benefit from XMX acceleration. @@ -167,7 +175,6 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_MKL; } } - for (const ggml_tensor * t : {Q, K, V, mask}) { if (t == nullptr || ggml_is_quantized(t->type)) { continue; @@ -215,6 +222,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const switch (K->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: + case GGML_TYPE_BF16: break; case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -233,8 +241,11 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_NONE; } - // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes: - const bool can_use_vector_kernel = Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0; + // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes. + // BF16 is excluded: the VEC kernel has no BF16 template (it needs GGML_SYCL_FA_ALL_QUANTS for non-F16/Q4_0/Q8_0). + const bool has_bf16 = (K->type == GGML_TYPE_BF16 || V->type == GGML_TYPE_BF16); + const bool can_use_vector_kernel = Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0 + && !has_bf16; // Fused-XMX path: oneDNN Graph SDPA (flash attention). Strictly // additive -- taken only when statically supported, otherwise falls through to VEC/TILE below. @@ -276,6 +287,7 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst const char * kname = "TILE"; best_fattn_kernel k = ggml_sycl_get_best_fattn_kernel(ctx.device, dst); if (k == BEST_FATTN_KERNEL_MKL) kname = "MKL"; + if (k == BEST_FATTN_KERNEL_ONEDNN) kname = "ONEDNN"; if (k == BEST_FATTN_KERNEL_VEC) kname = "VEC"; int64_t delta = 0; if (Dk == 256) { @@ -292,7 +304,8 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst (long long)V_dbg->ne[1]); } - switch (ggml_sycl_get_best_fattn_kernel(ggml_sycl_get_device(), dst)) { + const best_fattn_kernel fk = ggml_sycl_get_best_fattn_kernel(ggml_sycl_get_device(), dst); + switch (fk) { case BEST_FATTN_KERNEL_NONE: GGML_ABORT("Not support Flash-Attention"); case BEST_FATTN_KERNEL_ONEDNN: @@ -331,6 +344,7 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst q->wait(); const char * kname = "???"; best_fattn_kernel kb = ggml_sycl_get_best_fattn_kernel(ctx.device, dst); + if (kb == BEST_FATTN_KERNEL_ONEDNN) kname = "ONEDNN"; if (kb == BEST_FATTN_KERNEL_MKL) kname = "MKL"; if (kb == BEST_FATTN_KERNEL_TILE) kname = "TILE"; if (kb == BEST_FATTN_KERNEL_VEC) kname = "VEC"; @@ -354,6 +368,7 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst } } } + } bool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst) { From 6c8dcaa7ae41fa9f4aa2b3b68ee82cb8b2a03632 Mon Sep 17 00:00:00 2001 From: Titaniumtown Date: Mon, 3 Aug 2026 19:08:05 -0700 Subject: [PATCH 52/98] sycl: parallelize the non-contiguous concat kernel (#25852) * sycl: parallelize the non-contiguous concat kernel Launch geometry only: the non-contiguous concat kernel launched a single-lane work-group (1, 1, 1), now it will launch a (1, 1, SYCL_CONCAT_BLOCK_SIZE) one. SYCL_CONCAT_BLOCK_SIZE is defined in `ggml/src/ggml-sycl/presets.hpp`. llama-bench (Arc Pro B70, Qwen3.6-27B-UD-Q4_K_XL, -fa on, q8_0 KV), on top of upstream master: pp2048 920 -> 1006 t/s (+9.4%) * sycl: cap non-contiguous concat block at ne0 * sycl: make non-contiguous concat block width env-tunable (GGML_SYCL_CONCAT_BLOCK_SIZE) * Revert "sycl: make non-contiguous concat block width env-tunable (GGML_SYCL_CONCAT_BLOCK_SIZE)" This reverts commit 2709909e790d298cc0746950caff16ad8ecf9d6f. --- ggml/src/ggml-sycl/concat.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-sycl/concat.cpp b/ggml/src/ggml-sycl/concat.cpp index 93e00d65fcd3..1ad242fcafbb 100644 --- a/ggml/src/ggml-sycl/concat.cpp +++ b/ggml/src/ggml-sycl/concat.cpp @@ -127,7 +127,15 @@ static void concat_T_sycl_non_cont( int64_t ne2, int64_t ne3, uint64_t nb0, uint64_t nb1, uint64_t nb2, uint64_t nb3, int32_t dim) { sycl::range<3> gridDim(ne3, ne2, ne1); - stream->parallel_for(sycl::nd_range<3>(gridDim, sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + + // Avoid oversubscribing device when there is not enough elements along the innermost dim to + // fill a full SYCL_CONCAT_BLOCK_SIZE. For larger # of elements, the full SYCL_CONCAT_BLOCK_SIZE + // is used. + const int64_t ne0_pad = GGML_PAD(ne0, WARP_SIZE); + const int64_t block_ne0 = ne0_pad < SYCL_CONCAT_BLOCK_SIZE ? ne0_pad : (int64_t) SYCL_CONCAT_BLOCK_SIZE; + sycl::range<3> blockDim(1, 1, block_ne0); + + stream->parallel_for(sycl::nd_range<3>(gridDim * blockDim, blockDim), [=](sycl::nd_item<3> item_ct1) { int64_t i3 = item_ct1.get_group(0); int64_t i2 = item_ct1.get_group(1); int64_t i1 = item_ct1.get_group(2); From 22dc605c4ead20e36f447cc67b55ef87e523bd55 Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Tue, 4 Aug 2026 03:28:57 +0000 Subject: [PATCH 53/98] ci: fix vulkan llvmpipe runs (#26533) --- .github/workflows/build-vulkan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml index a103c50faf70..01113803fffd 100644 --- a/.github/workflows/build-vulkan.yml +++ b/.github/workflows/build-vulkan.yml @@ -119,6 +119,7 @@ jobs: run: | source ./vulkan_sdk/setup-env.sh cmake -B build \ + -DGGML_NATIVE=OFF \ -DGGML_VULKAN=ON cmake --build build --config Release -j $(nproc) From 935cad6497e8d1569b3302c1d12d03472b556401 Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Tue, 4 Aug 2026 08:02:49 +0200 Subject: [PATCH 54/98] llama : move n_vocab from llama_sampler_data to penalty_sampler (#26520) This matches how it is done for logit_bias and mirostat samplers, see https://github.com/ggml-org/llama.cpp/pull/25262#discussion_r3703951151 --- common/sampling.cpp | 2 +- include/llama.h | 2 +- src/llama-graph.cpp | 1 - src/llama-sampler.cpp | 11 +++++++---- tests/test-backend-sampler.cpp | 11 ++++++----- tests/test-sampling.cpp | 2 +- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 5698c0263b24..ba5504ed0118 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -383,7 +383,7 @@ struct common_sampler * common_sampler_init( samplers.push_back(llama_sampler_init_infill(vocab)); break; case COMMON_SAMPLER_TYPE_PENALTIES: - samplers.push_back(llama_sampler_init_penalties(params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present)); + samplers.push_back(llama_sampler_init_penalties(llama_vocab_n_tokens(vocab), params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present)); break; case COMMON_SAMPLER_TYPE_ADAPTIVE_P: // the `adaptive-p` sampler is like `dist` and `mirostat` in that it selects diff --git a/include/llama.h b/include/llama.h index f2d7e3885837..fb2ca38cee4f 100644 --- a/include/llama.h +++ b/include/llama.h @@ -1256,7 +1256,6 @@ extern "C" { struct ggml_tensor * probs; struct ggml_tensor * sampled; struct ggml_tensor * candidates; - int64_t n_vocab; }; // user code can implement the interface below in order to create custom llama_sampler @@ -1425,6 +1424,7 @@ extern "C" { /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. LLAMA_API struct llama_sampler * llama_sampler_init_penalties( + int32_t n_vocab, int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) float penalty_repeat, // must be > 0.0, 1.0 = disabled float penalty_freq, // must be finite, 0.0 = disabled diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 9dde345df406..2be3b75fb982 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -3683,7 +3683,6 @@ void llm_graph_context::build_sampling() const { /*.probs =*/ nullptr, /*.sampled =*/ nullptr, /*.candidates =*/ nullptr, - /*.n_vocab =*/ logits_seq->ne[0], }; assert(sampler->iface->backend_apply); diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cpp index b2f1abe73785..6cf2d27cf9ad 100644 --- a/src/llama-sampler.cpp +++ b/src/llama-sampler.cpp @@ -589,7 +589,6 @@ static bool llama_sampler_backend_support( /*.probs = */ nullptr, /*.sampled = */ nullptr, /*.candidates = */ ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n), - /*.n_vocab = */ n, }; ggml_cgraph * gf = ggml_new_graph(ctx); @@ -2640,6 +2639,7 @@ struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( // penalties struct llama_sampler_penalties : public llama_sampler_backend { + const int32_t n_vocab; const int32_t penalty_last_n; const float penalty_repeat; const float penalty_freq; @@ -2655,7 +2655,6 @@ struct llama_sampler_penalties : public llama_sampler_backend { ggml_tensor * inp_counts = nullptr; // backend helpers - int32_t n_vocab = 0; int32_t n_max = 0; bool has_candidates = false; @@ -2676,11 +2675,13 @@ struct llama_sampler_penalties : public llama_sampler_backend { } llama_sampler_penalties( + int32_t n_vocab, int32_t penalty_last_n, float penalty_repeat, float penalty_freq, float penalty_present) : llama_sampler_backend("penalties") + , n_vocab (n_vocab) , penalty_last_n (penalty_last_n) , penalty_repeat (penalty_repeat) , penalty_freq (penalty_freq) @@ -2766,6 +2767,7 @@ static void llama_sampler_penalties_reset(struct llama_sampler * smpl) { static struct llama_sampler * llama_sampler_penalties_clone(const struct llama_sampler * smpl) { const auto * ctx = (const llama_sampler_penalties *) smpl->ctx; auto * result = llama_sampler_init_penalties( + ctx->n_vocab, ctx->penalty_last_n, ctx->penalty_repeat, ctx->penalty_freq, @@ -2811,10 +2813,9 @@ static void llama_sampler_penalties_backend_apply( return; } - GGML_ASSERT(data->n_vocab > 0 && data->n_vocab <= INT32_MAX); + GGML_ASSERT(sctx->n_vocab > 0); sctx->has_candidates = data->candidates != nullptr; - sctx->n_vocab = (int32_t) data->n_vocab; sctx->n_max = std::min(sctx->penalty_last_n, sctx->n_vocab); sctx->inp_token_ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, sctx->n_max); @@ -2965,6 +2966,7 @@ static struct llama_sampler_i llama_sampler_penalties_i = { }; struct llama_sampler * llama_sampler_init_penalties( + int32_t n_vocab, int32_t penalty_last_n, float penalty_repeat, float penalty_freq, @@ -2979,6 +2981,7 @@ struct llama_sampler * llama_sampler_init_penalties( return llama_sampler_init( /* .iface = */ &llama_sampler_penalties_i, /* .ctx = */ new llama_sampler_penalties( + n_vocab, penalty_last_n, penalty_repeat, penalty_freq, diff --git a/tests/test-backend-sampler.cpp b/tests/test-backend-sampler.cpp index 1a46468ba264..1165f46f0c92 100644 --- a/tests/test-backend-sampler.cpp +++ b/tests/test-backend-sampler.cpp @@ -823,6 +823,7 @@ enum class penalties_position { static void add_filter_and_penalties( llama_sampler * chain, const sampler_init_fn & init_filter, + int32_t n_vocab, int32_t penalty_last_n, float penalty_repeat, float penalty_freq, @@ -830,7 +831,7 @@ static void add_filter_and_penalties( penalties_position position) { const auto add_penalties = [&]() { llama_sampler_chain_add(chain, llama_sampler_init_penalties( - penalty_last_n, penalty_repeat, penalty_freq, penalty_present)); + n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present)); }; if (position == penalties_position::before_filter) { @@ -1006,7 +1007,7 @@ static sampler_comparison_output run_penalties_comparison( const std::vector raw_logits = decode_raw_logits(params, prompt); const auto add_samplers = [&](llama_sampler * chain) { llama_sampler_chain_add(chain, llama_sampler_init_penalties( - penalty_last_n, penalty_repeat, penalty_freq, penalty_present)); + llama_vocab_n_tokens(vocab), penalty_last_n, penalty_repeat, penalty_freq, penalty_present)); }; const auto accept_history = [&](llama_sampler * chain) { accept_prompt(chain, vocab, prompt); @@ -1105,7 +1106,7 @@ static void compare_top_k_penalties_logits( GGML_ASSERT(excluded_history_token != LLAMA_TOKEN_NULL); const auto add_samplers = [&](llama_sampler * chain) { - add_filter_and_penalties(chain, init_top_k, + add_filter_and_penalties(chain, init_top_k, n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, position); }; @@ -1190,7 +1191,7 @@ static void compare_masking_penalties_logits( GGML_ASSERT(masked_token != LLAMA_TOKEN_NULL); const auto add_samplers = [&](llama_sampler * chain) { - add_filter_and_penalties(chain, init_filter, + add_filter_and_penalties(chain, init_filter, n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, position); }; auto accept_history = [&](llama_sampler * smpl) { @@ -1218,7 +1219,7 @@ static void compare_masking_penalties_logits( GGML_ASSERT(fabsf(expected_logits.at(penalized_token) - raw_logits[penalized_token]) > 1e-6f); } else { llama_sampler_ptr penalties(llama_sampler_init_penalties( - penalty_last_n, penalty_repeat, penalty_freq, penalty_present)); + n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present)); accept_history(penalties.get()); const std::unordered_map penalized_logits = map_logits(apply_cpu_sampler(raw_logits, penalties.get())); diff --git a/tests/test-sampling.cpp b/tests/test-sampling.cpp index 2aecff90e7bb..297f760157df 100644 --- a/tests/test-sampling.cpp +++ b/tests/test-sampling.cpp @@ -144,7 +144,7 @@ static void test_penalties( sampler_tester tester(probs, probs_expected); - auto * sampler = llama_sampler_init_penalties(last_tokens.size(), repeat_penalty, alpha_frequency, alpha_presence); + auto * sampler = llama_sampler_init_penalties((int32_t) probs.size(), (int32_t) last_tokens.size(), repeat_penalty, alpha_frequency, alpha_presence); for (size_t i = 0; i < last_tokens.size(); i++) { llama_sampler_accept(sampler, last_tokens[i]); From 1269cb1ff1598751f846241be90083ae9ad036fb Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 4 Aug 2026 09:06:44 +0300 Subject: [PATCH 55/98] model : allow reshape of tensors during load (#26531) --- src/llama-model-loader.cpp | 97 ++++++++++++++++++-------------------- src/llama-model-loader.h | 9 ++-- src/llama-model.cpp | 3 +- src/llama-model.h | 1 + src/models/deepseek4.cpp | 6 ++- 5 files changed, 59 insertions(+), 57 deletions(-) diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index df8313e81976..b31e92e2da7e 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -857,7 +857,11 @@ struct ggml_tensor * llama_model_loader::require_tensor_meta(const std::string & return tensor; } -const struct ggml_tensor * llama_model_loader::check_tensor_dims(const std::string & name, const std::vector & ne, bool required) const { +const struct ggml_tensor * llama_model_loader::check_tensor_dims( + const std::string & name, + const std::vector & ne, + bool required, + bool allow_reshape) const { const struct ggml_tensor * cur = get_tensor_meta(name.c_str()); if (cur == NULL) { @@ -867,21 +871,33 @@ const struct ggml_tensor * llama_model_loader::check_tensor_dims(const std::stri throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str())); } - { - bool is_ok = true; + bool is_ok = true; + + if (allow_reshape) { + // check total number of elements only + const int64_t ncur = ggml_nelements(cur); + int64_t nexp = 1; + for (size_t i = 0; i < ne.size(); ++i) { + nexp *= ne[i]; + } + if (ncur != nexp) { + is_ok = false; + } + } else { for (size_t i = 0; i < GGML_MAX_DIMS; ++i) { if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) { is_ok = false; break; } } - if (!is_ok) { - throw std::runtime_error( - format("%s: tensor '%s' has wrong shape; expected %s, got %s", - __func__, name.c_str(), - llama_format_tensor_shape(ne).c_str(), - llama_format_tensor_shape(cur).c_str())); - } + } + + if (!is_ok) { + throw std::runtime_error( + format("%s: tensor '%s' has wrong shape; expected %s, got %s", + __func__, name.c_str(), + llama_format_tensor_shape(ne).c_str(), + llama_format_tensor_shape(cur).c_str())); } return cur; @@ -1246,11 +1262,25 @@ struct ggml_tensor * llama_model_loader::create_tensor( return ret; } - ggml_tensor * t_meta = get_tensor_meta(tn.str().c_str()); - ggml_backend_buffer_type_t buft = buft_for_tensor(t_meta); + LLAMA_LOG_DEBUG("%s: loading tensor %s\n", __func__, tn.str().c_str()); + const struct ggml_tensor * cur = check_tensor_dims(tn.str(), ne, !(flags & TENSOR_NOT_REQUIRED), flags & TENSOR_ALLOW_RESHAPE); + if (cur == NULL) { + return NULL; + } + + ggml_tensor t_meta = *cur; + if (flags & TENSOR_ALLOW_RESHAPE) { + for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) { + t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1; + t_meta.nb[dim] = dim == 0 ? ggml_type_size(t_meta.type) : t_meta.ne[dim-1]*t_meta.nb[dim-1]; + } + } + + ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta); if (buft == nullptr) { - return nullptr; // return type is ggml_tensor * + return nullptr; } + ggml_context * ctx = ctx_for_buft(buft); // if duplicated, check if the original tensor was allocated in the same buffer type context and avoid creating a new one @@ -1261,20 +1291,13 @@ struct ggml_tensor * llama_model_loader::create_tensor( } } - LLAMA_LOG_DEBUG("%s: loading tensor %s\n", __func__, tn.str().c_str()); - const struct ggml_tensor * cur = check_tensor_dims(tn.str(), ne, !(flags & TENSOR_NOT_REQUIRED)); - - if (cur == NULL) { - return NULL; - } - const bool duplicated = flags & TENSOR_DUPLICATED; - struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur); - ggml_set_name(tensor, ggml_get_name(cur)); + struct ggml_tensor * tensor = ggml_dup_tensor(ctx, &t_meta); + ggml_set_name(tensor, ggml_get_name(&t_meta)); if (duplicated) { - size_data += ggml_nbytes(cur); + size_data += ggml_nbytes(&t_meta); } else { n_created++; } @@ -1282,34 +1305,6 @@ struct ggml_tensor * llama_model_loader::create_tensor( return tensor; } -struct ggml_tensor * llama_model_loader::create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list & ne, size_t offset, bool required) { - const struct ggml_tensor * cur = check_tensor_dims(name, ne, required); - - if (cur == NULL) { - return NULL; - } - - if (cur->type != base->type) { - throw std::runtime_error(format("%s: tensor '%s' has wrong type; expected %s, got %s", __func__, name.c_str(), ggml_type_name(base->type), ggml_type_name(cur->type))); - } - - std::array dims; - for (size_t i = 0; i < GGML_MAX_DIMS; ++i) { - dims[i] = i < ne.size() ? ne.begin()[i] : 1; - } - - struct ggml_tensor * tensor = ggml_view_4d(ctx, base, - dims[0], dims[1], dims[2], dims[3], - cur->nb[1], cur->nb[2], cur->nb[3], - offset); - - ggml_set_name(tensor, name.c_str()); - - n_created++; - - return tensor; -} - void llama_model_loader::done_getting_tensors(bool partial) const { if (n_created > n_tensors) { throw std::runtime_error(format("%s: too many tensors created; expected %d, got %d", __func__, n_tensors, n_created)); diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index 7ad380782291..d6b31c231118 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -67,6 +67,7 @@ struct llama_model_loader { static const int TENSOR_DUPLICATED = 1 << 1; static const int TENSOR_SKIP = 1 << 2; static const int TENSOR_SKIP_IF_VIRTUAL = 1 << 3; + static const int TENSOR_ALLOW_RESHAPE = 1 << 4; int n_kv = 0; int n_tensors = 0; @@ -177,14 +178,16 @@ struct llama_model_loader { struct ggml_tensor * require_tensor_meta(const std::string & name) const; - const struct ggml_tensor * check_tensor_dims(const std::string & name, const std::vector & ne, bool required) const; + const struct ggml_tensor * check_tensor_dims( + const std::string & name, + const std::vector & ne, + bool required, + bool allow_reshape) const; struct ggml_tensor * create_tensor( const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output, const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags); - struct ggml_tensor * create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list & ne, size_t offset, bool required = true); - void done_getting_tensors(bool partial = false) const; void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 8fff1a432667..333f506de557 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2867,7 +2867,8 @@ llama_model_base::llama_model_base(const struct llama_model_params & params) : l TENSOR_DUPLICATED (llama_model_loader::TENSOR_DUPLICATED), TENSOR_NOT_REQUIRED (llama_model_loader::TENSOR_NOT_REQUIRED), TENSOR_SKIP (llama_model_loader::TENSOR_SKIP), - TENSOR_SKIP_IF_VIRTUAL(llama_model_loader::TENSOR_SKIP_IF_VIRTUAL) {} + TENSOR_SKIP_IF_VIRTUAL(llama_model_loader::TENSOR_SKIP_IF_VIRTUAL), + TENSOR_ALLOW_RESHAPE (llama_model_loader::TENSOR_ALLOW_RESHAPE) {} ggml_tensor * llama_model_base::create_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags) { GGML_ASSERT(ml != nullptr); diff --git a/src/llama-model.h b/src/llama-model.h index 056a6efa59e8..6b9e94a0a692 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -719,6 +719,7 @@ struct llama_model_base : public llama_model { const int TENSOR_NOT_REQUIRED; const int TENSOR_SKIP; const int TENSOR_SKIP_IF_VIRTUAL; + const int TENSOR_ALLOW_RESHAPE; explicit llama_model_base(const llama_model_params & params); virtual ~llama_model_base() = default; diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index e68dc49b6dff..89cd461765ad 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -114,7 +114,9 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) { layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, flags); layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, flags); layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, flags); - layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, flags); + // for wo_a, the shape in the file is (n_head * n_embd_head / o_groups, o_lora_rank*o_groups) + // so we reshape here, to avoid reshaping the tensor in the graph + layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank, o_groups}, flags | TENSOR_ALLOW_RESHAPE); layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, flags); layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, flags); @@ -1258,7 +1260,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, nt); out = ggml_permute(ctx0, out, 0, 2, 1, 3); - ggml_tensor * oa = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, layer.wo_a, layer.wo_a->ne[0], o_lora_rank, n_groups), out); + ggml_tensor * oa = ggml_mul_mat(ctx0, layer.wo_a, out); cb(oa, "attn_wo_a", il); oa = ggml_permute(ctx0, oa, 0, 2, 1, 3); oa = ggml_cont_2d(ctx0, oa, o_lora_rank*n_groups, nt); From b06fbc968b980daa313687152114dd1bfd1edaa9 Mon Sep 17 00:00:00 2001 From: Caleb DeLeeuw <143902425+SolshineCode@users.noreply.github.com> Date: Tue, 4 Aug 2026 00:34:30 -0700 Subject: [PATCH 56/98] convert : import bytes_to_unicode from convert_slow_tokenizer (#26217) bytes_to_unicode was removed from transformers.models.gpt2.tokenization_gpt2 in huggingface/transformers#40936, but it had already been copied into transformers.convert_slow_tokenizer in huggingface/transformers#30334 (transformers 4.54.1), so import it directly from there. Applies the same fix to chatglm.py. --- conversion/chatglm.py | 2 +- conversion/qwen.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conversion/chatglm.py b/conversion/chatglm.py index 801913075dbc..d6385503877c 100644 --- a/conversion/chatglm.py +++ b/conversion/chatglm.py @@ -81,7 +81,7 @@ def set_vocab_chatglm3(self): @staticmethod def token_bytes_to_string(b): - from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import] + from transformers.convert_slow_tokenizer import bytes_to_unicode byte_encoder = bytes_to_unicode() return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')]) diff --git a/conversion/qwen.py b/conversion/qwen.py index 7e3d8c0d12d3..b4ae528bf2d4 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -18,7 +18,7 @@ class QwenModel(TextModel): @staticmethod def token_bytes_to_string(b): - from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import] + from transformers.convert_slow_tokenizer import bytes_to_unicode byte_encoder = bytes_to_unicode() return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')]) From cf06ad7dfe79e7bf2934150d4fefe1ce8db05490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 4 Aug 2026 10:40:02 +0200 Subject: [PATCH 57/98] vocab : validate plamo2 byte tokens (#26511) * validate plamo2 byte tokens * --typo --- src/llama-vocab.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 10032a8c6442..4a01dfd4cab6 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -1373,8 +1373,10 @@ struct llm_tokenizer_plamo2 : llm_tokenizer { if (vocab.is_byte(token_id)) { if (entry.text.length() == 6 && entry.text.substr(0, 3) == "<0x" && entry.text.back() == '>') { std::string hex_str = entry.text.substr(3, 2); - int byte_val = std::stoi(hex_str, nullptr, 16); - bytes_[byte_val] = static_cast(token_id); + if (std::isxdigit(static_cast(hex_str[0])) && std::isxdigit(static_cast(hex_str[1]))) { + int byte_val = std::stoi(hex_str, nullptr, 16); + bytes_[byte_val] = static_cast(token_id); + } } continue; } @@ -3625,12 +3627,15 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t if (vocab.is_byte(token)) { // Handle byte tokens like <0xXX> if (token_text.length() == 6 && token_text.substr(0, 3) == "<0x" && token_text.back() == '>') { - int hex_val = std::stoi(token_text.substr(3, 2), nullptr, 16); - if (length < 1) { - return -1; + std::string hex_str = token_text.substr(3, 2); + if (std::isxdigit(static_cast(hex_str[0])) && std::isxdigit(static_cast(hex_str[1]))) { + int hex_val = std::stoi(hex_str, nullptr, 16); + if (length < 1) { + return -1; + } + buf[0] = static_cast(hex_val); + return 1; } - buf[0] = static_cast(hex_val); - return 1; } } From f26efa02a77dc3660f94ac90efee59394f3bc74d Mon Sep 17 00:00:00 2001 From: Pranav Uttarkar <122235768+PranavUttarkar@users.noreply.github.com> Date: Tue, 4 Aug 2026 03:40:54 -0500 Subject: [PATCH 58/98] vulkan backend ops: implemented GATED_LINEAR_ATTN (#25601) * vulkan : add GATED_LINEAR_ATTN op * docs : update Vulkan ops * vulkan : remove unused GLA spec constant * Updated ops.md * ops.md update --- docs/ops.md | 30 +- docs/ops/Vulkan.csv | 5101 +++++++++++++---- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 62 + ggml/src/ggml-vulkan/vulkan-shaders/gla.comp | 82 + .../vulkan-shaders/vulkan-shaders-gen.cpp | 2 + 5 files changed, 4150 insertions(+), 1127 deletions(-) create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/gla.comp diff --git a/docs/ops.md b/docs/ops.md index 557b1a023d2e..71bd72011eae 100644 --- a/docs/ops.md +++ b/docs/ops.md @@ -23,16 +23,16 @@ Legend: | ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | -| CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | 🟡 | ✅ | ❌ | ❌ | -| COL2IM_1D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | +| COL2IM_1D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ | | CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ | -| CONV_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | +| CONV_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | | CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | -| CONV_3D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| CONV_3D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | -| COS | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ | +| COS | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | | CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | @@ -51,8 +51,8 @@ Legend: | FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | | FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | -| GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ | -| GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | +| GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | @@ -60,14 +60,14 @@ Legend: | GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ | -| GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | | GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | | HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ | -| LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ❌ | ✅ | 🟡 | ❌ | ❌ | ❌ | +| LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | LIGHTNING_INDEXER | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | @@ -76,13 +76,13 @@ Legend: | MUL_MAT_HADAMARD | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ❌ | | NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | -| NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ❌ | +| NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ | | OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | | OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | -| OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | 🟡 | +| OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | 🟡 | | PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | | PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | @@ -103,13 +103,13 @@ Legend: | SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | -| SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ | +| SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | | SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | -| SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | -| SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | +| SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | +| SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | | STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | diff --git a/docs/ops/Vulkan.csv b/docs/ops/Vulkan.csv index 3aa7976a2087..59e67e1b208d 100644 --- a/docs/ops/Vulkan.csv +++ b/docs/ops/Vulkan.csv @@ -167,6 +167,16 @@ "Vulkan0","ROUND","type=f32,ne_a=[5,7,11,13],v=1","support","1","yes","Vulkan" "Vulkan0","TRUNC","type=f32,ne_a=[128,2,2,2],v=1","support","1","yes","Vulkan" "Vulkan0","TRUNC","type=f32,ne_a=[5,7,11,13],v=1","support","1","yes","Vulkan" +"Vulkan0","DSV4_HC_COMB","n_tokens=1,n_iter=1,eps=0.000001","support","0","no","Vulkan" +"Vulkan0","DSV4_HC_COMB","n_tokens=17,n_iter=4,eps=0.000001","support","0","no","Vulkan" +"Vulkan0","DSV4_HC_COMB","n_tokens=257,n_iter=8,eps=0.000001","support","0","no","Vulkan" +"Vulkan0","DSV4_HC_PRE","n_embd=1,n_tokens=1","support","0","no","Vulkan" +"Vulkan0","DSV4_HC_PRE","n_embd=31,n_tokens=17","support","0","no","Vulkan" +"Vulkan0","DSV4_HC_PRE","n_embd=128,n_tokens=257","support","0","no","Vulkan" +"Vulkan0","DSV4_HC_PRE","n_embd=4096,n_tokens=21","support","0","no","Vulkan" +"Vulkan0","DSV4_HC_POST","n_embd=1,n_tokens=1","support","0","no","Vulkan" +"Vulkan0","DSV4_HC_POST","n_embd=31,n_tokens=17","support","0","no","Vulkan" +"Vulkan0","DSV4_HC_POST","n_embd=128,n_tokens=257","support","0","no","Vulkan" "Vulkan0","REGLU","type=f16,ne_a=[128,2,2,2],v=0,swapped=0","support","1","yes","Vulkan" "Vulkan0","REGLU","type=f16,ne_a=[5,7,11,13],v=0,swapped=0","support","1","yes","Vulkan" "Vulkan0","REGLU","type=f16,ne_a=[128,2,2,2],v=0,swapped=1","support","1","yes","Vulkan" @@ -338,6 +348,10 @@ "Vulkan0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","Vulkan" "Vulkan0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","Vulkan" "Vulkan0","GET_ROWS","type=q1_0,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","Vulkan" +"Vulkan0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","Vulkan" +"Vulkan0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","Vulkan" +"Vulkan0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","Vulkan" +"Vulkan0","GET_ROWS","type=q2_0,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","Vulkan" "Vulkan0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=1,be2=1,v=0","support","1","yes","Vulkan" "Vulkan0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","Vulkan" "Vulkan0","GET_ROWS","type=mxfp4,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","Vulkan" @@ -406,9 +420,10 @@ "Vulkan0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=1,be2=1,v=1","support","1","yes","Vulkan" "Vulkan0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=7,be2=1,v=0","support","1","yes","Vulkan" "Vulkan0","GET_ROWS","type=i32,n=256,m=5,r=4,be1=7,be2=1,v=1","support","1","yes","Vulkan" -"Vulkan0","GET_ROWS_BACK","type=f32,n=1,m=8,r=2,b=1,v=0","support","0","no","Vulkan" -"Vulkan0","GET_ROWS_BACK","type=f32,n=256,m=5,r=4,b=1,v=0","support","0","no","Vulkan" -"Vulkan0","GET_ROWS_BACK","type=f32,n=256,m=5,r=4,b=1,v=1","support","0","no","Vulkan" +"Vulkan0","GET_ROWS_BACK","type=f32,n=1,m=8,r=2,b=1,v=0","support","1","yes","Vulkan" +"Vulkan0","GET_ROWS_BACK","type=f32,n=1,m=70000,r=4,b=1,v=0","support","1","yes","Vulkan" +"Vulkan0","GET_ROWS_BACK","type=f32,n=256,m=5,r=4,b=1,v=0","support","1","yes","Vulkan" +"Vulkan0","GET_ROWS_BACK","type=f32,n=256,m=5,r=4,b=1,v=1","support","1","yes","Vulkan" "Vulkan0","GET_ROWS_BACK","type=f16,n=256,m=5,r=4,b=1,v=0","support","0","no","Vulkan" "Vulkan0","GET_ROWS_BACK","type=f16,n=256,m=5,r=4,b=1,v=1","support","0","no","Vulkan" "Vulkan0","GET_ROWS_BACK","type=bf16,n=256,m=5,r=4,b=1,v=0","support","0","no","Vulkan" @@ -425,6 +440,8 @@ "Vulkan0","GET_ROWS_BACK","type=q8_0,n=256,m=5,r=4,b=1,v=1","support","0","no","Vulkan" "Vulkan0","GET_ROWS_BACK","type=q1_0,n=256,m=5,r=4,b=1,v=0","support","0","no","Vulkan" "Vulkan0","GET_ROWS_BACK","type=q1_0,n=256,m=5,r=4,b=1,v=1","support","0","no","Vulkan" +"Vulkan0","GET_ROWS_BACK","type=q2_0,n=256,m=5,r=4,b=1,v=0","support","0","no","Vulkan" +"Vulkan0","GET_ROWS_BACK","type=q2_0,n=256,m=5,r=4,b=1,v=1","support","0","no","Vulkan" "Vulkan0","GET_ROWS_BACK","type=mxfp4,n=256,m=5,r=4,b=1,v=0","support","0","no","Vulkan" "Vulkan0","GET_ROWS_BACK","type=mxfp4,n=256,m=5,r=4,b=1,v=1","support","0","no","Vulkan" "Vulkan0","GET_ROWS_BACK","type=nvfp4,n=256,m=5,r=4,b=1,v=0","support","0","no","Vulkan" @@ -459,333 +476,685 @@ "Vulkan0","GET_ROWS_BACK","type=iq4_xs,n=256,m=5,r=4,b=1,v=1","support","0","no","Vulkan" "Vulkan0","GET_ROWS_BACK","type=i32,n=256,m=5,r=4,b=1,v=0","support","0","no","Vulkan" "Vulkan0","GET_ROWS_BACK","type=i32,n=256,m=5,r=4,b=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i32,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f32,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=f16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=bf16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q4_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q5_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q8_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[384,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[384,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[384,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=q1_0,type_idx=i64,ne=[384,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=mxfp4,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[192,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[192,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[192,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=nvfp4,type_idx=i64,ne=[192,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q2_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q3_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q4_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q5_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=q6_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq2_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq1_m,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_nl,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq3_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" -"Vulkan0","SET_ROWS","type=iq4_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i32,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f32,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=bf16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q8_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[384,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[384,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[384,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q1_0,type_idx=i64,ne=[384,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[192,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[192,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[192,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_0,type_idx=i64,ne=[192,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=mxfp4,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[192,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[192,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[192,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=nvfp4,type_idx=i64,ne=[192,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q2_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q3_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q4_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q5_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=q6_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq2_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq1_m,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_nl,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq3_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=iq4_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f32,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=f16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[3,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[31,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[33,5,1,1],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[3,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[31,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=bf16,type_idx=i64,ne=[33,5,1,7],nr23=[2,3],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_1,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q8_0,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[384,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[384,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[384,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q1_0,type_idx=i64,ne=[384,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[192,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[192,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[192,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_0,type_idx=i64,ne=[192,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=mxfp4,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[192,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[192,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[192,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=nvfp4,type_idx=i64,ne=[192,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q2_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q3_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q4_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q5_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=q6_K,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq2_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_xxs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq1_m,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[96,3,1,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_nl,type_idx=i64,ne=[96,3,7,1],nr23=[2,3],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq3_s,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[256,5,1,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[256,11,1,1],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[768,3,1,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=0","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[256,5,7,3],nr23=[1,1],r=1,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[256,11,1,7],nr23=[2,3],r=7,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f32,type_dst=iq4_xs,type_idx=i64,ne=[768,3,7,1],nr23=[2,3],r=2,v=1","support","0","no","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","1","yes","Vulkan" +"Vulkan0","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","1","yes","Vulkan" "Vulkan0","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","Vulkan" "Vulkan0","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","Vulkan" "Vulkan0","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","Vulkan" @@ -914,57 +1283,226 @@ "Vulkan0","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","Vulkan" "Vulkan0","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","Vulkan" "Vulkan0","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=0","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=1","support","0","no","Vulkan" -"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=1","support","0","no","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=2,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=2,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=2,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=1,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=2,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=3,p0=2","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[10,3,2,1],k0=3,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[11,1,3,2],k0=3,s0=3,p0=3","support","1","yes","Vulkan" +"Vulkan0","POOL_1D","pool_type=max,type_input=f32,ne_input=[128,2,1,3],k0=3,s0=3,p0=3","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[3000,128,1,1],ne_kernel=[3,128,1280,1],s0=1,s1=0,p0=1,p1=0,d0=1,d1=0,is_2D=0","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f16,dst_type=f32,ne_input=[3000,128,1,1],ne_kernel=[3,128,1280,1],s0=1,s1=0,p0=1,p1=0,d0=1,d1=0,is_2D=0","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f16,dst_type=f16,ne_input=[3000,128,1,1],ne_kernel=[3,128,1280,1],s0=1,s1=0,p0=1,p1=0,d0=1,d1=0,is_2D=0","support","1","yes","Vulkan" +"Vulkan0","IM2COL","type_input=f32,type_kernel=f16,dst_type=f16,ne_input=[3000,384,1,1],ne_kernel=[3,384,384,1],s0=1,s1=0,p0=1,p1=0,d0=1,d1=0,is_2D=0","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,2,2,1],ne_kernel=[3,2,2,1],s0=1,s1=0,p0=0,p1=0,d0=1,d1=0,is_2D=0","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,2,2,1],ne_kernel=[3,2,2,1],s0=1,s1=0,p0=0,p1=0,d0=3,d1=0,is_2D=0","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,2,2,1],ne_kernel=[3,2,2,1],s0=1,s1=0,p0=3,p1=0,d0=1,d1=0,is_2D=0","support","1","yes","Vulkan" @@ -974,6 +1512,7 @@ "Vulkan0","IM2COL","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,2,2,1],ne_kernel=[3,2,2,1],s0=3,s1=0,p0=3,p1=0,d0=1,d1=0,is_2D=0","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,2,2,1],ne_kernel=[3,2,2,1],s0=3,s1=0,p0=3,p1=0,d0=3,d1=0,is_2D=0","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[10,10,3,1],ne_kernel=[3,3,3,1],s0=1,s1=1,p0=1,p1=1,d0=1,d1=1,is_2D=1","support","1","yes","Vulkan" +"Vulkan0","IM2COL","type_input=f32,type_kernel=f32,dst_type=f16,ne_input=[10,10,3,1],ne_kernel=[3,3,3,1],s0=1,s1=1,p0=1,p1=1,d0=1,d1=1,is_2D=1","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f16,dst_type=f32,ne_input=[10,10,3,1],ne_kernel=[3,3,3,1],s0=1,s1=1,p0=1,p1=1,d0=1,d1=1,is_2D=1","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f16,dst_type=f16,ne_input=[10,10,3,1],ne_kernel=[3,3,3,1],s0=1,s1=1,p0=1,p1=1,d0=1,d1=1,is_2D=1","support","1","yes","Vulkan" "Vulkan0","IM2COL","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,20,2,2],ne_kernel=[3,3,2,2],s0=1,s1=1,p0=0,p1=0,d0=1,d1=1,is_2D=1","support","1","yes","Vulkan" @@ -3104,1835 +3643,3411 @@ "Vulkan0","IM2COL_3D","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,20,10,3],ne_kernel=[3,3,3,3],IC=3,s0=3,s1=3,s2=3,p0=3,p1=3,p2=3,d0=3,d1=3,d2=3,v=0","support","1","yes","Vulkan" "Vulkan0","IM2COL_3D","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,20,10,3],ne_kernel=[3,3,3,3],IC=3,s0=3,s1=3,s2=3,p0=3,p1=3,p2=3,d0=3,d1=3,d2=3,v=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","1","yes","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" "Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","Vulkan" -"Vulkan0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],stride=1,padding=0,dilation=1,cwhn=0","support","1","yes","Vulkan" -"Vulkan0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],stride=1,padding=0,dilation=1,cwhn=1","support","1","yes","Vulkan" -"Vulkan0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],stride=2,padding=1,dilation=1,cwhn=0","support","1","yes","Vulkan" -"Vulkan0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],stride=2,padding=1,dilation=1,cwhn=1","support","1","yes","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=4,ID=8,IH=8,IW=8,OC=8,KD=1,KH=1,KW=1,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","Vulkan" -"Vulkan0","CONV_3D","N=1,IC=4,ID=8,IH=8,IW=8,OC=8,KD=1,KH=1,KW=1,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","Vulkan" +"Vulkan0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=1","support","0","no","Vulkan" +"Vulkan0","CONV_2D","ne_input=[256,256,192,1],ne_kernel=[3,3,192,96],type_kernel=f32,stride0=1,stride1=1,padding0=1,padding1=1,dilation0=1,dilation1=1,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[256,256,192,1],ne_kernel=[3,3,192,96],type_kernel=f32,stride0=1,stride1=1,padding0=1,padding1=1,dilation0=1,dilation1=1,cwhn=1","support","0","no","Vulkan" +"Vulkan0","CONV_2D","ne_input=[256,256,192,1],ne_kernel=[3,3,192,96],type_kernel=f16,stride0=1,stride1=1,padding0=1,padding1=1,dilation0=1,dilation1=1,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D","ne_input=[256,256,192,1],ne_kernel=[3,3,192,96],type_kernel=f16,stride0=1,stride1=1,padding0=1,padding1=1,dilation0=1,dilation1=1,cwhn=1","support","0","no","Vulkan" +"Vulkan0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],type_kernel=f32,stride=1,padding=0,dilation=1,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],type_kernel=f32,stride=1,padding=0,dilation=1,cwhn=1","support","1","yes","Vulkan" +"Vulkan0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],type_kernel=f32,stride=2,padding=1,dilation=1,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],type_kernel=f32,stride=2,padding=1,dilation=1,cwhn=1","support","1","yes","Vulkan" +"Vulkan0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],type_kernel=f16,stride=1,padding=0,dilation=1,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],type_kernel=f16,stride=1,padding=0,dilation=1,cwhn=1","support","1","yes","Vulkan" +"Vulkan0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],type_kernel=f16,stride=2,padding=1,dilation=1,cwhn=0","support","1","yes","Vulkan" +"Vulkan0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],type_kernel=f16,stride=2,padding=1,dilation=1,cwhn=1","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=4,ID=8,IH=8,IW=8,OC=8,KD=1,KH=1,KW=1,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","Vulkan" +"Vulkan0","CONV_3D","N=1,IC=4,ID=8,IH=8,IW=8,OC=8,KD=1,KH=1,KW=1,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","Vulkan" "Vulkan0","CONV_TRANSPOSE_1D","ne_input=[1,1,1,1],ne_kernel=[1,1,1,1],s0=1,p0=0,d0=1","support","1","yes","Vulkan" "Vulkan0","CONV_TRANSPOSE_1D","ne_input=[1,1,1,1],ne_kernel=[1,1,1,1],s0=2,p0=0,d0=1","support","1","yes","Vulkan" "Vulkan0","CONV_TRANSPOSE_1D","ne_input=[1,1,1,1],ne_kernel=[1,1,1,1],s0=3,p0=0,d0=1","support","1","yes","Vulkan" @@ -5049,6 +7164,39 @@ "Vulkan0","CONV_TRANSPOSE_1D","ne_input=[3,2,1,1],ne_kernel=[3,2,2,1],s0=1,p0=0,d0=1","support","1","yes","Vulkan" "Vulkan0","CONV_TRANSPOSE_1D","ne_input=[3,2,1,1],ne_kernel=[3,1,2,1],s0=1,p0=0,d0=1","support","1","yes","Vulkan" "Vulkan0","CONV_TRANSPOSE_1D","ne_input=[2,1,1,1],ne_kernel=[3,1,1,1],s0=1,p0=0,d0=1","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=16,OC=32,T_in=197,s0=8,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=4,OC=3,T_in=7,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=1,OC=5,T_in=13,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=6,OC=4,T_in=11,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=2,OC=3,T_in=9,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=5,OC=4,T_in=11,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=8,OC=4,T_in=13,s0=4,p0=2","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=4,OC=3,T_in=1,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=16,OC=1,T_in=197,s0=8,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=1,OC=5,T_in=13,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f32,K=8,OC=2,T_in=3,s0=2,p0=5","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=16,OC=32,T_in=197,s0=8,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=4,OC=3,T_in=7,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=1,OC=5,T_in=13,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=6,OC=4,T_in=11,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=2,OC=3,T_in=9,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=5,OC=4,T_in=11,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=8,OC=4,T_in=13,s0=4,p0=2","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=4,OC=3,T_in=1,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=16,OC=1,T_in=197,s0=8,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=1,OC=5,T_in=13,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=f16,K=8,OC=2,T_in=3,s0=2,p0=5","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=16,OC=32,T_in=197,s0=8,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=4,OC=3,T_in=7,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=1,OC=5,T_in=13,s0=1,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=6,OC=4,T_in=11,s0=3,p0=1","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=2,OC=3,T_in=9,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=5,OC=4,T_in=11,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=8,OC=4,T_in=13,s0=4,p0=2","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=4,OC=3,T_in=1,s0=2,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=16,OC=1,T_in=197,s0=8,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=1,OC=5,T_in=13,s0=3,p0=0","support","1","yes","Vulkan" +"Vulkan0","COL2IM_1D","type=bf16,K=8,OC=2,T_in=3,s0=2,p0=5","support","1","yes","Vulkan" "Vulkan0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[3,2,3,1],ne_kernel=[2,2,1,3],stride=1","support","1","yes","Vulkan" "Vulkan0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[10,10,9,1],ne_kernel=[3,3,1,9],stride=2","support","1","yes","Vulkan" "Vulkan0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[129,63,35,1],ne_kernel=[3,3,48,35],stride=1","support","1","yes","Vulkan" @@ -5071,6 +7219,7 @@ "Vulkan0","REPEAT","type=f32,ne=[10,5,4,1],nr=[1,1,1,2]","support","1","yes","Vulkan" "Vulkan0","REPEAT","type=i32,ne=[10,5,4,1],nr=[2,1,1,1]","support","1","yes","Vulkan" "Vulkan0","REPEAT","type=i16,ne=[10,5,4,1],nr=[1,1,1,2]","support","1","yes","Vulkan" +"Vulkan0","REPEAT","type=bf16,ne=[10,5,4,1],nr=[2,1,1,1]","support","1","yes","Vulkan" "Vulkan0","REPEAT","type=f32,ne=[10,5,4,3],nr=[1,1,1,1]","support","1","yes","Vulkan" "Vulkan0","REPEAT","type=f32,ne=[10,5,4,3],nr=[2,1,1,1]","support","1","yes","Vulkan" "Vulkan0","REPEAT","type=f32,ne=[10,5,4,3],nr=[1,2,1,1]","support","1","yes","Vulkan" @@ -5078,6 +7227,7 @@ "Vulkan0","REPEAT","type=f32,ne=[10,5,4,3],nr=[1,1,1,2]","support","1","yes","Vulkan" "Vulkan0","REPEAT","type=i32,ne=[10,5,4,3],nr=[2,1,1,1]","support","1","yes","Vulkan" "Vulkan0","REPEAT","type=i16,ne=[10,5,4,3],nr=[1,1,1,2]","support","1","yes","Vulkan" +"Vulkan0","REPEAT","type=bf16,ne=[10,5,4,3],nr=[2,1,1,1]","support","1","yes","Vulkan" "Vulkan0","REPEAT_BACK","type=f32,ne=[8,6,4,2],nr=[1,1,1,1],v=0","support","1","yes","Vulkan" "Vulkan0","REPEAT_BACK","type=f32,ne=[8,6,4,2],nr=[2,1,1,1],v=0","support","1","yes","Vulkan" "Vulkan0","REPEAT_BACK","type=f32,ne=[8,6,4,2],nr=[1,2,1,1],v=0","support","1","yes","Vulkan" @@ -5191,6 +7341,15 @@ "Vulkan0","CPY","type_src=q1_0,type_dst=q1_0,ne_src=[384,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=q1_0,type_dst=q1_0,ne_src=[384,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=q1_0,type_dst=q1_0,ne_src=[384,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=q2_0,ne_src=[64,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=q2_0,ne_src=[64,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=q2_0,ne_src=[64,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=q2_0,ne_src=[128,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=q2_0,ne_src=[128,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=q2_0,ne_src=[128,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=q2_0,ne_src=[192,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=q2_0,ne_src=[192,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=q2_0,ne_src=[192,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=mxfp4,type_dst=mxfp4,ne_src=[32,2,3,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=mxfp4,type_dst=mxfp4,ne_src=[32,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=mxfp4,type_dst=mxfp4,ne_src=[32,2,3,4],permute_src=[0,3,1,2],permute_dst=[0,2,1,3],_src_transpose=0","support","0","no","Vulkan" @@ -5353,6 +7512,8 @@ "Vulkan0","CPY","type_src=f16,type_dst=q8_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=f16,type_dst=q1_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=f16,type_dst=q1_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" +"Vulkan0","CPY","type_src=f16,type_dst=q2_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" +"Vulkan0","CPY","type_src=f16,type_dst=q2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=f16,type_dst=mxfp4,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=f16,type_dst=mxfp4,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=f16,type_dst=nvfp4,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" @@ -5403,6 +7564,8 @@ "Vulkan0","CPY","type_src=bf16,type_dst=q8_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=bf16,type_dst=q1_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=bf16,type_dst=q1_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" +"Vulkan0","CPY","type_src=bf16,type_dst=q2_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" +"Vulkan0","CPY","type_src=bf16,type_dst=q2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=bf16,type_dst=mxfp4,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=bf16,type_dst=mxfp4,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=bf16,type_dst=nvfp4,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" @@ -5453,6 +7616,8 @@ "Vulkan0","CPY","type_src=f32,type_dst=q8_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=f32,type_dst=q1_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=f32,type_dst=q1_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=f32,type_dst=q2_0,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=f32,type_dst=q2_0,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=f32,type_dst=mxfp4,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=f32,type_dst=mxfp4,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=f32,type_dst=nvfp4,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" @@ -5503,6 +7668,8 @@ "Vulkan0","CPY","type_src=q8_0,type_dst=f32,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=q1_0,type_dst=f32,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=q1_0,type_dst=f32,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=f32,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=q2_0,type_dst=f32,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=mxfp4,type_dst=f32,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=mxfp4,type_dst=f32,ne_src=[256,2,3,4],permute_src=[0,2,1,3],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" "Vulkan0","CPY","type_src=nvfp4,type_dst=f32,ne_src=[256,4,4,4],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","0","no","Vulkan" @@ -5553,6 +7720,10 @@ "Vulkan0","CPY","type_src=i32,type_dst=i32,ne_src=[256,4,1,1],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=1","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=i32,type_dst=i32,ne_src=[256,1,4,1],permute_src=[1,2,0,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=f32,type_dst=f32,ne_src=[256,1,4,1],permute_src=[1,2,0,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=f32,type_dst=f32,ne_src=[2,2097121,1,1],permute_src=[1,0,2,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=f32,type_dst=f32,ne_src=[2,2,524281,1],permute_src=[1,0,2,3],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=f32,type_dst=f32,ne_src=[128,2,3,1],ne_dst=[128,2,3,1],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0,dst_alloc=[128,4,3,1]","support","1","yes","Vulkan" +"Vulkan0","CPY","type_src=f16,type_dst=f16,ne_src=[128,2,3,1],ne_dst=[128,2,3,1],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0,dst_alloc=[128,4,3,1]","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=f32,type_dst=f32,ne_src=[3,5,7,32],ne_dst=[32,7,5,3],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=f32,type_dst=f32,ne_src=[3,5,32,7],ne_dst=[3,5,7,32],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" "Vulkan0","CPY","type_src=f32,type_dst=f32,ne_src=[3,5,32,7],ne_dst=[32,7,5,3],permute_src=[0,0,0,0],permute_dst=[0,0,0,0],_src_transpose=0","support","1","yes","Vulkan" @@ -6052,76 +8223,96 @@ "Vulkan0","SCALE","type=f32,ne=[10,10,10,10],scale=2.000000,bias=1.000000,inplace=1","support","1","yes","Vulkan" "Vulkan0","SCALE","type=f32,ne=[100,10,10,10],scale=2.000000,bias=1.000000,inplace=0","support","1","yes","Vulkan" "Vulkan0","SILU_BACK","type=f32,ne=[64,5,4,3],eps=0.000001","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000000","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000000,inplace=0","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.000000","support","0","no","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.000000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.000000,inplace=0","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000000,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM_BACK","type=f32,ne=[64,5,4,3],eps=0.000000","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000000,v=0","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000000,v=1","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000000","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000000,v=0,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000000,v=1,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000000,v=0,noncontig_rows=1","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000000,inplace=0","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.000000","support","0","no","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.000000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.000000,inplace=0","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000000,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM_BACK","type=f32,ne=[1025,5,4,3],eps=0.000000","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000000,v=0","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000000,v=1","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000001","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000000,v=0,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000000,v=1,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000000,v=0,noncontig_rows=1","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000001,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000001,inplace=0","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.000001","support","0","no","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.000001,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.000001,inplace=0","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000001,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM_BACK","type=f32,ne=[64,5,4,3],eps=0.000001","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000001,v=0","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000001,v=1","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000001","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000001,v=0,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000001,v=1,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000001,v=0,noncontig_rows=1","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000001,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000001,inplace=0","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.000001","support","0","no","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.000001,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.000001,inplace=0","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000001,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM_BACK","type=f32,ne=[1025,5,4,3],eps=0.000001","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000001,v=0","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000001,v=1","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000100","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000001,v=0,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000001,v=1,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000001,v=0,noncontig_rows=1","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000100,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000100,inplace=0","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.000100","support","0","no","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.000100,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.000100,inplace=0","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000100,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM_BACK","type=f32,ne=[64,5,4,3],eps=0.000100","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000100,v=0","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000100,v=1","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000100","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000100,v=0,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000100,v=1,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.000100,v=0,noncontig_rows=1","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000100,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000100,inplace=0","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.000100","support","0","no","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.000100,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.000100,inplace=0","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.000100,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM_BACK","type=f32,ne=[1025,5,4,3],eps=0.000100","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000100,v=0","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000100,v=1","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.100000","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000100,v=0,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000100,v=1,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.000100,v=0,noncontig_rows=1","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.100000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.100000,inplace=0","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.100000","support","0","no","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.100000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=1,eps=0.100000,inplace=0","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.100000,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM_BACK","type=f32,ne=[64,5,4,3],eps=0.100000","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.100000,v=0","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.100000,v=1","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.100000","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.100000,v=0,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.100000,v=1,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=0.100000,v=0,noncontig_rows=1","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.100000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.100000,inplace=0","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.100000","support","0","no","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.100000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[1025,5,4,3],v=1,eps=0.100000,inplace=0","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=0.100000,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM_BACK","type=f32,ne=[1025,5,4,3],eps=0.100000","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.100000,v=0","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.100000,v=1","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=10.000000","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.100000,v=0,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.100000,v=1,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=0.100000,v=0,noncontig_rows=1","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=10.000000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=0,eps=10.000000,inplace=0","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=1,eps=10.000000","support","0","no","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=1,eps=10.000000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=1,eps=10.000000,inplace=0","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[64,5,4,3],v=0,eps=10.000000,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM_BACK","type=f32,ne=[64,5,4,3],eps=10.000000","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=10.000000,v=0","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=10.000000,v=1","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=10.000000","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=10.000000,v=0,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=10.000000,v=1,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[64,5,4,3],eps=10.000000,v=0,noncontig_rows=1","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=10.000000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[1025,5,4,3],v=0,eps=10.000000,inplace=0","support","1","yes","Vulkan" -"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=1,eps=10.000000","support","0","no","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=1,eps=10.000000,noncontig_rows=0","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[1025,5,4,3],v=1,eps=10.000000,inplace=0","support","1","yes","Vulkan" +"Vulkan0","NORM","type=f32,ne=[1025,5,4,3],v=0,eps=10.000000,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM_BACK","type=f32,ne=[1025,5,4,3],eps=10.000000","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=10.000000,v=0","support","1","yes","Vulkan" -"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=10.000000,v=1","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=10.000000,v=0,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=10.000000,v=1,noncontig_rows=0","support","1","yes","Vulkan" +"Vulkan0","L2_NORM","type=f32,ne=[1025,5,4,3],eps=10.000000,v=0,noncontig_rows=1","support","1","yes","Vulkan" "Vulkan0","RMS_NORM","type=f32,ne=[64,5,4,3],v=0,eps=0.000001,inplace=1","support","1","yes","Vulkan" "Vulkan0","SSM_CONV","type=f32,ne_a=[3,1024,1,1],ne_b=[3,1024,1,1]","support","1","yes","Vulkan" "Vulkan0","SSM_CONV","type=f32,ne_a=[6,1024,1,1],ne_b=[3,1024,1,1]","support","1","yes","Vulkan" @@ -6172,6 +8363,9 @@ "Vulkan0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=16,n_group=2,n_seq_tokens=32,n_seqs=4,xbc_overlap=0","support","1","yes","Vulkan" "Vulkan0","SSM_SCAN","type=f32,d_state=256,head_dim=64,n_head=8,n_group=2,n_seq_tokens=32,n_seqs=4,xbc_overlap=0","support","1","yes","Vulkan" "Vulkan0","SSM_SCAN","type=f32,d_state=128,head_dim=128,n_head=4,n_group=4,n_seq_tokens=16,n_seqs=2,xbc_overlap=1","support","1","yes","Vulkan" +"Vulkan0","SSM_SCAN","type=f32,d_state=128,head_dim=80,n_head=128,n_group=1,n_seq_tokens=256,n_seqs=1,xbc_overlap=0","support","1","yes","Vulkan" +"Vulkan0","SSM_SCAN","type=f32,d_state=128,head_dim=80,n_head=128,n_group=1,n_seq_tokens=512,n_seqs=1,xbc_overlap=0","support","1","yes","Vulkan" +"Vulkan0","SSM_SCAN","type=f32,d_state=128,head_dim=64,n_head=80,n_group=8,n_seq_tokens=300,n_seqs=2,xbc_overlap=0","support","1","yes","Vulkan" "Vulkan0","RWKV_WKV6","type=f32,head_count=32,head_size=64,n_seq_tokens=1,n_seqs=1","support","1","yes","Vulkan" "Vulkan0","RWKV_WKV6","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=1","support","1","yes","Vulkan" "Vulkan0","RWKV_WKV6","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=4","support","1","yes","Vulkan" @@ -6180,16 +8374,19 @@ "Vulkan0","RWKV_WKV7","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=1","support","1","yes","Vulkan" "Vulkan0","RWKV_WKV7","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=4","support","1","yes","Vulkan" "Vulkan0","RWKV_WKV7","type=f32,head_count=32,head_size=64,n_seq_tokens=128,n_seqs=4","support","1","yes","Vulkan" -"Vulkan0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=1,n_seqs=1","support","0","no","Vulkan" -"Vulkan0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=1","support","0","no","Vulkan" -"Vulkan0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=4","support","0","no","Vulkan" -"Vulkan0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=128,n_seqs=4","support","0","no","Vulkan" +"Vulkan0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=1,n_seqs=1","support","1","yes","Vulkan" +"Vulkan0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=1","support","1","yes","Vulkan" +"Vulkan0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=32,n_seqs=4","support","1","yes","Vulkan" +"Vulkan0","GATED_LINEAR_ATTN","type=f32,head_count=32,head_size=64,n_seq_tokens=128,n_seqs=4","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=4,k=128,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=32,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=1024,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" @@ -6271,6 +8468,15 @@ "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=4,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=5,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=6,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=2,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=3,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" @@ -6415,6 +8621,16 @@ "Vulkan0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=7,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=8,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=iq4_xs,type_b=f32,m=16,n=9,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=2880,n=32,k=2880,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=2880,n=32,k=2880,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=2880,n=32,k=2880,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=1,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=7,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=8,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=9,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=16,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=128,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=1,n=512,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6424,6 +8640,7 @@ "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6436,6 +8653,7 @@ "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" @@ -6451,6 +8669,7 @@ "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6463,6 +8682,7 @@ "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" @@ -6482,6 +8702,7 @@ "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6494,6 +8715,7 @@ "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" @@ -6509,6 +8731,7 @@ "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6521,6 +8744,7 @@ "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f32,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" @@ -6540,6 +8764,7 @@ "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6552,6 +8777,7 @@ "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" @@ -6567,6 +8793,7 @@ "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6579,6 +8806,7 @@ "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f32,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" @@ -6598,6 +8826,7 @@ "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6610,6 +8839,7 @@ "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" @@ -6625,6 +8855,7 @@ "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=4,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=16,k=4,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6637,6 +8868,7 @@ "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=1,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=4,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=f16,type_b=f16,m=16,n=8,k=4,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","1","yes","Vulkan" @@ -6656,6 +8888,7 @@ "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6668,6 +8901,7 @@ "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6687,6 +8921,7 @@ "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6699,6 +8934,7 @@ "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6718,6 +8954,7 @@ "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6730,6 +8967,7 @@ "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6749,6 +8987,7 @@ "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6761,6 +9000,7 @@ "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6771,6 +9011,72 @@ "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=1,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=16,k=1024,bs=[3,2],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f16,m=16,n=8,k=256,bs=[1536,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6780,6 +9086,7 @@ "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6792,6 +9099,7 @@ "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6811,6 +9119,7 @@ "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6823,6 +9132,7 @@ "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_0,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6842,6 +9152,7 @@ "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6854,6 +9165,7 @@ "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6873,6 +9185,7 @@ "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6885,6 +9198,7 @@ "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_1,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6904,6 +9218,7 @@ "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6916,6 +9231,7 @@ "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6935,6 +9251,7 @@ "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6947,6 +9264,7 @@ "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=q4_K,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6966,6 +9284,7 @@ "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -6978,6 +9297,7 @@ "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -6997,6 +9317,7 @@ "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -7009,6 +9330,7 @@ "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -7028,6 +9350,7 @@ "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -7040,6 +9363,7 @@ "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -7059,6 +9383,7 @@ "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -7071,6 +9396,7 @@ "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -7090,6 +9416,7 @@ "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -7102,6 +9429,7 @@ "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f32,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -7121,6 +9449,7 @@ "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=4,k=256,bs=[3,2],nr=[2,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[2,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=16,k=256,bs=[1,1],nr=[1,2],per=[0,1,2,3],k_v=0,o=1","support","0","no","Vulkan" @@ -7133,6 +9462,7 @@ "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=1,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" +"Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=4,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,2,1,3],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,1,3,2],k_v=0,o=1","support","0","no","Vulkan" "Vulkan0","MUL_MAT","type_a=iq2_xxs,type_b=f16,m=16,n=8,k=256,bs=[2,3],nr=[1,1],per=[0,3,2,1],k_v=0,o=1","support","0","no","Vulkan" @@ -7162,6 +9492,8 @@ "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q2_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q3_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q5_K,type_b=f32,m=16,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" @@ -7197,6 +9529,7 @@ "Vulkan0","MUL_MAT","type_a=q5_1,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q8_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q1_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT","type_a=q2_0,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=mxfp4,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=nvfp4,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT","type_a=q2_K,type_b=f32,m=1,n=64,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1","support","1","yes","Vulkan" @@ -7606,6 +9939,33 @@ "Vulkan0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=16,n_used=16,b=1,m=50,n=200,k=64","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=1,n_used=1,b=0,m=8,n=16,k=1","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=32,n_used=2,b=0,m=2880,n=32,k=2880","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=32,n_used=2,b=0,m=2880,n=32,k=2880","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=3","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=f16,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=3","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=3","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q4_1,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q5_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q5_1,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=384","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=192","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=mxfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=nvfp4,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=192","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q3_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q4_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q5_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q6_K,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=iq2_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=iq2_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=iq2_s,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=iq3_xxs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=iq1_s,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=iq1_m,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=iq4_nl,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=96","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=iq3_s,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=iq4_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=64,n=16,k=768","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=f32,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","Vulkan" @@ -7894,6 +10254,78 @@ "Vulkan0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=0,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=1,b=1,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=0,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=2,b=1,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=0,m=512,n=129,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=4,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=5,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=17,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=8,n_used=4,b=1,m=512,n=129,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=4,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=q4_0,type_b=f32,n_mats=4,n_used=1,b=0,m=512,n=5,k=256","support","1","yes","Vulkan" @@ -8336,6 +10768,8 @@ "Vulkan0","MUL_MAT_ID","type_a=q8_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=q1_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" +"Vulkan0","MUL_MAT_ID","type_a=q2_0,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=q2_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=q2_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=q3_K,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" @@ -8362,70 +10796,70 @@ "Vulkan0","MUL_MAT_ID","type_a=iq4_xs,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=1,k=256","support","1","yes","Vulkan" "Vulkan0","MUL_MAT_ID","type_a=bf16,type_b=f32,n_mats=4,n_used=2,b=0,m=512,n=32,k=256","support","1","yes","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[2,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[1,2],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[2,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[2,2],trans_b=0","support","1","yes","Vulkan" "Vulkan0","OUT_PROD","type_a=f32,type_b=f16,m=256,n=1,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" "Vulkan0","OUT_PROD","type_a=f32,type_b=f16,m=256,n=1,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" "Vulkan0","OUT_PROD","type_a=f32,type_b=f16,m=256,n=1,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" @@ -8874,6 +11308,134 @@ "Vulkan0","OUT_PROD","type_a=q1_0,type_b=f16,m=256,n=16,k=16,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" "Vulkan0","OUT_PROD","type_a=q1_0,type_b=f16,m=256,n=16,k=16,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" "Vulkan0","OUT_PROD","type_a=q1_0,type_b=f16,m=256,n=16,k=16,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=1,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=1,k=16,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=1,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f32,m=256,n=16,k=16,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=1,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=1,k=16,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=1,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[1,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[1,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[1,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[1,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[1,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[3,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[3,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[3,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[3,1],nr=[2,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[3,3],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=q2_0,type_b=f16,m=256,n=16,k=16,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" "Vulkan0","OUT_PROD","type_a=q4_0,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" "Vulkan0","OUT_PROD","type_a=q4_0,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[1,2],trans_b=0","support","0","no","Vulkan" "Vulkan0","OUT_PROD","type_a=q4_0,type_b=f32,m=256,n=1,k=1,bs=[1,1],nr=[2,1],trans_b=0","support","0","no","Vulkan" @@ -9642,10 +12204,13 @@ "Vulkan0","OUT_PROD","type_a=iq2_xxs,type_b=f16,m=256,n=16,k=16,bs=[3,3],nr=[1,2],trans_b=0","support","0","no","Vulkan" "Vulkan0","OUT_PROD","type_a=iq2_xxs,type_b=f16,m=256,n=16,k=16,bs=[3,3],nr=[2,1],trans_b=0","support","0","no","Vulkan" "Vulkan0","OUT_PROD","type_a=iq2_xxs,type_b=f16,m=256,n=16,k=16,bs=[3,3],nr=[2,2],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[8,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[16,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" -"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[32,1],nr=[1,1],trans_b=0","support","0","no","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[8,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[16,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[32,1],nr=[1,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[8,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[16,1],trans_b=0","support","1","yes","Vulkan" +"Vulkan0","OUT_PROD","type_a=f32,type_b=f32,m=256,n=16,k=16,bs=[1,1],nr=[32,1],trans_b=0","support","1","yes","Vulkan" "Vulkan0","ADD_ID","type_a=f32,type_b=f32,n_embd=32,n_experts=4,n_experts_used=1,n_token=1","support","1","yes","Vulkan" "Vulkan0","ADD_ID","type_a=f32,type_b=f32,n_embd=32,n_experts=4,n_experts_used=1,n_token=32","support","1","yes","Vulkan" "Vulkan0","ADD_ID","type_a=f32,type_b=f32,n_embd=32,n_experts=4,n_experts_used=1,n_token=129","support","1","yes","Vulkan" @@ -9682,31 +12247,31 @@ "Vulkan0","ADD_ID","type_a=f32,type_b=f32,n_embd=129,n_experts=8,n_experts_used=4,n_token=1","support","1","yes","Vulkan" "Vulkan0","ADD_ID","type_a=f32,type_b=f32,n_embd=129,n_experts=8,n_experts_used=4,n_token=32","support","1","yes","Vulkan" "Vulkan0","ADD_ID","type_a=f32,type_b=f32,n_embd=129,n_experts=8,n_experts_used=4,n_token=129","support","1","yes","Vulkan" -"Vulkan0","SQR","type=f16,ne=[10,5,4,3]","support","0","no","Vulkan" -"Vulkan0","SQRT","type=f16,ne=[10,3,3,2]","support","0","no","Vulkan" +"Vulkan0","SQR","type=f16,ne=[10,5,4,3]","support","1","yes","Vulkan" +"Vulkan0","SQRT","type=f16,ne=[10,3,3,2]","support","1","yes","Vulkan" "Vulkan0","LOG","type=f16,ne=[10,5,4,3]","support","1","yes","Vulkan" -"Vulkan0","SIN","type=f16,ne=[10,2,2,2]","support","0","no","Vulkan" -"Vulkan0","COS","type=f16,ne=[10,2,2,2]","support","0","no","Vulkan" -"Vulkan0","CLAMP","type=f16,ne=[10,5,4,3],min=-0.500000,max=0.500000","support","0","no","Vulkan" -"Vulkan0","LEAKY_RELU","type=f16,ne_a=[10,5,4,3],negative_slope=0.100000","support","0","no","Vulkan" +"Vulkan0","SIN","type=f16,ne=[10,2,2,2]","support","1","yes","Vulkan" +"Vulkan0","COS","type=f16,ne=[10,2,2,2]","support","1","yes","Vulkan" +"Vulkan0","CLAMP","type=f16,ne=[10,5,4,3],min=-0.500000,max=0.500000","support","1","yes","Vulkan" +"Vulkan0","LEAKY_RELU","type=f16,ne_a=[10,5,4,3],negative_slope=0.100000","support","1","yes","Vulkan" "Vulkan0","FLOOR","type=f16,ne=[10,2,2,2]","support","1","yes","Vulkan" "Vulkan0","CEIL","type=f16,ne=[10,2,2,2]","support","1","yes","Vulkan" "Vulkan0","ROUND","type=f16,ne=[10,2,2,2]","support","1","yes","Vulkan" "Vulkan0","TRUNC","type=f16,ne=[10,2,2,2]","support","1","yes","Vulkan" -"Vulkan0","SQR","type=f16,ne=[7,1,5,3]","support","0","no","Vulkan" -"Vulkan0","SQR","type=f16,ne=[1024,1024,1,1]","support","0","no","Vulkan" -"Vulkan0","SQRT","type=f16,ne=[7,1,5,3]","support","0","no","Vulkan" -"Vulkan0","SQRT","type=f16,ne=[1024,1024,1,1]","support","0","no","Vulkan" +"Vulkan0","SQR","type=f16,ne=[7,1,5,3]","support","1","yes","Vulkan" +"Vulkan0","SQR","type=f16,ne=[1024,1024,1,1]","support","1","yes","Vulkan" +"Vulkan0","SQRT","type=f16,ne=[7,1,5,3]","support","1","yes","Vulkan" +"Vulkan0","SQRT","type=f16,ne=[1024,1024,1,1]","support","1","yes","Vulkan" "Vulkan0","LOG","type=f16,ne=[7,1,5,3]","support","1","yes","Vulkan" "Vulkan0","LOG","type=f16,ne=[1024,1024,1,1]","support","1","yes","Vulkan" -"Vulkan0","SIN","type=f16,ne=[7,1,5,3]","support","0","no","Vulkan" -"Vulkan0","SIN","type=f16,ne=[1024,1024,1,1]","support","0","no","Vulkan" -"Vulkan0","COS","type=f16,ne=[7,1,5,3]","support","0","no","Vulkan" -"Vulkan0","COS","type=f16,ne=[1024,1024,1,1]","support","0","no","Vulkan" -"Vulkan0","CLAMP","type=f16,ne=[7,1,5,3],min=-0.500000,max=0.500000","support","0","no","Vulkan" -"Vulkan0","CLAMP","type=f16,ne=[1024,1024,1,1],min=-0.500000,max=0.500000","support","0","no","Vulkan" -"Vulkan0","LEAKY_RELU","type=f16,ne_a=[7,1,5,3],negative_slope=0.100000","support","0","no","Vulkan" -"Vulkan0","LEAKY_RELU","type=f16,ne_a=[1024,1024,1,1],negative_slope=0.100000","support","0","no","Vulkan" +"Vulkan0","SIN","type=f16,ne=[7,1,5,3]","support","1","yes","Vulkan" +"Vulkan0","SIN","type=f16,ne=[1024,1024,1,1]","support","1","yes","Vulkan" +"Vulkan0","COS","type=f16,ne=[7,1,5,3]","support","1","yes","Vulkan" +"Vulkan0","COS","type=f16,ne=[1024,1024,1,1]","support","1","yes","Vulkan" +"Vulkan0","CLAMP","type=f16,ne=[7,1,5,3],min=-0.500000,max=0.500000","support","1","yes","Vulkan" +"Vulkan0","CLAMP","type=f16,ne=[1024,1024,1,1],min=-0.500000,max=0.500000","support","1","yes","Vulkan" +"Vulkan0","LEAKY_RELU","type=f16,ne_a=[7,1,5,3],negative_slope=0.100000","support","1","yes","Vulkan" +"Vulkan0","LEAKY_RELU","type=f16,ne_a=[1024,1024,1,1],negative_slope=0.100000","support","1","yes","Vulkan" "Vulkan0","FLOOR","type=f16,ne=[7,1,5,3]","support","1","yes","Vulkan" "Vulkan0","FLOOR","type=f16,ne=[1024,1024,1,1]","support","1","yes","Vulkan" "Vulkan0","CEIL","type=f16,ne=[7,1,5,3]","support","1","yes","Vulkan" @@ -10848,37 +13413,197 @@ "Vulkan0","ROPE","type=f16,ne_a=[128,32,2,1],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","Vulkan" "Vulkan0","ROPE","type=f16,ne_a=[128,32,2,3],n_dims=128,mode=24,n_ctx=512,fs=1.424500,ef=0.746500,af=1.424500,ff=1,v=1,inplace=1","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=0","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=0","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=1","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=1","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=1","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=1","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=1","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=1","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=1","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=1","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=1","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=2","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=2","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=2","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=2","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=2","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=2","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=2","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=2","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=2","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=3","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=0,v=3","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=3","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=1,v=3","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=3","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=2,v=3","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=f32,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=f16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=bf16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i8,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i16,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=3","support","1","yes","Vulkan" "Vulkan0","CONCAT","type=i32,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=i64,ne_a=[11,12,13,14],ne_b_d=7,dim=3,v=3","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q4_1,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q5_1,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=0","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=4","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=8","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=256,dim=0,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=1,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=2,v=12","support","1","yes","Vulkan" +"Vulkan0","CONCAT","type=q8_0,ne_a=[128,12,13,14],ne_b_d=7,dim=3,v=12","support","1","yes","Vulkan" "Vulkan0","ARGSORT","type=f32,ne=[3,1,1,1],order=0","support","1","yes","Vulkan" "Vulkan0","ARGSORT","type=f32,ne=[4,1,1,1],order=0","support","1","yes","Vulkan" "Vulkan0","ARGSORT","type=f32,ne=[7,1,1,1],order=0","support","1","yes","Vulkan" @@ -11426,6 +14151,42 @@ "Vulkan0","PAD","type=f32,ne_a=[11,22,33,44],lp0=1,rp0=2,lp1=3,rp1=4,lp2=5,rp2=6,lp3=7,rp3=8,tfrm=2,circular=0","support","1","yes","Vulkan" "Vulkan0","PAD","type=f32,ne_a=[512,512,1,1],lp0=0,rp0=1,lp1=0,rp1=1,lp2=0,rp2=0,lp3=0,rp3=0,tfrm=2,circular=1","support","1","yes","Vulkan" "Vulkan0","PAD","type=f32,ne_a=[11,22,33,44],lp0=1,rp0=2,lp1=3,rp1=4,lp2=5,rp2=6,lp3=7,rp3=8,tfrm=2,circular=1","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=1024,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q8_0,type_V=q8_0,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=8,nr23=[4,1],kv=2048,nb=64,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=40,hsv=40,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11658,7 +14419,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11667,7 +14428,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11676,7 +14437,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11685,7 +14446,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11702,8 +14463,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11720,8 +14481,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11738,8 +14499,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11756,8 +14517,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11766,7 +14527,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11775,7 +14536,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11784,7 +14545,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11793,7 +14554,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11810,8 +14571,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11828,8 +14589,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11846,8 +14607,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11864,8 +14625,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11874,7 +14635,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11883,7 +14644,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11892,7 +14653,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11901,7 +14662,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11918,8 +14679,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11936,8 +14697,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11954,8 +14715,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11972,8 +14733,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11982,7 +14743,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -11991,7 +14752,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12000,7 +14761,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12009,7 +14770,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12026,8 +14787,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12044,8 +14805,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12062,8 +14823,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12080,8 +14841,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12090,7 +14851,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12099,7 +14860,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12108,7 +14869,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12117,7 +14878,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12126,7 +14887,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12135,7 +14896,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12144,7 +14905,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12153,7 +14914,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12162,7 +14923,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12171,7 +14932,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12180,7 +14941,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12189,7 +14950,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12198,7 +14959,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12207,7 +14968,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12216,7 +14977,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12225,7 +14986,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12234,7 +14995,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12243,7 +15004,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12252,7 +15013,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12261,7 +15022,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12270,7 +15031,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12279,7 +15040,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12288,7 +15049,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12297,7 +15058,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12306,7 +15067,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12315,7 +15076,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12324,7 +15085,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12333,7 +15094,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12342,7 +15103,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12351,7 +15112,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12360,7 +15121,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12369,7 +15130,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12378,7 +15139,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12387,7 +15148,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12396,7 +15157,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12405,7 +15166,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12422,8 +15183,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12440,8 +15201,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12458,8 +15219,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12476,8 +15237,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12486,7 +15247,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12495,7 +15256,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12504,7 +15265,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12513,7 +15274,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12530,8 +15291,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12548,8 +15309,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12566,8 +15327,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12584,8 +15345,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12594,7 +15355,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12603,7 +15364,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12612,7 +15373,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12621,7 +15382,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12638,8 +15399,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12656,8 +15417,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12674,8 +15435,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12692,8 +15453,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12702,7 +15463,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12711,7 +15472,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12720,7 +15481,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12729,7 +15490,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12746,8 +15507,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12764,8 +15525,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12782,8 +15543,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12800,8 +15561,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12810,7 +15571,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12819,7 +15580,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12828,7 +15589,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12837,7 +15598,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12846,7 +15607,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12855,7 +15616,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12864,7 +15625,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12873,7 +15634,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12882,7 +15643,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12891,7 +15652,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12900,7 +15661,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12909,7 +15670,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12918,7 +15679,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12927,7 +15688,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12936,7 +15697,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12945,7 +15706,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12954,7 +15715,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12963,7 +15724,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12972,7 +15733,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12981,7 +15742,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12990,7 +15751,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -12999,7 +15760,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13008,7 +15769,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13017,7 +15778,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13026,7 +15787,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13035,7 +15796,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13044,7 +15805,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13053,7 +15814,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13062,7 +15823,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13071,7 +15832,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13080,7 +15841,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13089,7 +15850,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13098,7 +15859,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13107,7 +15868,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13116,7 +15877,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13125,7 +15886,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13134,7 +15895,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13143,7 +15904,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13152,7 +15913,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13161,7 +15922,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13170,7 +15931,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13179,7 +15940,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13188,7 +15949,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13197,7 +15958,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13206,7 +15967,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13215,7 +15976,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13224,7 +15985,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13233,7 +15994,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13242,7 +16003,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13251,7 +16012,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13260,7 +16021,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13269,7 +16030,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13278,7 +16039,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13287,7 +16048,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13296,7 +16057,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13305,7 +16066,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13314,7 +16075,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13323,7 +16084,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13332,7 +16093,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13341,7 +16102,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13350,7 +16111,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13359,7 +16120,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13368,7 +16129,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13377,7 +16138,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13386,7 +16147,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13395,7 +16156,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13404,7 +16165,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13413,7 +16174,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13422,7 +16183,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13431,7 +16192,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13440,7 +16201,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13449,7 +16210,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13458,7 +16219,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13467,7 +16228,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13476,7 +16237,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13485,7 +16246,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13494,7 +16255,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13503,7 +16264,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13512,7 +16273,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13521,7 +16282,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13530,7 +16291,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13539,7 +16300,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13548,7 +16309,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13557,7 +16318,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13566,7 +16327,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13575,7 +16336,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13584,7 +16345,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13593,7 +16354,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13602,7 +16363,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13611,7 +16372,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13620,7 +16381,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13629,7 +16390,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[1,3],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13638,7 +16399,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13647,7 +16408,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13656,7 +16417,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13665,7 +16426,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=64,nh=4,nr23=[4,3],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13674,7 +16435,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13683,7 +16444,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13692,7 +16453,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13701,7 +16462,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13718,8 +16479,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13736,8 +16497,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13754,8 +16515,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13772,8 +16533,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13782,7 +16543,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13791,7 +16552,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13800,7 +16561,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13809,7 +16570,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13826,8 +16587,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13844,8 +16605,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13862,8 +16623,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13880,8 +16641,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13890,7 +16651,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13899,7 +16660,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13908,7 +16669,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13917,7 +16678,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13926,7 +16687,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13935,7 +16696,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13944,7 +16705,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13953,7 +16714,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13962,7 +16723,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13971,7 +16732,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13980,7 +16741,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13989,7 +16750,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -13998,7 +16759,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14007,7 +16768,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14016,7 +16777,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14025,7 +16786,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=1,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14034,7 +16795,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14043,7 +16804,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14052,7 +16813,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14061,7 +16822,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14078,8 +16839,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14096,8 +16857,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14114,8 +16875,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14132,8 +16893,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14142,7 +16903,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14151,7 +16912,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14160,7 +16921,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14169,7 +16930,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14186,8 +16947,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14204,8 +16965,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14222,8 +16983,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14240,8 +17001,8 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,2,1,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,2,1,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14250,7 +17011,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14259,7 +17020,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14268,7 +17029,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14277,7 +17038,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14286,7 +17047,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14295,7 +17056,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14304,7 +17065,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14313,7 +17074,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14322,7 +17083,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14331,7 +17092,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14340,7 +17101,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14349,7 +17110,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14358,7 +17119,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14367,7 +17128,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14376,7 +17137,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14385,7 +17146,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14394,7 +17155,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14403,7 +17164,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14412,7 +17173,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14421,7 +17182,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14430,7 +17191,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14439,7 +17200,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14448,7 +17209,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14457,7 +17218,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14466,7 +17227,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14475,7 +17236,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14484,7 +17245,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14493,7 +17254,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14502,7 +17263,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14511,7 +17272,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14520,7 +17281,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14529,7 +17290,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14538,7 +17299,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14547,7 +17308,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14556,7 +17317,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14565,7 +17326,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=113,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14574,7 +17335,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14583,7 +17344,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14592,7 +17353,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14601,7 +17362,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14610,7 +17371,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14619,7 +17380,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14628,7 +17389,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14637,7 +17398,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14646,7 +17407,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=1,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14655,7 +17416,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=3,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14664,7 +17425,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f32,type_V=f32,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=bf16,type_V=bf16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -14673,7 +17434,7 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q5_0,type_V=q5_0,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_1,type_V=q4_1,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q4_0,permute=[0,1,2,3]","support","1","yes","Vulkan" -"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=72,hsv=72,nh=4,nr23=[4,1],kv=512,nb=75,mask=0,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=iq4_nl,type_V=iq4_nl,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=1,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=80,hsv=80,nh=4,nr23=[1,1],kv=113,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" @@ -16523,6 +19284,14 @@ "Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q1_0,type_V=q4_0,permute=[0,1,2,3]","support","0","no","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=128,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q1_0,permute=[0,1,2,3]","support","0","no","Vulkan" "Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=64,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q1_0,type_V=f16,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=4,nr23=[1,1],kv=96,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q2_0,type_V=q2_0,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q2_0,type_V=q4_0,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=64,hsv=128,nh=4,nr23=[1,1],kv=128,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q4_0,type_V=q2_0,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=64,nh=4,nr23=[1,1],kv=64,nb=2,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=q2_0,type_V=f16,permute=[0,1,2,3]","support","0","no","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[6,1],kv=4096,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=4096,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=256,hsv=256,nh=4,nr23=[6,1],kv=16384,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" +"Vulkan0","FLASH_ATTN_EXT","hsk=128,hsv=128,nh=8,nr23=[4,1],kv=16384,nb=512,mask=1,sinks=0,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]","support","1","yes","Vulkan" "Vulkan0","CROSS_ENTROPY_LOSS","type=f32,ne=[10,5,4,3]","support","0","no","Vulkan" "Vulkan0","CROSS_ENTROPY_LOSS","type=f32,ne=[30000,1,1,1]","support","0","no","Vulkan" "Vulkan0","CROSS_ENTROPY_LOSS_BACK","type=f32,ne=[10,5,4,3]","support","0","no","Vulkan" @@ -16530,9 +19299,9 @@ "Vulkan0","OPT_STEP_ADAMW","type=f32,ne=[10,5,4,3]","support","1","yes","Vulkan" "Vulkan0","OPT_STEP_SGD","type=f32,ne=[10,5,4,3]","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=32,head_size=128,n_seq_tokens=1,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=1","support","1","yes","Vulkan" -"Vulkan0","GATED_DELTA_NET","type=f32,head_count=32,head_size=16,n_seq_tokens=1,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=1","support","0","no","Vulkan" -"Vulkan0","GATED_DELTA_NET","type=f32,head_count=32,head_size=16,n_seq_tokens=1,n_seqs=1,v_repeat=1,permuted=1,kda=1,K=1","support","0","no","Vulkan" -"Vulkan0","GATED_DELTA_NET","type=f32,head_count=32,head_size=16,n_seq_tokens=1,n_seqs=1,v_repeat=1,permuted=0,kda=1,K=1","support","0","no","Vulkan" +"Vulkan0","GATED_DELTA_NET","type=f32,head_count=32,head_size=16,n_seq_tokens=1,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=1","support","1","yes","Vulkan" +"Vulkan0","GATED_DELTA_NET","type=f32,head_count=32,head_size=16,n_seq_tokens=1,n_seqs=1,v_repeat=1,permuted=1,kda=1,K=1","support","1","yes","Vulkan" +"Vulkan0","GATED_DELTA_NET","type=f32,head_count=32,head_size=16,n_seq_tokens=1,n_seqs=1,v_repeat=1,permuted=0,kda=1,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=16,head_size=64,n_seq_tokens=1,n_seqs=2,v_repeat=1,permuted=0,kda=0,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=4,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=4,n_seqs=2,v_repeat=1,permuted=0,kda=0,K=1","support","1","yes","Vulkan" @@ -16541,12 +19310,12 @@ "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=4,n_seqs=1,v_repeat=1,permuted=1,kda=0,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=1,n_seqs=1,v_repeat=1,permuted=0,kda=1,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=1,n_seqs=2,v_repeat=1,permuted=0,kda=1,K=1","support","1","yes","Vulkan" -"Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=16,n_seq_tokens=1,n_seqs=2,v_repeat=1,permuted=0,kda=1,K=1","support","0","no","Vulkan" +"Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=16,n_seq_tokens=1,n_seqs=2,v_repeat=1,permuted=0,kda=1,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=32,n_seq_tokens=4,n_seqs=1,v_repeat=1,permuted=0,kda=1,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=4,n_seqs=2,v_repeat=1,permuted=0,kda=1,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=8,head_size=32,n_seq_tokens=4,n_seqs=2,v_repeat=2,permuted=0,kda=1,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=4,n_seqs=2,v_repeat=1,permuted=1,kda=1,K=1","support","1","yes","Vulkan" -"Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=16,n_seq_tokens=4,n_seqs=2,v_repeat=1,permuted=1,kda=1,K=1","support","0","no","Vulkan" +"Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=16,n_seq_tokens=4,n_seqs=2,v_repeat=1,permuted=1,kda=1,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=64,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=127,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=256,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=1","support","1","yes","Vulkan" @@ -16557,7 +19326,7 @@ "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=64,n_seqs=1,v_repeat=1,permuted=0,kda=1,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=33,n_seqs=1,v_repeat=1,permuted=0,kda=1,K=1","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=100,n_seqs=1,v_repeat=1,permuted=0,kda=1,K=1","support","1","yes","Vulkan" -"Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=16,n_seq_tokens=2,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=2","support","0","no","Vulkan" +"Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=16,n_seq_tokens=2,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=2","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=32,n_seq_tokens=4,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=4","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=4,n_seqs=2,v_repeat=1,permuted=0,kda=0,K=4","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=8,head_size=128,n_seq_tokens=4,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=4","support","1","yes","Vulkan" @@ -16565,3 +19334,111 @@ "Vulkan0","GATED_DELTA_NET","type=f32,head_count=8,head_size=32,n_seq_tokens=4,n_seqs=2,v_repeat=2,permuted=0,kda=1,K=4","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=32,n_seq_tokens=8,n_seqs=1,v_repeat=1,permuted=0,kda=0,K=3","support","1","yes","Vulkan" "Vulkan0","GATED_DELTA_NET","type=f32,head_count=4,head_size=64,n_seq_tokens=16,n_seqs=2,v_repeat=1,permuted=0,kda=0,K=4","support","1","yes","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=1,nm=1,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=1,nm=1,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=1,nm=1,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=1,nm=1,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=1,nm=1,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=1,nm=1,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=1,nm=1,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=1,nm=1,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=1,nm=1,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=4,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=4,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=4,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=4,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=4,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=4,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=4,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=4,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=4,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=1,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=1,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=1,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=1,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=1,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=1,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=1,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=1,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=1,ns=4,nm=1,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=1,nm=1,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=1,nm=1,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=1,nm=1,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=1,nm=1,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=1,nm=1,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=1,nm=1,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=1,nm=1,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=1,nm=1,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=1,nm=1,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=4,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=4,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=4,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=4,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=4,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=4,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=4,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=4,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=4,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=1,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=1,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=1,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=1,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=1,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=1,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=1,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=1,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=1,ns=4,nm=1,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=1,nm=1,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=1,nm=1,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=1,nm=1,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=1,nm=1,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=1,nm=1,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=1,nm=1,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=1,nm=1,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=1,nm=1,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=1,nm=1,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=4,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=4,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=4,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=4,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=4,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=4,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=4,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=4,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=4,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=1,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=1,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=1,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=1,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=1,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=1,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=1,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=1,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=32,kv=256,nb=512,ns=4,nm=1,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=1,nm=1,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=1,nm=1,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=1,nm=1,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=1,nm=1,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=1,nm=1,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=1,nm=1,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=1,nm=1,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=1,nm=1,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=1,nm=1,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=4,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=4,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=4,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=4,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=4,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=4,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=4,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=4,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=4,type_K=iq4_nl","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=f32","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=f16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=bf16","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=q8_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=q5_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=q5_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=q4_1","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=q4_0","support","0","no","Vulkan" +"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=iq4_nl","support","0","no","Vulkan" diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 510fb6892b37..2418ba6d1786 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1026,6 +1026,7 @@ struct vk_device_struct { vk_pipeline pipeline_pool2d_f32; vk_pipeline pipeline_rwkv_wkv6_f32; vk_pipeline pipeline_rwkv_wkv7_f32; + vk_pipeline pipeline_gated_linear_attn_f32; // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128 vk_pipeline pipeline_gated_delta_net[4][2]; vk_pipeline pipeline_ssm_scan_f32_d128; @@ -1747,6 +1748,13 @@ struct vk_op_rwkv_wkv7_push_constants { uint32_t C; uint32_t H; }; +struct vk_op_gated_linear_attn_push_constants { + uint32_t B; + uint32_t T; + uint32_t C; + uint32_t H; + float scale; +}; struct vk_op_gated_delta_net_push_constants { uint32_t H; uint32_t n_tokens; @@ -5665,6 +5673,8 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv7_f32, "rwkv_wkv7_f32", rwkv_wkv7_f32_len, rwkv_wkv7_f32_data, "main", 8, sizeof(vk_op_rwkv_wkv7_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); + ggml_vk_create_pipeline(device, device->pipeline_gated_linear_attn_f32, "gated_linear_attn_f32", gated_linear_attn_f32_len, gated_linear_attn_f32_data, "main", 6, sizeof(vk_op_gated_linear_attn_push_constants), {1, 1, 1}, {}, 1); + { const uint32_t gdn_sizes[] = {16, 32, 64, 128}; const char * gdn_names[][2] = { @@ -11392,6 +11402,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return ctx->device->pipeline_rwkv_wkv7_f32; } return nullptr; + case GGML_OP_GATED_LINEAR_ATTN: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_gated_linear_attn_f32; + } + return nullptr; case GGML_OP_GATED_DELTA_NET: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { const uint32_t S_v = dst->src[2]->ne[0]; @@ -12422,6 +12437,41 @@ static void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ); } +static void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { + const size_t seq_length = dst->src[0]->ne[2]; + const size_t n_embed = dst->ne[0]; + const size_t n_heads = dst->src[0]->ne[1]; + const size_t n_seqs = dst->src[4]->ne[1]; + + float scale; + memcpy(&scale, dst->op_params, sizeof(float)); + + GGML_ASSERT(dst->buffer != nullptr); + + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op); + GGML_ASSERT(pipeline != nullptr); + + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); + vk_subbuffer src_buf[5] = {}; + for (int i = 0; i < 5; i++) { + src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]); + } + + const vk_op_gated_linear_attn_push_constants pc = { + (uint32_t)n_seqs, + (uint32_t)seq_length, + (uint32_t)n_embed, + (uint32_t)n_heads, + scale, + }; + + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], dst_buf}, + pc, { (uint32_t)(n_seqs * n_heads), 1, 1 }); +} + static void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { const ggml_tensor * src_q = dst->src[0]; const ggml_tensor * src_v = dst->src[2]; @@ -15421,6 +15471,11 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr break; + case GGML_OP_GATED_LINEAR_ATTN: + ggml_vk_gated_linear_attn(ctx, compute_ctx, node); + + break; + case GGML_OP_GATED_DELTA_NET: ggml_vk_gated_delta_net(ctx, compute_ctx, node); @@ -18128,6 +18183,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_RWKV_WKV6: case GGML_OP_RWKV_WKV7: return true; // all inputs are contiguous, see ggml.c + case GGML_OP_GATED_LINEAR_ATTN: + // the shader block size is hardcoded to head_size 64 + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && op->src[0]->ne[0] == 64; case GGML_OP_GATED_DELTA_NET: { const uint32_t S_v = op->src[2]->ne[0]; @@ -19117,6 +19175,10 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * } else if (tensor->op == GGML_OP_RWKV_WKV7) { tensor_clone = ggml_rwkv_wkv7(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], src_clone[4], src_clone[5], src_clone[6]); + } else if (tensor->op == GGML_OP_GATED_LINEAR_ATTN) { + const float * op_params = (const float *)tensor->op_params; + tensor_clone = ggml_gated_linear_attn(ggml_ctx, src_clone[0], src_clone[1], + src_clone[2], src_clone[3], src_clone[4], op_params[0]); } else if (tensor->op == GGML_OP_GATED_DELTA_NET) { tensor_clone = ggml_gated_delta_net(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], src_clone[4], src_clone[5], diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp b/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp new file mode 100644 index 000000000000..b3387616b642 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp @@ -0,0 +1,82 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : require + +#define BLOCK_SIZE 64 +layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; + +layout(push_constant) uniform Parameters { + uint B; + uint T; + uint C; + uint H; + float scale; +}; + +layout(binding = 0) readonly buffer KBuf { A_TYPE k[]; }; +layout(binding = 1) readonly buffer VBuf { A_TYPE v[]; }; +layout(binding = 2) readonly buffer QBuf { A_TYPE q[]; }; +layout(binding = 3) readonly buffer GBuf { A_TYPE g[]; }; +layout(binding = 4) readonly buffer StateBuf { A_TYPE state_in[]; }; +layout(binding = 5) buffer DstBuf { A_TYPE dst[]; }; + +shared A_TYPE _k[BLOCK_SIZE], _q[BLOCK_SIZE], _g[BLOCK_SIZE]; + +void main() { + const uint head_size = BLOCK_SIZE; + const uint batch_id = gl_WorkGroupID.x / H; + const uint head_id = gl_WorkGroupID.x % H; + const uint tid = gl_LocalInvocationID.x; + + const uint state_size = C * head_size; + const uint n_seq_tokens = T / B; + + if (batch_id >= B || head_id >= H) { + return; + } + + // state[i] holds column tid of this head's state matrix: S[i][tid] + A_TYPE state[BLOCK_SIZE]; + [[unroll]] for (uint i = 0; i < head_size; i++) { + state[i] = state_in[batch_id * state_size + head_id * head_size * head_size + + i * head_size + tid]; + } + + const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; + const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; + + for (uint t = start_t; t < end_t; t += C) { + barrier(); + _k[tid] = k[t]; + _q[tid] = q[t]; + _g[tid] = g[t]; + barrier(); + + const A_TYPE v_val = v[t]; + A_TYPE y = 0.0; + + [[unroll]] for (uint i = 0; i < head_size; i += 4) { + vec4 k_vec = vec4(_k[i], _k[i+1], _k[i+2], _k[i+3]); + vec4 q_vec = vec4(_q[i], _q[i+1], _q[i+2], _q[i+3]); + vec4 g_vec = vec4(_g[i], _g[i+1], _g[i+2], _g[i+3]); + vec4 s_vec = vec4(state[i], state[i+1], state[i+2], state[i+3]); + + vec4 kv = k_vec * v_val; + + s_vec = s_vec * g_vec + kv; + y += dot(q_vec, s_vec); + + state[i] = s_vec.x; + state[i+1] = s_vec.y; + state[i+2] = s_vec.z; + state[i+3] = s_vec.w; + } + + dst[t] = y * scale; + } + + [[unroll]] for (uint i = 0; i < head_size; i++) { + dst[T * C + batch_id * state_size + head_id * head_size * head_size + + i * head_size + tid] = state[i]; + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 0223d2e01e91..c93d6eecee1b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -1057,6 +1057,8 @@ void process_shaders() { string_to_spv("rwkv_wkv6_f32", "wkv6.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); + string_to_spv("gated_linear_attn_f32", "gla.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); + string_to_spv("rwkv_wkv7_f32", "wkv7.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); string_to_spv("gated_delta_net_f32", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "1"}})); From b5746d28cee85f8d2a3b01addd0987fa33180497 Mon Sep 17 00:00:00 2001 From: Angel Galindo <131726962+AngelGalindo7@users.noreply.github.com> Date: Tue, 4 Aug 2026 01:41:18 -0700 Subject: [PATCH 59/98] convert : add missing return after setting tekken vocab (#25947) --- conversion/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conversion/llama.py b/conversion/llama.py index 9b3373f911e9..1aced49c54d2 100644 --- a/conversion/llama.py +++ b/conversion/llama.py @@ -119,7 +119,7 @@ def set_vocab(self): path_tekken_json = self.dir_model / "tekken.json" path_tokenizer_json = self.dir_model / "tokenizer.json" if path_tekken_json.is_file() and not path_tokenizer_json.is_file(): - self._set_vocab_mistral() + return self._set_vocab_mistral() tokenizer_config_file = self.dir_model / 'tokenizer_config.json' if tokenizer_config_file.is_file(): From 15831f579a70ca8b3ed0871c1a702df356e5e4c1 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 4 Aug 2026 11:44:52 +0300 Subject: [PATCH 60/98] ggml : bump version to 0.18.1 (ggml/1578) --- ggml/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 159da3afa0b0..6c7337edd398 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -5,7 +5,7 @@ project("ggml" C CXX ASM) ### GGML Version set(GGML_VERSION_MAJOR 0) set(GGML_VERSION_MINOR 18) -set(GGML_VERSION_PATCH 0) +set(GGML_VERSION_PATCH 1) set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") From 2e17f69ef4e7fcb9ee950362b3530d8e4e2ea316 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 4 Aug 2026 11:54:13 +0300 Subject: [PATCH 61/98] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 230e59a95bbe..35e94d9fb614 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -06ca97616793248fadb410ea8d69c7511b2005e4 +90951f99af1fbebef3fbdd58ff5b8715b0bb9c43 From 5788b510a1e3394fcc2d6b13ba2d9d8fc4a5c139 Mon Sep 17 00:00:00 2001 From: hcl Date: Tue, 4 Aug 2026 17:12:48 +0800 Subject: [PATCH 62/98] gguf-py: validate n_dims and guard against uint64 overflow in reader (#25401) The Python GGUF reader lacked two guards the C++ loader has: - n_dims read as uint32 with no GGML_MAX_DIMS bound -> crafted file with huge n_dims triggers oversized memmap read / OOM. - np.prod(dims) on uint64 wraps silently -> a crafted dims triple can overflow to a tiny element count, passing an undersized read through. Add a GGML_MAX_DIMS check and compute the element count with Python ints. Fixes #25378 --- gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_reader.py | 8 ++++- gguf-py/tests/test_gguf_reader_validation.py | 37 ++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 gguf-py/tests/test_gguf_reader_validation.py diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index c9ec92bd8ed4..6b0a26b63d89 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -11,6 +11,7 @@ GGUF_VERSION = 3 GGUF_DEFAULT_ALIGNMENT = 32 GGML_QUANT_VERSION = 2 # GGML_QNT_VERSION from ggml.h +GGML_MAX_DIMS = 4 # GGML_MAX_DIMS from ggml.h # # metadata keys diff --git a/gguf-py/gguf/gguf_reader.py b/gguf-py/gguf/gguf_reader.py index 0a1b85f50641..ea241ada285c 100644 --- a/gguf-py/gguf/gguf_reader.py +++ b/gguf-py/gguf/gguf_reader.py @@ -22,6 +22,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from gguf.constants import ( + GGML_MAX_DIMS, GGML_QUANT_SIZES, GGUF_DEFAULT_ALIGNMENT, GGUF_MAGIC, @@ -266,6 +267,8 @@ def _get_tensor_info_field(self, orig_offs: int) -> ReaderField: # Get Tensor Dimensions Count n_dims = self._get(offs, np.uint32) offs += int(n_dims.nbytes) + if n_dims[0] > GGML_MAX_DIMS: + raise ValueError(f'Tensor dimensions count {n_dims[0]} exceeds GGML_MAX_DIMS ({GGML_MAX_DIMS})') # Get Tensor Dimension Array dims = self._get(offs, np.uint64, n_dims[0]) @@ -326,7 +329,10 @@ def _build_tensors(self, start_offs: int, fields: list[ReaderField]) -> None: raise ValueError(f'Found duplicated tensor with name {tensor_name}') tensor_names.add(tensor_name) ggml_type = GGMLQuantizationType(raw_dtype[0]) - n_elems = int(np.prod(dims)) + # use Python ints: np.prod on uint64 wraps silently on overflow + n_elems = 1 + for dim in dims.tolist(): + n_elems *= int(dim) np_dims = tuple(reversed(dims.tolist())) block_size, type_size = GGML_QUANT_SIZES[ggml_type] n_bytes = n_elems * type_size // block_size diff --git a/gguf-py/tests/test_gguf_reader_validation.py b/gguf-py/tests/test_gguf_reader_validation.py new file mode 100644 index 000000000000..98f30a969608 --- /dev/null +++ b/gguf-py/tests/test_gguf_reader_validation.py @@ -0,0 +1,37 @@ +import struct +import numpy as np +import pytest + +from gguf.gguf_reader import GGUFReader + + +def _write_gguf(path, n_dims_field, dims): + buf = b'GGUF' + struct.pack(' Date: Tue, 4 Aug 2026 13:17:15 +0200 Subject: [PATCH 63/98] speculative : refactor enabled configs common_speculative_init (#26510) This commit contains a suggestion to reduce some code duplication in common_speculative_init when adding the enabled speculative decoding configurations. No tests were added but the existing server tests still passes with this change: ```console $ ./tests.sh unit/test_speculative.py -v -x ``` --- common/speculative.cpp | 61 +++++++++++------------------------------- 1 file changed, 16 insertions(+), 45 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index 514fe852180f..70dc0ac3b1b7 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2385,57 +2385,28 @@ common_speculative * common_speculative_init(common_params_speculative & params, { uint32_t enabled_configs = common_get_enabled_speculative_configs(params.types); - bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); - bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr; - bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; - bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr; - bool has_draft_dspark = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)) && params.draft.ctx_dft != nullptr; - - - - bool has_ngram_cache = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_CACHE)); - bool has_ngram_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE)); - bool has_ngram_map_k = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K)); - bool has_ngram_map_k4v = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V)); - bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD)); + auto add_config_if_enabled = [&](common_speculative_type type, bool available = true) { + if (available && (enabled_configs & (1u << type))) { + configs.emplace_back(type, params); + } + }; // when adding a new type - update here the logic above static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11); // this list here defines the priority of the speculators // the one with highest priority are listed first - if (has_ngram_simple) { - // This implementation can guess a lot of tokens without any draft model. - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, params)); - } - if (has_ngram_map_k) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, params)); - } - if (has_ngram_map_k4v) { - // This implementation can guess tokens with high acceptance rate but is more expensive. - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, params)); - } - if (has_ngram_mod) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, params)); - } - if (has_ngram_cache) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params)); - } - if (has_draft_simple) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, params)); - } - if (has_draft_eagle3) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params)); - } - if (has_draft_mtp) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params)); - } - if (has_draft_dflash) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params)); - } - if (has_draft_dspark) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params)); - } + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MOD); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE); + + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params.draft.ctx_dft != nullptr); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params.draft.ctx_dft != nullptr); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params.draft.ctx_dft != nullptr); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params.draft.ctx_dft != nullptr); } std::vector> impls = {}; From 6b5224cfccdb9caf4c0a0a87692fddad22c7e969 Mon Sep 17 00:00:00 2001 From: Niklas Wenzel Date: Tue, 4 Aug 2026 15:03:38 +0200 Subject: [PATCH 64/98] ci: fix pre-built binaries no longer working on macOS 15 and below (#26375) * ci: fix pre-built binaries no longer working on macOS 15 and below * ci: add macOS deployment target to disabled KleidiAI build --- .github/workflows/build-apple.yml | 3 ++- .github/workflows/release.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-apple.yml b/.github/workflows/build-apple.yml index 2b3d14d1f3cf..75add999341c 100644 --- a/.github/workflows/build-apple.yml +++ b/.github/workflows/build-apple.yml @@ -63,7 +63,8 @@ jobs: -DGGML_METAL_USE_BF16=ON \ -DGGML_METAL_EMBED_LIBRARY=OFF \ -DGGML_METAL_SHADER_DEBUG=ON \ - -DGGML_RPC=ON + -DGGML_RPC=ON \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) leaks -atExit -- ./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3460485af388..7a69caaefdab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,13 +93,13 @@ jobs: - build: 'arm64' arch: 'arm64' os: macos-26 - defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON" + defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3" # TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23780) # in order to enable it again, we have to provision dedicated runners to run it #- build: 'arm64-kleidiai' # arch: 'arm64' # os: macos-14 - # defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON -DGGML_CPU_KLEIDIAI=ON" + # defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 -DGGML_CPU_KLEIDIAI=ON" - build: 'x64' arch: 'x64' os: macos-15-intel From 1c3c9674de4d455f1e571bed808252af54932767 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 4 Aug 2026 16:56:49 +0300 Subject: [PATCH 65/98] models : fix dflash wo_a reshape on load (#26577) --- src/models/dflash.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 6c82ab3da43a..daff6e78f1cf 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -125,7 +125,7 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, 0); layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, 0); layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, 0); - layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, 0); + layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank, o_groups}, TENSOR_ALLOW_RESHAPE); layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, 0); layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0); From 07132750825a4f2d27a547cd9cdde1c6f6001885 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Tue, 4 Aug 2026 17:26:15 +0200 Subject: [PATCH 66/98] mtmd: support Qwen3-TTS (note: breaking change to llama-tts binary) (#26254) * convert text model * main model load ok * convert encoder ok * speaker encoder loading ok * speaker enc graph * adapt vocab for backbone (with some tricks) * add suppress_tokens * poc new mtmd gen api * convert code_predictor to gguf * load gen_code model ok * add clip_encode * wire up * code gen cgraph init version Co-authored-by: Pascal * code2wav convert to gguf * code2wav graph ok * wire up in/out * (wip) subgraph * wire up * wip, correct code2wav * demo (to be removed) * code2wav preserve kv between calls * demo voice clone * llama: add llama_model_get_tok_embd * mtmd_helper_gen_audio API * fix clamp cold prefix Co-authored-by: Pascal * fuse snake op Co-authored-by: Pascal * demo: use proper sampling * update dev docs * polymorphism helper * revamp llama-tts binary * update docs * fix compile * fix lint * nits * add guide + docs * more timings info * clean up code comments * security fixes * update docs * use ggml_build_forward_select, clean up comments * fix ci * use ISO 639-1 language code * rename CODE2WAV --> GEN_WAV, update docs * clean up * clean up tts.cpp * add seq_id * add step_prompt() * mtmd_helper_model_can_chat * clean up comments --------- Co-authored-by: Pascal --- common/arg.cpp | 69 +- common/arg.h | 1 - common/common.h | 13 +- conversion/__init__.py | 2 + conversion/qwen3tts.py | 471 ++++++++++ docs/development/HOWTO-add-model.md | 1 + gguf-py/gguf/constants.py | 206 +++++ gguf-py/gguf/gguf_writer.py | 39 +- gguf-py/gguf/tensor_mapping.py | 23 +- skills/code-review/SKILL.md | 1 + src/llama-arch.cpp | 2 + src/llama-arch.h | 1 + src/llama-ext.h | 6 + src/llama-model.cpp | 38 + src/models/models.h | 5 + src/models/qwen3tts.cpp | 3 + src/models/qwen3vl.cpp | 25 +- tests/test-llama-archs.cpp | 2 + tools/mtmd/CMakeLists.txt | 4 + tools/mtmd/README-dev.md | 49 ++ tools/mtmd/clip-graph.h | 3 + tools/mtmd/clip-impl.h | 49 ++ tools/mtmd/clip-model.h | 106 +++ tools/mtmd/clip.cpp | 453 +++++++++- tools/mtmd/clip.h | 29 + tools/mtmd/models/models.h | 110 +++ tools/mtmd/models/qwen3tts-gen.cpp | 766 ++++++++++++++++ tools/mtmd/models/qwen3tts-spkenc.cpp | 197 +++++ tools/mtmd/mtmd-audio.cpp | 60 ++ tools/mtmd/mtmd-audio.h | 9 + tools/mtmd/mtmd-cli.cpp | 10 +- tools/mtmd/mtmd-helper-common.h | 180 ++++ tools/mtmd/mtmd-helper-gen.cpp | 505 +++++++++++ tools/mtmd/mtmd-helper.cpp | 171 +--- tools/mtmd/mtmd-helper.h | 92 ++ tools/mtmd/mtmd.cpp | 141 +++ tools/mtmd/mtmd.h | 54 ++ tools/tts/CMakeLists.txt | 2 +- tools/tts/README.md | 127 +-- tools/tts/convert_pt_to_hf.py | 180 ---- tools/tts/tts-outetts.py | 299 ------- tools/tts/tts.cpp | 1159 +++---------------------- 42 files changed, 3788 insertions(+), 1875 deletions(-) create mode 100644 conversion/qwen3tts.py create mode 100644 src/models/qwen3tts.cpp create mode 100644 tools/mtmd/models/qwen3tts-gen.cpp create mode 100644 tools/mtmd/models/qwen3tts-spkenc.cpp create mode 100644 tools/mtmd/mtmd-helper-common.h create mode 100644 tools/mtmd/mtmd-helper-gen.cpp delete mode 100644 tools/tts/convert_pt_to_hf.py delete mode 100644 tools/tts/tts-outetts.py diff --git a/common/arg.cpp b/common/arg.cpp index b75f4f05f039..86af0ba10a32 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -61,6 +61,7 @@ static std::initializer_list mmproj_examples = { LLAMA_EXAMPLE_MTMD, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI, + LLAMA_EXAMPLE_TTS, }; static std::string read_file(const std::string & fname) { @@ -360,7 +361,6 @@ static bool spec_types_is_default(const common_params & params) { common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) { common_download_hf_plan plan; common_download_hf_plan plan_spec; - common_download_hf_plan plan_voc; common_download_opts opts; const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), @@ -413,11 +413,7 @@ common_models_handler common_models_handler_init(const common_params & params, l plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec); } - if (!params.vocoder.model.hf_repo.empty()) { - plan_voc = common_download_get_hf_plan(params.vocoder.model, opts); - } - - return common_models_handler{plan, plan_spec, plan_voc, opts}; + return common_models_handler{plan, plan_spec, opts}; } bool common_models_handler_is_preset_repo(const common_models_handler & handler) { @@ -467,7 +463,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params auto & plan = handler.plan; auto & plan_spec = handler.plan_spec; - auto & plan_voc = handler.plan_voc; auto opts = handler.opts; // copy opts.callback = callback; @@ -482,7 +477,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params }; handle_url(params.model); handle_url(params.mmproj); - handle_url(params.vocoder.model); handle_url(params.speculative.draft.mparams); // optionally, if docker repo is set, resolve it @@ -510,14 +504,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params task.opts = opts; tasks.push_back(task); } - if (!params.vocoder.model.url.empty()) { - common_download_task task; - task.url = params.vocoder.model.url; - task.local_path = params.vocoder.model.path; - task.opts = opts; - tasks.push_back(task); - } - bool had_spec_url = false; if (!params.speculative.draft.mparams.url.empty()) { common_download_task task; @@ -631,11 +617,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params had_spec_url = true; } - // handle vocoder plan (e.g. --hf-repo-v) - if (!plan_voc.model_files.empty()) { - add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model); - } - if (!plan.model_files.empty()) { add_tasks(plan.model_files, plan.primary, params.model); } @@ -1361,6 +1342,10 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.n_parallel = -1; // auto by default } else if (ex == LLAMA_EXAMPLE_TOKENIZE) { params.parse_special = true; // parse special tokens by default, like the old tokenize tool + } else if (ex == LLAMA_EXAMPLE_TTS) { + params.out_file = "output.wav"; + params.sampling.penalty_repeat = 1.05f; + params.sampling.penalty_last_n = -1; } params.use_color = tty_can_use_colors(); @@ -2983,20 +2968,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.model.hf_file = value; } ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE")); - add_opt(common_arg( - {"-hfv", "-hfrv", "--hf-repo-v"}, "/[:quant]", - "Hugging Face model repository for the vocoder model (default: unused)", - [](common_params & params, const std::string & value) { - params.vocoder.model.hf_repo = value; - } - ).set_env("LLAMA_ARG_HF_REPO_V")); - add_opt(common_arg( - {"-hffv", "--hf-file-v"}, "FILE", - "Hugging Face model file for the vocoder model (default: unused)", - [](common_params & params, const std::string & value) { - params.vocoder.model.hf_file = value; - } - ).set_env("LLAMA_ARG_HF_FILE_V")); add_opt(common_arg( {"-hft", "--hf-token"}, "TOKEN", "Hugging Face access token (default: value from HF_TOKEN environment variable)", @@ -4272,24 +4243,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex // add_opt(common_arg( - {"-mv", "--model-vocoder"}, "FNAME", - "vocoder model for audio generation (default: unused)", + {"--tts-lang"}, "FNAME", + "language (ISO 639-1) for audio generation\n" + "see tts/README.md for per-model usage notes", [](common_params & params, const std::string & value) { - params.vocoder.model.path = value; - } - ).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER})); - add_opt(common_arg( - {"--tts-use-guide-tokens"}, - "Use guide tokens to improve TTS word recall", - [](common_params & params) { - params.vocoder.use_guide_tokens = true; + params.tts_lang = value; } - ).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER})); + ).set_examples({LLAMA_EXAMPLE_TTS})); add_opt(common_arg( {"--tts-speaker-file"}, "FNAME", "speaker file path for audio generation", [](common_params & params, const std::string & value) { - params.vocoder.speaker_file = value; + params.tts_speaker_file = value; } ).set_examples({LLAMA_EXAMPLE_TTS})); @@ -4409,16 +4374,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({LLAMA_EXAMPLE_DEBUG})); // presets - add_opt(common_arg( - {"--tts-oute-default"}, - string_format("use default OuteTTS models (note: can download weights from the internet)"), - [](common_params & params) { - params.model.hf_repo = "OuteAI/OuteTTS-0.2-500M-GGUF"; - params.model.hf_file = "OuteTTS-0.2-500M-Q8_0.gguf"; - params.vocoder.model.hf_repo = "ggml-org/WavTokenizer"; - params.vocoder.model.hf_file = "WavTokenizer-Large-75-F16.gguf"; - } - ).set_examples({LLAMA_EXAMPLE_TTS})); add_opt(common_arg( {"--embd-gemma-default"}, diff --git a/common/arg.h b/common/arg.h index 8f609e356fe2..44b9e887cfb9 100644 --- a/common/arg.h +++ b/common/arg.h @@ -137,7 +137,6 @@ void common_params_add_preset_options(std::vector & args); struct common_models_handler { common_download_hf_plan plan; common_download_hf_plan plan_spec; - common_download_hf_plan plan_voc; common_download_opts opts; }; diff --git a/common/common.h b/common/common.h index 78d0877566b6..3444aa157e9b 100644 --- a/common/common.h +++ b/common/common.h @@ -392,14 +392,6 @@ struct common_params_speculative { } }; -struct common_params_vocoder { - struct common_params_model model; - - std::string speaker_file; // speaker file path - - bool use_guide_tokens = false; // enable guide tokens to improve TTS accuracy -}; - struct common_params_diffusion { int32_t steps = 128; bool visual_mode = false; @@ -497,7 +489,6 @@ struct common_params { struct common_params_sampling sampling; struct common_params_speculative speculative; - struct common_params_vocoder vocoder; struct common_params_diffusion diffusion; struct common_params_model model; @@ -740,6 +731,10 @@ struct common_params { void * load_progress_callback_user_data = NULL; bool no_alloc = false; // Don't allocate model buffers + // TTS params + std::string tts_lang = ""; + std::string tts_speaker_file = ""; + bool is_gen_docs = false; // whether we are running inside llama-gen-docs }; diff --git a/conversion/__init__.py b/conversion/__init__.py index 534f9e309a13..06c2c50ad245 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -210,6 +210,7 @@ "Qwen3MoeForCausalLM": "qwen", "Qwen3NextForCausalLM": "qwen", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", + "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", "Qwen3_5ForCausalLM": "qwen", @@ -304,6 +305,7 @@ "Qwen2_5_VLForConditionalGeneration": "qwenvl", "Qwen3ASRForConditionalGeneration": "qwen3vl", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", + "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", "Qwen3_5ForConditionalGeneration": "qwen3vl", diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py new file mode 100644 index 000000000000..d21a5059517a --- /dev/null +++ b/conversion/qwen3tts.py @@ -0,0 +1,471 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Callable, Iterable, TYPE_CHECKING + +import torch +import torch.nn.functional as F + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, MmprojModel, TextModel, gguf + +# Tricks being used to support this model via existing llama.cpp code paths: +# - Text projection MLP is folded into the embedding table +# - codec_embedding is concat to the text embedding table, vocab is extended +# example: codec_bos_id(2149) --> "<|codec_bos|>" +# codec_eos_token_id(2150) --> "<|codec_eos_token|>" +# codec_language_id.chinese(2055) --> "<|codec_language_chinese|>" +# other rows --> "<|codec_0|>", "<|codec_1|>", ..., "<|codec_1023|>" +# - output tensor codec_head is smaller than vocab, so logits will be padded at inference time +# - suppress_tokens is used to limit the backbone to only sample either semantic or EOS (stop) token + +# pipeline stage mapping: +# speaker reference encoder --> mapped to normal mtmd audio encoder +# backbone --> mapped to normal libllama text model (autoregressive) +# code_predictor --> MTMD_GEN_PROCESS_TYPE_GEN_CODE +# code2wav --> MTMD_GEN_PROCESS_TYPE_GEN_WAV + +# torch activation functions used by Qwen3TTSTalkerResizeMLP (config's hidden_act) +_ACT2FN = { + "silu": F.silu, + "gelu": F.gelu, + "relu": F.relu, +} + + +@ModelBase.register("Qwen3TTSForConditionalGeneration") +class Qwen3TTSTalkerModel(TextModel): + model_arch = gguf.MODEL_ARCH.QWEN3TTS + + _TEXT_PROJ_KEYS = ( + "model.text_embedding.weight", + "text_projection.linear_fc1.weight", + "text_projection.linear_fc1.bias", + "text_projection.linear_fc2.weight", + "text_projection.linear_fc2.bias", + ) + + _text_proj_buffer: dict[str, Tensor] + _folded_text_embed: Tensor | None + _codec_embed: Tensor | None + + def __init__(self, dir_model: Path, *args, **kwargs): + hparams = kwargs.pop("hparams", None) + if hparams is None: + hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) + raw_talker_config = dict(hparams["talker_config"]) + self._talker_config = raw_talker_config + self.n_codec_vocab = raw_talker_config["vocab_size"] + talker_config = dict(raw_talker_config) + talker_config["vocab_size"] = talker_config["text_vocab_size"] + hparams["text_config"] = talker_config + super().__init__(dir_model, *args, hparams=hparams, **kwargs) + self._text_proj_buffer = {} + self._folded_text_embed = None + self._codec_embed = None + + def _codec_token_names(self) -> list[str]: + # start every row with a generic name, then override the ones with a + # known meaning (bos/eos/language/etc, derived from the *_id fields + # of talker_config) with a more descriptive one + names = [f"<|codec_{i}|>" for i in range(self.n_codec_vocab)] + for key, val in self._talker_config.items(): + if not key.endswith("_id"): + continue + prefix = key[:-len("_id")] + if isinstance(val, int): + names[val] = f"<|{prefix}|>" + elif isinstance(val, dict): + for subkey, subval in val.items(): + names[subval] = f"<|{prefix}_{subkey}|>" + return names + + def set_vocab(self): + codec_tokens = self._codec_token_names() + codec_toktypes = [gguf.TokenType.CONTROL] * len(codec_tokens) + + try: + tokens, scores, toktypes = self._create_vocab_sentencepiece() + self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_tokenizer_pre("default") + tokens += [t.encode("utf-8") for t in codec_tokens] + scores += [0.0] * len(codec_tokens) + toktypes += codec_toktypes + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_scores(scores) + self.gguf_writer.add_token_types(toktypes) + special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) + special_vocab.add_to_gguf(self.gguf_writer) + return + except FileNotFoundError: + pass + + tokens, toktypes, tokpre = self.get_vocab_base() + tokens += codec_tokens + toktypes += codec_toktypes + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + special_vocab.add_to_gguf(self.gguf_writer) + + # make sure that the model has no chat template, so chat will be disabled + self.gguf_writer.add_chat_template(None) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + # note: final vocab layout is [text_vocab | codec_vocab], with text_vocab is actually padded with -inf in cgraph + # for codec_vocab, only first 2048 rows can be sampled for semantic code + # plus codec_eos_token_id that used for signaling end of generation + # ref: https://github.com/QwenLM/Qwen3-TTS/blob/022e286b98fbec7e1e916cb940cdf532cd9f488e/qwen_tts/core/models/modeling_qwen3_tts.py#L2059-L2063 + + vocab_size = self.hparams["vocab_size"] + self.n_codec_vocab + codec_eos_token_id = self.hparams["vocab_size"] + self._talker_config["codec_eos_token_id"] + self.gguf_writer.add_suppress_tokens([ + i for i in range(vocab_size - 1024, vocab_size) + if i != codec_eos_token_id + ]) + self.gguf_writer.add_eos_token_id(codec_eos_token_id) + self.gguf_writer.add_add_eos_token(False) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if not name.startswith("talker.") or name.startswith("talker.code_predictor."): + return None + + name = name[len("talker."):] + return super().filter_tensors((name, gen)) + + def _maybe_emit_token_embd(self) -> Iterable[tuple[str, Tensor]]: + if self._folded_text_embed is None or self._codec_embed is None: + return + combined = torch.cat([self._folded_text_embed, self._codec_embed], dim=0) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), combined) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # codec_embedding rows are appended after the text vocab, extending the embedding table + if name == "model.codec_embedding.weight": + self._codec_embed = data_torch + yield from self._maybe_emit_token_embd() + return + + # codec_head is the output head for the (smaller) codec vocab; logits get padded to + # the extended vocab size at inference time + if name == "codec_head.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch) + return + + if name in self._TEXT_PROJ_KEYS: + self._text_proj_buffer[name] = data_torch + if len(self._text_proj_buffer) < len(self._TEXT_PROJ_KEYS): + return + + # fold MLP into the embedding table at conversion time, MLP won't be used at inference time anyway + act_fn = _ACT2FN[self.hparams["hidden_act"]] + embed = self._text_proj_buffer["model.text_embedding.weight"] + hidden = act_fn(F.linear(embed, + self._text_proj_buffer["text_projection.linear_fc1.weight"], + self._text_proj_buffer["text_projection.linear_fc1.bias"])) + folded = F.linear(hidden, + self._text_proj_buffer["text_projection.linear_fc2.weight"], + self._text_proj_buffer["text_projection.linear_fc2.bias"]) + self._folded_text_embed = folded + yield from self._maybe_emit_token_embd() + return + + yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("Qwen3TTSForConditionalGeneration") +class Qwen3TTSSpeakerEncoderModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = True + + # talker.code_predictor.model.layers.{bid}. -> A_GEN_CODE_* + # bypass tensor_mapping.py for now to make it simple + _CODE_LAYER_TENSOR_MAP = { + "input_layernorm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM, + "self_attn.q_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q, + "self_attn.q_norm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM, + "self_attn.k_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K, + "self_attn.k_norm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM, + "self_attn.v_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_V, + "self_attn.o_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT, + "post_attention_layernorm": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM, + "mlp.gate_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE, + "mlp.up_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP, + "mlp.down_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, + } + + # note: codebook pages will be stacked to 3D + _CODE_GEN_N_CODEBOOKS = 15 + _code_embed_buffer: dict[int, Tensor] = {} + _code_head_buffer: dict[int, Tensor] = {} + _wav_config_cache: dict[str, Any] | None = None + + def __init__(self, dir_model: Path, *args, **kwargs): + hparams = kwargs.pop("hparams", None) + if hparams is None: + hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) + hparams["text_config"] = {"hidden_size": hparams["talker_config"]["hidden_size"]} + # ECAPA-TDNN has a fixed 4-stage backbone, but MmprojModel.__init__ needs a n_block_keys + hparams["speaker_encoder_config"]["n_layers"] = 4 + super().__init__(dir_model, *args, hparams=hparams, **kwargs) + self._wav_config_cache = None + + def get_audio_config(self) -> dict[str, Any] | None: + return self.global_config.get("speaker_encoder_config") + + def set_gguf_parameters(self): + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_clip_has_audio_encoder(True) + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_SPKENC) + + # handle speaker encoder config + self.gguf_writer.add_audio_projection_dim(self.n_embd_text) + # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default) + self.gguf_writer.add_audio_num_mel_bins(128) + # 3 SE-Res2Net stages; the stem conv, mfa, asp and fc are not counted here + self.gguf_writer.add_audio_block_count(3) + # ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it + self.gguf_writer.add_audio_embedding_length(1536) + self.gguf_writer.add_audio_head_count(1) + self.gguf_writer.add_audio_feed_forward_length(1536) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + + # handle code predictor config + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_GEN) + code_predictor_config = self.global_config["talker_config"]["code_predictor_config"] + self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_gen_audio_embedding_length(code_predictor_config["hidden_size"]) + self.gguf_writer.add_gen_audio_feed_forward_length(code_predictor_config["intermediate_size"]) + self.gguf_writer.add_gen_audio_block_count(code_predictor_config["num_hidden_layers"]) + self.gguf_writer.add_gen_audio_head_count(code_predictor_config["num_attention_heads"]) + self.gguf_writer.add_gen_audio_head_count_kv(code_predictor_config["num_key_value_heads"]) + self.gguf_writer.add_gen_audio_attention_layernorm_eps(code_predictor_config["rms_norm_eps"]) + # note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here + + def _wav_decoder_config(self) -> dict[str, Any] | None: + # code2wav has its own config.json, inside the speech_tokenizer dir + if self._wav_config_cache is None: + path = self.dir_model / "speech_tokenizer" / "config.json" + with open(path, "r", encoding="utf-8") as f: + cfg = json.load(f) + self._wav_config_cache = cfg["decoder_config"] + return self._wav_config_cache + + def tensor_force_quant(self, name, new_name, bid, n_dims): + # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path + if new_name.endswith(".weight") and ( + new_name in ("a.gen.wav.pre_conv.weight", "a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight") + or (".up.blk." in new_name and new_name.endswith(".dwconv.weight")) + or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight"))) + ): + return gguf.GGMLQuantizationType.F16 + # ConvTranspose1d kernels: only F16/F32 are implemented, no BF16 + if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if not ( + name.startswith("speaker_encoder.") + or name.startswith("talker.code_predictor.") + or name == "talker.model.codec_embedding.weight" + ): + return None + + return super().filter_tensors((name, gen)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # code2wav tensors are already named by generate_extra_tensors(), pass them through + if name.startswith("a.gen.wav."): + yield (name, data_torch) + return + + # codebook-0 embedding, fed back to the talker backbone (codebooks 1-15 live in code_predictor) + if name == "talker.model.codec_embedding.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD), data_torch) + return + + if name == "talker.code_predictor.model.norm.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM), data_torch) + return + + if name.startswith("talker.code_predictor.small_to_mtp_projection."): + suffix = "." + name.rsplit(".", 1)[1] + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_PROJ_IN, suffix=suffix), data_torch) + return + + if name.startswith("talker.code_predictor.model.codec_embedding."): + idx = int(name.split("codec_embedding.")[1].split(".")[0]) + self._code_embed_buffer[idx] = data_torch + if len(self._code_embed_buffer) < self._CODE_GEN_N_CODEBOOKS: + return + stacked = torch.stack([self._code_embed_buffer.pop(i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_EMBD), stacked) + return + + if name.startswith("talker.code_predictor.lm_head."): + idx = int(name.split("lm_head.")[1].split(".")[0]) + self._code_head_buffer[idx] = data_torch + if len(self._code_head_buffer) < self._CODE_GEN_N_CODEBOOKS: + return + stacked = torch.stack([self._code_head_buffer.pop(i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_HEAD), stacked) + return + + if name.startswith("talker.code_predictor.model.layers."): + rest = name.split("model.layers.")[1] # "{bid}..weight" + _, key_with_suffix = rest.split(".", 1) # ".weight" + key = key_with_suffix.rsplit(".", 1)[0] # "" + tensor = self._CODE_LAYER_TENSOR_MAP.get(key) + if tensor is not None: + yield (self.format_tensor_name(tensor, bid), data_torch) + return + + if "res2net_block.blocks." in name: + assert bid is not None # the outer stage index, picked up from the tensor name automatically + xid = int(name.split("res2net_block.blocks.")[1].split(".")[0]) + suffix = "." + name.rsplit(".", 1)[1] + new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CONV_RES2].format(bid=bid, xid=xid) + suffix + yield (new_name, data_torch) + return + + yield from super().modify_tensors(data_torch, name, bid) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + yield from self._generate_code2wav_tensors() + + def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]: + # code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors + from safetensors.torch import load_file + + wav_config = self._wav_decoder_config() + state_dict = load_file(self.dir_model / "speech_tokenizer" / "model.safetensors") + + def get(name: str) -> Tensor: + return state_dict[name] + + def snake_fold(alpha: Tensor, beta: Tensor) -> tuple[Tensor, Tensor]: + # fold SnakeBeta's exp()/reciprocal here, so the graph is only mul/sin/sqr/mul/add + return torch.exp(alpha), 1.0 / (torch.exp(beta) + 1e-9) + + def rvq_codebook(prefix: str, n_layers: int) -> Tensor: + # checkpoint has EMA accumulators, so codebook[i] = embedding_sum[i] / cluster_usage[i] + books = [] + for i in range(n_layers): + embedding_sum = get(f"{prefix}.vq.layers.{i}._codebook.embedding_sum") + cluster_usage = get(f"{prefix}.vq.layers.{i}._codebook.cluster_usage") + books.append(embedding_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1)) + return torch.stack(books, dim=0) if n_layers > 1 else books[0] + + T = gguf.MODEL_TENSOR + + # --- quantizer: RVQ codebook decode --- + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_IN), get("decoder.quantizer.rvq_first.input_proj.weight").squeeze(-1)) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT), get("decoder.quantizer.rvq_first.output_proj.weight").squeeze(-1)) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), rvq_codebook("decoder.quantizer.rvq_first", 1)) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_IN), get("decoder.quantizer.rvq_rest.input_proj.weight").squeeze(-1)) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_OUT), get("decoder.quantizer.rvq_rest.output_proj.weight").squeeze(-1)) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB), rvq_codebook("decoder.quantizer.rvq_rest", self._CODE_GEN_N_CODEBOOKS)) + + # --- pre_conv --- + yield (self.format_tensor_name(T.A_GEN_WAV_PRE_CONV, suffix=".weight"), get("decoder.pre_conv.conv.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_PRE_CONV, suffix=".bias"), get("decoder.pre_conv.conv.bias")) + + # --- pre_transformer --- + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=".weight"), get("decoder.pre_transformer.input_proj.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=".bias"), get("decoder.pre_transformer.input_proj.bias")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=".weight"), get("decoder.pre_transformer.output_proj.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=".bias"), get("decoder.pre_transformer.output_proj.bias")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), get("decoder.pre_transformer.norm.weight")) + + tfm_layer_map = { + "input_layernorm.weight": T.A_GEN_WAV_TFM_ATTN_NORM, + "self_attn.q_proj.weight": T.A_GEN_WAV_TFM_ATTN_Q, + "self_attn.k_proj.weight": T.A_GEN_WAV_TFM_ATTN_K, + "self_attn.v_proj.weight": T.A_GEN_WAV_TFM_ATTN_V, + "self_attn.o_proj.weight": T.A_GEN_WAV_TFM_ATTN_OUT, + "self_attn_layer_scale.scale": T.A_GEN_WAV_TFM_ATTN_SCALE, + "post_attention_layernorm.weight": T.A_GEN_WAV_TFM_FFN_NORM, + "mlp.gate_proj.weight": T.A_GEN_WAV_TFM_FFN_GATE, + "mlp.up_proj.weight": T.A_GEN_WAV_TFM_FFN_UP, + "mlp.down_proj.weight": T.A_GEN_WAV_TFM_FFN_DOWN, + "mlp_layer_scale.scale": T.A_GEN_WAV_TFM_FFN_SCALE, + } + assert wav_config is not None + for bid in range(wav_config["num_hidden_layers"]): + for key, tensor_id in tfm_layer_map.items(): + yield (self.format_tensor_name(tensor_id, bid), get(f"decoder.pre_transformer.layers.{bid}.{key}")) + + # --- upsample: 2x (causal ConvTranspose1d + ConvNeXt block) --- + up_map = { + "0.conv.weight": (T.A_GEN_WAV_UP_CONV, ".weight"), + "0.conv.bias": (T.A_GEN_WAV_UP_CONV, ".bias"), + "1.dwconv.conv.weight": (T.A_GEN_WAV_UP_DWCONV, ".weight"), + "1.dwconv.conv.bias": (T.A_GEN_WAV_UP_DWCONV, ".bias"), + "1.norm.weight": (T.A_GEN_WAV_UP_NORM, ".weight"), + "1.norm.bias": (T.A_GEN_WAV_UP_NORM, ".bias"), + "1.pwconv1.weight": (T.A_GEN_WAV_UP_PW1, ".weight"), + "1.pwconv1.bias": (T.A_GEN_WAV_UP_PW1, ".bias"), + "1.pwconv2.weight": (T.A_GEN_WAV_UP_PW2, ".weight"), + "1.pwconv2.bias": (T.A_GEN_WAV_UP_PW2, ".bias"), + "1.gamma": (T.A_GEN_WAV_UP_GAMMA, ""), + } + for bid in range(len(wav_config["upsampling_ratios"])): + for key, (tensor_id, suffix) in up_map.items(): + yield (self.format_tensor_name(tensor_id, bid, suffix=suffix), get(f"decoder.upsample.{bid}.{key}")) + + # --- DAC decoder --- + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".weight"), get("decoder.decoder.0.conv.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get("decoder.decoder.0.conv.bias")) + + n_dac_blocks = len(wav_config["upsample_rates"]) + for bid in range(n_dac_blocks): + py = bid + 1 # decoder.decoder.0 is the entry conv, blocks start at 1 + + a, b = snake_fold(get(f"decoder.decoder.{py}.block.0.alpha"), get(f"decoder.decoder.{py}.block.0.beta")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), get(f"decoder.decoder.{py}.block.1.conv.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".bias"), get(f"decoder.decoder.{py}.block.1.conv.bias")) + + for xid in range(3): + ridx = xid + 2 # block.2/3/4 are the 3 residual units + + a1, b1 = snake_fold(get(f"decoder.decoder.{py}.block.{ridx}.act1.alpha"), get(f"decoder.decoder.{py}.block.{ridx}.act1.beta")) + name1 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_ACT1].format(bid=bid, xid=xid) + yield (name1 + ".alpha", a1) + yield (name1 + ".beta", b1) + + name_conv1 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_CONV1].format(bid=bid, xid=xid) + yield (name_conv1 + ".weight", get(f"decoder.decoder.{py}.block.{ridx}.conv1.conv.weight")) + yield (name_conv1 + ".bias", get(f"decoder.decoder.{py}.block.{ridx}.conv1.conv.bias")) + + a2, b2 = snake_fold(get(f"decoder.decoder.{py}.block.{ridx}.act2.alpha"), get(f"decoder.decoder.{py}.block.{ridx}.act2.beta")) + name2 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_ACT2].format(bid=bid, xid=xid) + yield (name2 + ".alpha", a2) + yield (name2 + ".beta", b2) + + name_conv2 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_CONV2].format(bid=bid, xid=xid) + yield (name_conv2 + ".weight", get(f"decoder.decoder.{py}.block.{ridx}.conv2.conv.weight")) + yield (name_conv2 + ".bias", get(f"decoder.decoder.{py}.block.{ridx}.conv2.conv.bias")) + + a5, b5 = snake_fold(get("decoder.decoder.5.alpha"), get("decoder.decoder.5.beta")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a5) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b5) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), get("decoder.decoder.6.conv.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get("decoder.decoder.6.conv.bias")) diff --git a/docs/development/HOWTO-add-model.md b/docs/development/HOWTO-add-model.md index 102f479eb02c..270e6b735651 100644 --- a/docs/development/HOWTO-add-model.md +++ b/docs/development/HOWTO-add-model.md @@ -133,6 +133,7 @@ Note: - To debug the multimodal preprocessor and encoder, you can use [llama-mtmd-debug](tools/mtmd/debug/mtmd-debug.cpp). - Adding a model-specific API or CLI is an anti-pattern in `libmtmd`. The goal of `libmtmd` is to provide an easy-to-use, model-agnostic library for multimodal pipeline. - In most cases, `llama-mtmd-cli` should not be modified. If a model requires a specific prompt, either let the user provide it or bake it into the Jinja chat template. +- For audio generation models, see `tools/mtmd/README-dev.md` ## Tips and tricks diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 6b0a26b63d89..8516222cccbb 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -323,6 +323,7 @@ class Clip: PROJECTOR_TYPE = "clip.projector_type" HAS_VISION_ENCODER = "clip.has_vision_encoder" HAS_AUDIO_ENCODER = "clip.has_audio_encoder" + HAS_GEN_AUDIO_ENCODER = "clip.has_gen_audio_encoder" HAS_LLAVA_PROJECTOR = "clip.has_llava_projector" class ClipVision: @@ -397,6 +398,18 @@ class Projector: DOWNSAMPLE_RATE = "clip.audio.projector.downsample_rate" HEAD_COUNT = "clip.audio.projector.head_count" + class ClipGenAudio: + PROJECTOR_TYPE = "clip.gen.audio.projector_type" # for mixed modality models + EMBEDDING_LENGTH = "clip.gen.audio.embedding_length" + FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length" + BLOCK_COUNT = "clip.gen.audio.block_count" + PROJECTION_DIM = "clip.gen.audio.projection_dim" + + class Attention: + HEAD_COUNT = "clip.gen.audio.attention.head_count" + HEAD_COUNT_KV = "clip.gen.audio.attention.head_count_kv" + LAYERNORM_EPS = "clip.gen.audio.attention.layer_norm_epsilon" + class Diffusion: SHIFT_LOGITS = "diffusion.shift_logits" @@ -558,6 +571,7 @@ class MODEL_ARCH(IntEnum): TALKIE = auto() MELLUM = auto() NANBEIGE = auto() + QWEN3TTS = auto() class VISION_PROJECTOR_TYPE(IntEnum): @@ -958,6 +972,65 @@ class MODEL_TENSOR(IntEnum): A_ENC_DOWNSAMPLE_CONV = auto() # mimo-audio-tokenizer: post-transformer downsample conv A_ENC_DOWNSAMPLE_NORM = auto() # mimo-audio-tokenizer: post-transformer downsample norm A_ENC_RVQ_CODEBOOK = auto() # mimo-audio-tokenizer: residual vector quantizer codebook, per quantizer index + A_ENC_CONV_RES2 = auto() # qwen3tts + A_ENC_SE_CONV1 = auto() # qwen3tts + A_ENC_SE_CONV2 = auto() # qwen3tts + A_ENC_ASP_ATTN = auto() # qwen3tts + A_ENC_ASP_TDNN = auto() # qwen3tts + # qwen3tts code_predictor: predicts the remaining RVQ codebooks + A_GEN_CODE_PROJ_IN = auto() # small_to_mtp_projection + A_GEN_CODE_EMBD = auto() # per-codebook embedding table, merged 3D [n_codebooks, vocab, dim] + A_GEN_CODE_HEAD = auto() # per-codebook output head, merged 3D [n_codebooks, vocab, dim] + A_GEN_CODE_OUT_EMBD = auto() # codebook-0 embedding, re-fed into the talker backbone (talker.model.codec_embedding) + A_GEN_CODE_ATTN_NORM = auto() + A_GEN_CODE_ATTN_Q = auto() + A_GEN_CODE_ATTN_Q_NORM = auto() + A_GEN_CODE_ATTN_K = auto() + A_GEN_CODE_ATTN_K_NORM = auto() + A_GEN_CODE_ATTN_V = auto() + A_GEN_CODE_ATTN_OUT = auto() + A_GEN_CODE_FFN_NORM = auto() + A_GEN_CODE_FFN_GATE = auto() + A_GEN_CODE_FFN_UP = auto() + A_GEN_CODE_FFN_DOWN = auto() + A_GEN_CODE_OUTPUT_NORM = auto() + # qwen3tts code2wav: RVQ codes -> raw PCM + A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D) + A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj + A_GEN_WAV_QUANT_FIRST_CB = auto() # semantic RVQ codebook (1 layer), folded from embedding_sum/cluster_usage + A_GEN_WAV_QUANT_REST_IN = auto() # acoustic RVQ, in_proj + A_GEN_WAV_QUANT_REST_OUT = auto() # acoustic RVQ, out_proj + A_GEN_WAV_QUANT_REST_CB = auto() # acoustic RVQ codebooks, merged 3D [15, vocab, dim] + A_GEN_WAV_PRE_CONV = auto() + A_GEN_WAV_TFM_IN_PROJ = auto() + A_GEN_WAV_TFM_OUT_PROJ = auto() + A_GEN_WAV_TFM_OUTPUT_NORM = auto() + A_GEN_WAV_TFM_ATTN_NORM = auto() + A_GEN_WAV_TFM_ATTN_Q = auto() + A_GEN_WAV_TFM_ATTN_K = auto() + A_GEN_WAV_TFM_ATTN_V = auto() + A_GEN_WAV_TFM_ATTN_OUT = auto() + A_GEN_WAV_TFM_ATTN_SCALE = auto() # layer scale (gamma) on the attn output + A_GEN_WAV_TFM_FFN_NORM = auto() + A_GEN_WAV_TFM_FFN_GATE = auto() + A_GEN_WAV_TFM_FFN_UP = auto() + A_GEN_WAV_TFM_FFN_DOWN = auto() + A_GEN_WAV_TFM_FFN_SCALE = auto() # layer scale (gamma) on the FFN output + A_GEN_WAV_UP_CONV = auto() # causal ConvTranspose1d, 2x upsample + A_GEN_WAV_UP_DWCONV = auto() # ConvNeXt depthwise conv + A_GEN_WAV_UP_NORM = auto() # ConvNeXt LayerNorm + A_GEN_WAV_UP_PW1 = auto() # ConvNeXt pointwise conv 1 (expand) + A_GEN_WAV_UP_PW2 = auto() # ConvNeXt pointwise conv 2 (project) + A_GEN_WAV_UP_GAMMA = auto() # ConvNeXt layer scale + A_GEN_WAV_DAC_ENTRY = auto() # DAC conv_pre + A_GEN_WAV_DAC_UP_SNAKE = auto() # DAC per-block SnakeBeta before the upsample conv + A_GEN_WAV_DAC_UP_CONV = auto() # DAC per-block causal ConvTranspose1d + A_GEN_WAV_DAC_RES_ACT1 = auto() # DAC residual unit, SnakeBeta before conv1 + A_GEN_WAV_DAC_RES_CONV1 = auto() # DAC residual unit, dilated causal conv + A_GEN_WAV_DAC_RES_ACT2 = auto() # DAC residual unit, SnakeBeta before conv2 + A_GEN_WAV_DAC_RES_CONV2 = auto() # DAC residual unit, pointwise causal conv + A_GEN_WAV_DAC_POST_SNAKE = auto() # DAC final SnakeBeta + A_GEN_WAV_DAC_POST_CONV = auto() # DAC conv_post -> 1-channel PCM A_MMPROJ = auto() A_MMPROJ_FC = auto() A_MM_NORM_PRE = auto() @@ -1170,6 +1243,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.TALKIE: "talkie", MODEL_ARCH.MELLUM: "mellum", MODEL_ARCH.NANBEIGE: "nanbeige", + MODEL_ARCH.QWEN3TTS: "qwen3tts", } VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = { @@ -1567,6 +1641,63 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: "a.downsample.conv", MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: "a.downsample.norm", MODEL_TENSOR.A_ENC_RVQ_CODEBOOK: "a.rvq.codebook", + MODEL_TENSOR.A_ENC_CONV_RES2: "a.blk.{bid}.res2.{xid}", + MODEL_TENSOR.A_ENC_SE_CONV1: "a.blk.{bid}.se_conv1", + MODEL_TENSOR.A_ENC_SE_CONV2: "a.blk.{bid}.se_conv2", + MODEL_TENSOR.A_ENC_ASP_ATTN: "a.asp_attn", + MODEL_TENSOR.A_ENC_ASP_TDNN: "a.asp_tdnn", + MODEL_TENSOR.A_GEN_CODE_PROJ_IN: "a.gen.code.proj_in", + MODEL_TENSOR.A_GEN_CODE_EMBD: "a.gen.code.embd", + MODEL_TENSOR.A_GEN_CODE_HEAD: "a.gen.code.head", + MODEL_TENSOR.A_GEN_CODE_OUT_EMBD: "a.gen.code.out_embd", + MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic clip.cpp block loader (TN_LN_1) + MODEL_TENSOR.A_GEN_CODE_ATTN_Q: "a.gen.code.blk.{bid}.attn_q", + MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM: "a.gen.code.blk.{bid}.attn_q_norm", + MODEL_TENSOR.A_GEN_CODE_ATTN_K: "a.gen.code.blk.{bid}.attn_k", + MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM: "a.gen.code.blk.{bid}.attn_k_norm", + MODEL_TENSOR.A_GEN_CODE_ATTN_V: "a.gen.code.blk.{bid}.attn_v", + MODEL_TENSOR.A_GEN_CODE_ATTN_OUT: "a.gen.code.blk.{bid}.attn_out", + MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic clip.cpp block loader (TN_LN_2) + MODEL_TENSOR.A_GEN_CODE_FFN_GATE: "a.gen.code.blk.{bid}.ffn_gate", + MODEL_TENSOR.A_GEN_CODE_FFN_UP: "a.gen.code.blk.{bid}.ffn_up", + MODEL_TENSOR.A_GEN_CODE_FFN_DOWN: "a.gen.code.blk.{bid}.ffn_down", + MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM: "a.gen.code.output_norm", + MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj", + MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj", + MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook", + MODEL_TENSOR.A_GEN_WAV_QUANT_REST_IN: "a.gen.wav.quant.rest.in_proj", + MODEL_TENSOR.A_GEN_WAV_QUANT_REST_OUT: "a.gen.wav.quant.rest.out_proj", + MODEL_TENSOR.A_GEN_WAV_QUANT_REST_CB: "a.gen.wav.quant.rest.codebook", + MODEL_TENSOR.A_GEN_WAV_PRE_CONV: "a.gen.wav.pre_conv", + MODEL_TENSOR.A_GEN_WAV_TFM_IN_PROJ: "a.gen.wav.tfm.in_proj", + MODEL_TENSOR.A_GEN_WAV_TFM_OUT_PROJ: "a.gen.wav.tfm.out_proj", + MODEL_TENSOR.A_GEN_WAV_TFM_OUTPUT_NORM: "a.gen.wav.tfm.output_norm", + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM: "a.gen.wav.tfm.blk.{bid}.ln1", + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q: "a.gen.wav.tfm.blk.{bid}.attn_q", + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K: "a.gen.wav.tfm.blk.{bid}.attn_k", + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V: "a.gen.wav.tfm.blk.{bid}.attn_v", + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT: "a.gen.wav.tfm.blk.{bid}.attn_out", + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE: "a.gen.wav.tfm.blk.{bid}.ls1", + MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM: "a.gen.wav.tfm.blk.{bid}.ln2", + MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE: "a.gen.wav.tfm.blk.{bid}.ffn_gate", + MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP: "a.gen.wav.tfm.blk.{bid}.ffn_up", + MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN: "a.gen.wav.tfm.blk.{bid}.ffn_down", + MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE: "a.gen.wav.tfm.blk.{bid}.ls2", + MODEL_TENSOR.A_GEN_WAV_UP_CONV: "a.gen.wav.up.blk.{bid}.conv", + MODEL_TENSOR.A_GEN_WAV_UP_DWCONV: "a.gen.wav.up.blk.{bid}.dwconv", + MODEL_TENSOR.A_GEN_WAV_UP_NORM: "a.gen.wav.up.blk.{bid}.norm", + MODEL_TENSOR.A_GEN_WAV_UP_PW1: "a.gen.wav.up.blk.{bid}.pw1", + MODEL_TENSOR.A_GEN_WAV_UP_PW2: "a.gen.wav.up.blk.{bid}.pw2", + MODEL_TENSOR.A_GEN_WAV_UP_GAMMA: "a.gen.wav.up.blk.{bid}.gamma", + MODEL_TENSOR.A_GEN_WAV_DAC_ENTRY: "a.gen.wav.dac.entry", + MODEL_TENSOR.A_GEN_WAV_DAC_UP_SNAKE: "a.gen.wav.dac.blk.{bid}.snake", + MODEL_TENSOR.A_GEN_WAV_DAC_UP_CONV: "a.gen.wav.dac.blk.{bid}.conv", + MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT1: "a.gen.wav.dac.blk.{bid}.res.{xid}.act1", + MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV1: "a.gen.wav.dac.blk.{bid}.res.{xid}.conv1", + MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT2: "a.gen.wav.dac.blk.{bid}.res.{xid}.act2", + MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV2: "a.gen.wav.dac.blk.{bid}.res.{xid}.conv2", + MODEL_TENSOR.A_GEN_WAV_DAC_POST_SNAKE: "a.gen.wav.dac.post_snake", + MODEL_TENSOR.A_GEN_WAV_DAC_POST_CONV: "a.gen.wav.dac.post_conv", MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}", MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc", MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre", @@ -1821,6 +1952,63 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_ENC_CONV_NORM, MODEL_TENSOR.A_ENC_CONV_PW1, MODEL_TENSOR.A_ENC_CONV_PW2, + MODEL_TENSOR.A_ENC_CONV_RES2, + MODEL_TENSOR.A_ENC_SE_CONV1, + MODEL_TENSOR.A_ENC_SE_CONV2, + MODEL_TENSOR.A_ENC_ASP_ATTN, + MODEL_TENSOR.A_ENC_ASP_TDNN, + MODEL_TENSOR.A_GEN_CODE_PROJ_IN, + MODEL_TENSOR.A_GEN_CODE_EMBD, + MODEL_TENSOR.A_GEN_CODE_HEAD, + MODEL_TENSOR.A_GEN_CODE_OUT_EMBD, + MODEL_TENSOR.A_GEN_CODE_ATTN_NORM, + MODEL_TENSOR.A_GEN_CODE_ATTN_Q, + MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM, + MODEL_TENSOR.A_GEN_CODE_ATTN_K, + MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM, + MODEL_TENSOR.A_GEN_CODE_ATTN_V, + MODEL_TENSOR.A_GEN_CODE_ATTN_OUT, + MODEL_TENSOR.A_GEN_CODE_FFN_NORM, + MODEL_TENSOR.A_GEN_CODE_FFN_GATE, + MODEL_TENSOR.A_GEN_CODE_FFN_UP, + MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, + MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM, + MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN, + MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT, + MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB, + MODEL_TENSOR.A_GEN_WAV_QUANT_REST_IN, + MODEL_TENSOR.A_GEN_WAV_QUANT_REST_OUT, + MODEL_TENSOR.A_GEN_WAV_QUANT_REST_CB, + MODEL_TENSOR.A_GEN_WAV_PRE_CONV, + MODEL_TENSOR.A_GEN_WAV_TFM_IN_PROJ, + MODEL_TENSOR.A_GEN_WAV_TFM_OUT_PROJ, + MODEL_TENSOR.A_GEN_WAV_TFM_OUTPUT_NORM, + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM, + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V, + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT, + MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE, + MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM, + MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE, + MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP, + MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN, + MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE, + MODEL_TENSOR.A_GEN_WAV_UP_CONV, + MODEL_TENSOR.A_GEN_WAV_UP_DWCONV, + MODEL_TENSOR.A_GEN_WAV_UP_NORM, + MODEL_TENSOR.A_GEN_WAV_UP_PW1, + MODEL_TENSOR.A_GEN_WAV_UP_PW2, + MODEL_TENSOR.A_GEN_WAV_UP_GAMMA, + MODEL_TENSOR.A_GEN_WAV_DAC_ENTRY, + MODEL_TENSOR.A_GEN_WAV_DAC_UP_SNAKE, + MODEL_TENSOR.A_GEN_WAV_DAC_UP_CONV, + MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT1, + MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV1, + MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT2, + MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV2, + MODEL_TENSOR.A_GEN_WAV_DAC_POST_SNAKE, + MODEL_TENSOR.A_GEN_WAV_DAC_POST_CONV, MODEL_TENSOR.A_ENC_CONV_NORM_MEAN, MODEL_TENSOR.A_ENC_CONV_NORM_VAR, MODEL_TENSOR.A_ENC_MEL_FILTERS, @@ -4648,6 +4836,22 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.QWEN3TTS: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], } # tensors that will not be serialized @@ -4922,6 +5126,8 @@ class VisionProjectorType: GLM4V = "glm4v" YOUTUVL = "youtuvl" NEMOTRON_V2_VL = "nemotron_v2_vl" + QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder + QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor HUNYUANVL = "hunyuanvl" PARAKEET = "parakeet" # audio MINIMAXM3 = "minimax_m3" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index c5905164c356..39da9f2c05fb 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -280,6 +280,10 @@ def add_key_value(self, key: str, val: Any, vtype: GGUFValueType, sub_type: GGUF self.kv_data[0][key] = GGUFValue(value=val, type=vtype, sub_type=sub_type) + def remove_key(self, key: str) -> None: + for kv_data in self.kv_data: + kv_data.pop(key, None) + def add_uint8(self, key: str, val: int) -> None: self.add_key_value(key,val, GGUFValueType.UINT8) @@ -1144,7 +1148,11 @@ def add_remove_extra_whitespaces(self, value: bool) -> None: def add_precompiled_charsmap(self, charsmap: bytes) -> None: self.add_array(Keys.Tokenizer.PRECOMPILED_CHARSMAP, charsmap) - def add_chat_template(self, value: str | Sequence[Mapping[str, str]]) -> None: + def add_chat_template(self, value: str | Sequence[Mapping[str, str]] | None) -> None: + if value is None: + self.remove_key(Keys.Tokenizer.CHAT_TEMPLATE) + return + if not isinstance(value, str): template_default = None template_names = set() @@ -1199,6 +1207,9 @@ def add_clip_has_vision_encoder(self, value: bool) -> None: def add_clip_has_audio_encoder(self, value: bool) -> None: self.add_bool(Keys.Clip.HAS_AUDIO_ENCODER, value) + def add_clip_has_gen_audio_encoder(self, value: bool) -> None: + self.add_bool(Keys.Clip.HAS_GEN_AUDIO_ENCODER, value) + def add_clip_projector_type(self, value: str) -> None: self.add_string(Keys.Clip.PROJECTOR_TYPE, value) @@ -1401,6 +1412,32 @@ def add_audio_projector_downsample_rate(self, value: int) -> None: def add_audio_projector_head_count(self, value: int) -> None: self.add_uint32(Keys.ClipAudio.Projector.HEAD_COUNT, value) + # audio generation (mmproj) + + def add_clip_gen_audio_projector_type(self, value: str) -> None: + self.add_string(Keys.ClipGenAudio.PROJECTOR_TYPE, value) + + def add_gen_audio_projection_dim(self, value: int) -> None: + self.add_uint32(Keys.ClipGenAudio.PROJECTION_DIM, value) + + def add_gen_audio_embedding_length(self, value: int) -> None: + self.add_uint32(Keys.ClipGenAudio.EMBEDDING_LENGTH, value) + + def add_gen_audio_feed_forward_length(self, value: int) -> None: + self.add_uint32(Keys.ClipGenAudio.FEED_FORWARD_LENGTH, value) + + def add_gen_audio_block_count(self, value: int) -> None: + self.add_uint32(Keys.ClipGenAudio.BLOCK_COUNT, value) + + def add_gen_audio_head_count(self, value: int) -> None: + self.add_uint32(Keys.ClipGenAudio.Attention.HEAD_COUNT, value) + + def add_gen_audio_head_count_kv(self, value: int) -> None: + self.add_uint32(Keys.ClipGenAudio.Attention.HEAD_COUNT_KV, value) + + def add_gen_audio_attention_layernorm_eps(self, value: float) -> None: + self.add_float32(Keys.ClipGenAudio.Attention.LAYERNORM_EPS, value) + def add_xielu_alpha_p(self, values: Sequence[float]): self.add_array(Keys.xIELU.ALPHA_P, values) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 1e991b873cea..7892342e473d 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -2109,6 +2109,7 @@ class TensorNameMap: "conformer.subsample_conv_projection.layer{bid}.conv", # gemma4 "sound_encoder.encoder.subsampling.layers.{bid}", # parakeet "encoder.conv{bid}", # mimo-audio-tokenizer + "speaker_encoder.blocks.{bid}.conv", # qwen3tts speaker encoder (only bid=0, the stem TDNN) ), MODEL_TENSOR.A_ENC_CONV1D_NORM: ( @@ -2126,6 +2127,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_CONV_OUT: ( "audio_tower.conv_out", # qwen3omni + "speaker_encoder.mfa.conv", # qwen3tts speaker encoder: multi-layer feature aggregation ), MODEL_TENSOR.A_PRE_NORM: (), @@ -2336,7 +2338,8 @@ class TensorNameMap: MODEL_TENSOR.A_MMPROJ_FC: ( "audio.multi_modal_projector.linear", # qwen2audio "audio_tower.proj", # qwen2omni - "model.audio_tower.output_proj" # gemma4 + "model.audio_tower.output_proj", # gemma4 + "speaker_encoder.fc", # qwen3tts speaker encoder: final speaker embedding projection ), MODEL_TENSOR.A_MM_NORM_PRE: ( @@ -2411,6 +2414,7 @@ class TensorNameMap: "conformer.layers.{bid}.lconv1d.linear_start", # gemma3n "sound_encoder.encoder.layers.{bid}.conv.pointwise_conv1", # parakeet "encoder.layers.{bid}.conv.up_conv", # granite_speech + "speaker_encoder.blocks.{bid}.tdnn1.conv", # qwen3tts speaker encoder ), MODEL_TENSOR.A_ENC_CONV_PW2: ( @@ -2418,6 +2422,23 @@ class TensorNameMap: "conformer.layers.{bid}.lconv1d.linear_end", # gemma3n "sound_encoder.encoder.layers.{bid}.conv.pointwise_conv2", # parakeet "encoder.layers.{bid}.conv.down_conv", # granite_speech + "speaker_encoder.blocks.{bid}.tdnn2.conv", # qwen3tts speaker encoder + ), + + MODEL_TENSOR.A_ENC_SE_CONV1: ( + "speaker_encoder.blocks.{bid}.se_block.conv1", # qwen3tts + ), + + MODEL_TENSOR.A_ENC_SE_CONV2: ( + "speaker_encoder.blocks.{bid}.se_block.conv2", # qwen3tts + ), + + MODEL_TENSOR.A_ENC_ASP_ATTN: ( + "speaker_encoder.asp.conv", # qwen3tts + ), + + MODEL_TENSOR.A_ENC_ASP_TDNN: ( + "speaker_encoder.asp.tdnn.conv", # qwen3tts ), MODEL_TENSOR.A_ENC_NORM_CONV: ( diff --git a/skills/code-review/SKILL.md b/skills/code-review/SKILL.md index 726edbb0ccb1..b9372ddda870 100644 --- a/skills/code-review/SKILL.md +++ b/skills/code-review/SKILL.md @@ -119,6 +119,7 @@ Public API changes carry a higher bar than internal ones (`CONTRIBUTING.md`). Re - In most cases, `build_vit` should be enough to build the transformer graph for vision models. Do not add a loop to build the transformer graph manually, unless you have a very good reason to do so. If you do, please explain why in the PR description. - If you need a dedicated preprocessor, there is a high chance that it can be a derived class from one of the existing preprocessors. Check carefully before adding a new preprocessor class. - If the model need a new public API in `mtmd.h`, open a discussion first. +- For audio generation models, see `tools/mtmd/README-dev.md` ## General (always) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index ea0ddd114c0c..836cfade226c 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -144,6 +144,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_TALKIE, "talkie" }, { LLM_ARCH_MELLUM, "mellum" }, { LLM_ARCH_NANBEIGE, "nanbeige" }, + { LLM_ARCH_QWEN3TTS, "qwen3tts" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; @@ -1026,6 +1027,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_MINIMAX_M3: case LLM_ARCH_MISTRAL4: case LLM_ARCH_KIMI_LINEAR: + case LLM_ARCH_QWEN3TTS: return false; default: return true; diff --git a/src/llama-arch.h b/src/llama-arch.h index cbc97085ea79..49c2a6ac3997 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -149,6 +149,7 @@ enum llm_arch { LLM_ARCH_MINIMAX_M3, LLM_ARCH_DFLASH, LLM_ARCH_NANBEIGE, + LLM_ARCH_QWEN3TTS, LLM_ARCH_UNKNOWN, }; diff --git a/src/llama-ext.h b/src/llama-ext.h index 348bbae95770..35d6e58adfa8 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -124,3 +124,9 @@ LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model); // returns the number of extracted layers from target model LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); + +// retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab) +// returns total number of elements or 0 on error +// if out is nullptr, returns the number of tokens without writing to out +// caller must allocate enough memory for out before calling +LLAMA_API uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 333f506de557..dda311c47bbf 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -112,6 +112,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_qwen3vl(params); case LLM_ARCH_QWEN3VLMOE: return new llama_model_qwen3vlmoe(params); + case LLM_ARCH_QWEN3TTS: + return new llama_model_qwen3tts(params); case LLM_ARCH_PHI2: return new llama_model_phi2(params); case LLM_ARCH_PHI3: @@ -2693,6 +2695,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_QWEN3VLMOE: case LLM_ARCH_QWEN35: case LLM_ARCH_QWEN35MOE: + case LLM_ARCH_QWEN3TTS: return LLAMA_ROPE_TYPE_IMROPE; case LLM_ARCH_GLM4: @@ -2908,3 +2911,38 @@ const int32_t * llama_model_target_layer_ids(const struct llama_model * model) { uint32_t llama_model_target_layer_ids_n(const struct llama_model * model) { return (uint32_t) model->target_layer_ids.size(); } + +uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out) { + if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr) { + return 0; + } + + const ggml_tensor * tensor = model->tok_embd; + const size_t nelements = ggml_nelements(tensor); + GGML_ASSERT(nelements <= UINT32_MAX); // for the return type + + if (out == nullptr) { + return (uint32_t) nelements; + } + + if (tensor->type == GGML_TYPE_F32) { + ggml_backend_tensor_get(tensor, out, 0, nelements * sizeof(float)); + return (uint32_t) nelements; + } + + std::vector buf(ggml_nbytes(tensor)); + ggml_backend_tensor_get(tensor, buf.data(), 0, buf.size()); + + const ggml_type_traits * traits = ggml_get_type_traits(tensor->type); + if (tensor->type == GGML_TYPE_F16) { + ggml_fp16_to_fp32_row((const ggml_fp16_t *) buf.data(), out, nelements); + } else if (tensor->type == GGML_TYPE_BF16) { + ggml_bf16_to_fp32_row((const ggml_bf16_t *) buf.data(), out, nelements); + } else if (ggml_is_quantized(tensor->type) && traits->to_float != nullptr) { + traits->to_float(buf.data(), out, nelements); + } else { + GGML_ABORT("unsupported tensor type for dequantization: %s", ggml_type_name(tensor->type)); + } + + return (uint32_t) nelements; +} diff --git a/src/models/models.h b/src/models/models.h index 5f206621d579..ad3dadaf3932 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -596,6 +596,11 @@ struct llama_model_qwen3vlmoe : public llama_model_base { }; +struct llama_model_qwen3tts : public llama_model_qwen3vl { + llama_model_qwen3tts(const struct llama_model_params & params) : llama_model_qwen3vl(params) {} +}; + + struct llama_model_phi2 : public llama_model_base { llama_model_phi2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/src/models/qwen3tts.cpp b/src/models/qwen3tts.cpp new file mode 100644 index 000000000000..3604f844c393 --- /dev/null +++ b/src/models/qwen3tts.cpp @@ -0,0 +1,3 @@ +#include "models.h" + +// llama_model_qwen3tts reuses llama_model_qwen3vl's hparams/tensors/graph logic diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 724d6140d193..5596620f0782 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -16,11 +16,16 @@ void llama_model_qwen3vl::load_arch_hparams(llama_model_loader & ml) { void llama_model_qwen3vl::load_arch_tensors(llama_model_loader &) { LLAMA_LOAD_LOCALS; + int64_t n_vocab_out = n_vocab; + if (arch == LLM_ARCH_QWEN3TTS) { + n_vocab_out = 3072; + } + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); // output output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_out}, TENSOR_NOT_REQUIRED); // if output is NULL, init from the input tok embed if (output == NULL) { output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); @@ -166,6 +171,24 @@ llama_model_qwen3vl::graph::graph(const llama_model & model, const llm_graph_par // lm_head cur = build_lora_mm(model.output, cur, model.output_s); + int64_t n_vocab_in = model.tok_embd->ne[1]; + int64_t n_vocab_out = model.output->ne[1]; + if (n_vocab_in > n_vocab_out) { + // case: Qwen3TTS model with codec_head as output + GGML_ASSERT(model.output_norm); + int64_t pad = n_vocab_in - n_vocab_out; + + // using this trick to get a scalar -inf tensor to pad the output + ggml_tensor * neg_inf = ggml_scale_bias(ctx0, + ggml_view_1d(ctx0, model.output_norm, 1, 0), + 0.0f, -INFINITY); + neg_inf = ggml_repeat_4d(ctx0, neg_inf, pad, cur->ne[1], 1, 1); + cur = ggml_concat(ctx0, neg_inf, cur, 0); // [padded .. n_vocab_out, n_stream] + + } else if (n_vocab_in < n_vocab_out) { + GGML_ABORT("invalid case"); + } + cb(cur, "result_output", -1); res->t_logits = cur; diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 4336e4e13d4f..1654f122a731 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -113,6 +113,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_layer = 3; } else if (arch == LLM_ARCH_CHAMELEON) { n_vocab = 10240; + } else if (arch == LLM_ARCH_QWEN3TTS) { + n_vocab = 4096; // must be >= the hard-coded codec head size (3072) } const uint32_t n_embd_head = n_embd / n_head; diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 15040e4af5f9..4675fb9a97b6 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -18,6 +18,8 @@ add_library(mtmd mtmd-image.cpp mtmd.h mtmd-helper.cpp + mtmd-helper-gen.cpp + mtmd-helper-common.h mtmd-helper.h clip.cpp clip.h @@ -52,6 +54,8 @@ add_library(mtmd models/mimovl.cpp models/qwen3a.cpp models/mimo-audio.cpp + models/qwen3tts-spkenc.cpp + models/qwen3tts-gen.cpp models/step3vl.cpp models/siglip.cpp models/whisper-enc.cpp diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md index 3a08915876c1..3cddd085ec6c 100644 --- a/tools/mtmd/README-dev.md +++ b/tools/mtmd/README-dev.md @@ -33,3 +33,52 @@ A typical pipeline of the core libmtmd is as follows: We provide a set of helper functions via `mtmd_helper` to make using libmtmd easier. The helper provides: - Image, audio and video file decoding (for example, decode raw JPEG into RGB bitmap) - Manage `llama_batch` and calls to `llama_decode` + +## Audio generation support + +Audio generation is added to mtmd in PR [#26254](https://github.com/ggml-org/llama.cpp/pull/26254) + +Currently, we support the 3-stage pipeline below which should cover most TTS models: +- Stage 1: Backbone / Semantic Stage: Backbone model accepts text prompt and reference voice as input +- Stage 2: Acoustic Detail Generator: A model takes the hidden state from backbone and generate audio details (usually as audio codes or mel-spectrogram) +- Stage 3: Waveform Reconstruction: Convert the semantic and acoustic data from previous stages to the final waveform + +For example, Qwen3-TTS: +- Reference voice is encoded using ECAPA-TDNN speaker encoder (`speaker_encoder`) +- Text prompt and reference voice are processed via a backbone (`talker.model`) +- A model converts sampled semantic token and hidden state from stage 2 into a list of 15 acoustic codes (`talker.code_predictor`) +- 16 generated codes are converted into waveform (`code2wav`) + +### API design constraints + +Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system is designed to be flexible and reusable by new models. + +`mtmd_gen_audio` is split into 2 main API: +- Core API `mtmd.h`: handles main inference. Important: the API surface must be stateless; caller must handle state management and audio frame accumulation. +- Helper API `mtmd-helper.h`: provides a model-agnostic stateful API. Usage example can be found in the `tools/tts` directory. + +### Checklist for porting new audio generation models to mtmd + +1. Establish a list of reusable and missing components from the current mtmd implementation. +2. For GGUF conversion: + - Backbone model should be converted to a normal text model (loadable via `libllama`) + - If model used hard-coded embedding row ID, append them to token embeddings and assign token name for them (see `qwen3tts.py`) + - If model have a specific output logits head for audio codes (usually semantic code), keep the head as-is and pad the logits at inference time (see `src/models/qwen3vl.cpp`) + - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `clip_context` if necessary) + - Note: it should use `ggml_build_forward_select` to select graphs if multiple graphs living in the same context + - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams to `clip.cpp` as they won't likely to change. + - For tensor naming: + - Prefixed with `a.*` for tensors used by speaker encoder pipeline + - Prefixed with `a.gen.*` for generation stages (code / mel-spectrogram / PCM generation) +3. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this: + - 10-20% changes is to add new backbone (text) model and conversion + - 60% changes inside `mtmd-helper-gen.cpp` + - 10% changes inside `libmtmd` and `clip.cpp` systems + - The rest downstream code (CLI, server) should have no changes at all +4. Update usage documentation in `tools/tts/README.md` + +IMPORTANT: If your model needs changes that don't fit the existing infrastructure, **open an issue first for discussion**. + +No-go checklist (these will get the PR rejected and require discussion before proceeding): +- Violating the API design constraints stated above +- Adding a new model-specific binary: the API and binary surface must stay model-agnostic diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h index 29352abb4c0b..e12140ba009d 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/clip-graph.h @@ -54,6 +54,9 @@ struct clip_graph { clip_graph(clip_ctx * ctx, const clip_image_f32 & img); + // build sub-graph, reuse buf from parent + clip_graph(const clip_graph & parent); + virtual ~clip_graph() = default; virtual ggml_cgraph * build() = 0; diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index d42b38222c27..e1567ee5bafb 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -32,6 +32,7 @@ #define KEY_PROJ_TYPE "clip.projector_type" #define KEY_HAS_AUDIO_ENC "clip.has_audio_encoder" #define KEY_HAS_VISION_ENC "clip.has_vision_encoder" +#define KEY_HAS_GEN_AUDIO_ENC "clip.has_gen_audio_encoder" #define KEY_USE_GELU "clip.use_gelu" #define KEY_USE_SILU "clip.use_silu" @@ -89,6 +90,8 @@ #define KEY_A_ATTN_WINDOW_SIZE "clip.audio.window_size" // mimo-audio-tokenizer: sliding-window radius #define KEY_A_LOCAL_BLOCK_COUNT "clip.audio.local_block_count" // mimo-v2.5: input_local_transformer layer count #define KEY_A_LOCAL_GROUP_SIZE "clip.audio.local_group_size" // mimo-v2.5: input_local_transformer grouping size +// audio generation (gen-audio)-specific +#define KEY_GEN_AUDIO_PROJ_TYPE "clip.gen.audio.projector_type" // for models with mixed modalities #define KEY_AUDIO_SUBSAMPLING_FACTOR "clip.audio.subsampling_factor" // @@ -201,6 +204,48 @@ #define TN_MM_A_LOCAL_LN2 "mm.a.local_blk.%d.ln2.%s" #define TN_MM_A_LOCAL_NORM "mm.a.local_norm.%s" +// qwen3tts speaker encoder (ECAPA-TDNN) +#define TN_A_SE_CONV1 "a.blk.%d.se_conv1.%s" +#define TN_A_SE_CONV2 "a.blk.%d.se_conv2.%s" +#define TN_A_CONV_RES2 "a.blk.%d.res2.%d.%s" +#define TN_A_ASP_ATTN "a.asp_attn.%s" +#define TN_A_ASP_TDNN "a.asp_tdnn.%s" + +// qwen3tts code_predictor +#define TN_A_GEN_CODE_PROJ_IN "a.gen.code.proj_in.%s" +#define TN_A_GEN_CODE_EMBD "a.gen.code.embd.%s" +#define TN_A_GEN_CODE_HEAD "a.gen.code.head.%s" +#define TN_A_GEN_CODE_OUT_EMBD "a.gen.code.out_embd.%s" +#define TN_A_GEN_CODE_NORM "a.gen.code.output_norm.%s" + +// qwen3tts code2wav (RVQ codes -> raw PCM) +// pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm" +#define TN_A_GEN_WAV_QUANT_FIRST_IN "a.gen.wav.quant.first.in_proj.%s" +#define TN_A_GEN_WAV_QUANT_FIRST_OUT "a.gen.wav.quant.first.out_proj.%s" +#define TN_A_GEN_WAV_QUANT_FIRST_CB "a.gen.wav.quant.first.codebook.%s" +#define TN_A_GEN_WAV_QUANT_REST_IN "a.gen.wav.quant.rest.in_proj.%s" +#define TN_A_GEN_WAV_QUANT_REST_OUT "a.gen.wav.quant.rest.out_proj.%s" +#define TN_A_GEN_WAV_QUANT_REST_CB "a.gen.wav.quant.rest.codebook.%s" +#define TN_A_GEN_WAV_PRE_CONV "a.gen.wav.pre_conv.%s" +#define TN_A_GEN_WAV_TFM_IN_PROJ "a.gen.wav.tfm.in_proj.%s" +#define TN_A_GEN_WAV_TFM_OUT_PROJ "a.gen.wav.tfm.out_proj.%s" +#define TN_A_GEN_WAV_TFM_OUT_NORM "a.gen.wav.tfm.output_norm.%s" +#define TN_A_GEN_WAV_UP_CONV "a.gen.wav.up.blk.%d.conv.%s" +#define TN_A_GEN_WAV_UP_DWCONV "a.gen.wav.up.blk.%d.dwconv.%s" +#define TN_A_GEN_WAV_UP_NORM "a.gen.wav.up.blk.%d.norm.%s" +#define TN_A_GEN_WAV_UP_PW1 "a.gen.wav.up.blk.%d.pw1.%s" +#define TN_A_GEN_WAV_UP_PW2 "a.gen.wav.up.blk.%d.pw2.%s" +#define TN_A_GEN_WAV_UP_GAMMA "a.gen.wav.up.blk.%d.gamma" +#define TN_A_GEN_WAV_DAC_ENTRY "a.gen.wav.dac.entry.%s" +#define TN_A_GEN_WAV_DAC_SNAKE "a.gen.wav.dac.blk.%d.snake.%s" +#define TN_A_GEN_WAV_DAC_CONV "a.gen.wav.dac.blk.%d.conv.%s" +#define TN_A_GEN_WAV_DAC_RES_ACT1 "a.gen.wav.dac.blk.%d.res.%d.act1.%s" +#define TN_A_GEN_WAV_DAC_RES_CONV1 "a.gen.wav.dac.blk.%d.res.%d.conv1.%s" +#define TN_A_GEN_WAV_DAC_RES_ACT2 "a.gen.wav.dac.blk.%d.res.%d.act2.%s" +#define TN_A_GEN_WAV_DAC_RES_CONV2 "a.gen.wav.dac.blk.%d.res.%d.conv2.%s" +#define TN_A_GEN_WAV_DAC_POST_SNAKE "a.gen.wav.dac.post_snake.%s" +#define TN_A_GEN_WAV_DAC_POST_CONV "a.gen.wav.dac.post_conv.%s" + // cogvlm #define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s" #define TN_MM_H_TO_4H "mm.up.%s" @@ -408,6 +453,8 @@ enum projector_type { PROJECTOR_TYPE_MINIMAX_M3, PROJECTOR_TYPE_GRANITE4_VISION, PROJECTOR_TYPE_MIMO_AUDIO, + PROJECTOR_TYPE_QWEN3TTS_SPKENC, + PROJECTOR_TYPE_QWEN3TTS_GEN, PROJECTOR_TYPE_UNKNOWN, }; @@ -465,6 +512,8 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, { PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"}, { PROJECTOR_TYPE_PARAKEET, "parakeet"}, + { PROJECTOR_TYPE_QWEN3TTS_SPKENC, "qwen3tts_spkenc"}, + { PROJECTOR_TYPE_QWEN3TTS_GEN, "qwen3tts_gen"}, }; static projector_type clip_projector_type_from_string(const std::string & str) { diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 8b9db5101d2c..101f49cd1849 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -136,6 +136,19 @@ struct clip_hparams { int32_t rvq_num_quantizers = 0; std::vector rvq_codebook_size; // per-quantizer bin count (ragged, e.g. 1024/1024/256/128x17) + // qwen3tts code2wav + int32_t wav_tfm_n_layer = 0; + int32_t wav_tfm_n_embd = 0; + int32_t wav_tfm_n_ff = 0; + int32_t wav_tfm_n_head = 0; + int32_t wav_tfm_n_head_kv = 0; + float wav_tfm_eps = 1e-5f; + float wav_tfm_rope_theta = 10000.0f; + int32_t wav_upsample_n_block = 0; + int32_t wav_dac_n_block = 0; + int32_t wav_dac_n_res = 0; + int32_t wav_tfm_swa = 0; // pre_transformer's KV cache size, in frames + // mimo-v2.5: LLM-side connector (input_local_transformer) int32_t audio_local_n_layer = 0; int32_t audio_local_group_size = 0; @@ -286,6 +299,14 @@ struct clip_layer { ggml_tensor * cross_attn_norm_w = nullptr; ggml_tensor * cross_attn_norm_b = nullptr; + // qwen3tts speaker encoder: SE-Res2Net block, tdnn1/tdnn2 reuse conv_pw1_w/b and conv_pw2_w/b above + ggml_tensor * se_conv1_w = nullptr; + ggml_tensor * se_conv1_b = nullptr; + ggml_tensor * se_conv2_w = nullptr; + ggml_tensor * se_conv2_b = nullptr; + std::vector res2_conv_w; // Res2Net hierarchical branches + std::vector res2_conv_b; + bool has_deepstack() const { return deepstack_fc1_w != nullptr; } @@ -365,6 +386,73 @@ struct qf_block { std::vector qf_proj_layers; }; +// qwen3tts code2wav: RVQ codes -> raw PCM +struct clip_code2wav { + // "upsample" stage: one ConvNeXt block plus the causal ConvTranspose1d before it + struct upsample_block { + ggml_tensor * conv_w = nullptr; // causal ConvTranspose1d, 2x + ggml_tensor * conv_b = nullptr; + ggml_tensor * dwconv_w = nullptr; // depthwise causal conv, k=7 + ggml_tensor * dwconv_b = nullptr; + ggml_tensor * norm_w = nullptr; // LayerNorm + ggml_tensor * norm_b = nullptr; + ggml_tensor * pw1_w = nullptr; // pointwise expand + ggml_tensor * pw1_b = nullptr; + ggml_tensor * pw2_w = nullptr; // pointwise project + ggml_tensor * pw2_b = nullptr; + ggml_tensor * gamma = nullptr; // layer scale + }; + + // one DAC residual unit: SnakeBeta -> dilated causal conv -> SnakeBeta -> pointwise causal conv + struct dac_res { + ggml_tensor * act1_alpha = nullptr; + ggml_tensor * act1_beta = nullptr; + ggml_tensor * conv1_w = nullptr; + ggml_tensor * conv1_b = nullptr; + ggml_tensor * act2_alpha = nullptr; + ggml_tensor * act2_beta = nullptr; + ggml_tensor * conv2_w = nullptr; + ggml_tensor * conv2_b = nullptr; + }; + + // one DAC upsample block (SnakeBeta -> causal ConvTranspose1d -> 3 residual units) + struct dac_block { + ggml_tensor * snake_alpha = nullptr; + ggml_tensor * snake_beta = nullptr; + ggml_tensor * conv_w = nullptr; // causal ConvTranspose1d + ggml_tensor * conv_b = nullptr; + std::vector res; + }; + + // quantizer: RVQ codebook decode + ggml_tensor * quant_first_in_w = nullptr; // semantic RVQ, in_proj (1x1 conv, loaded as 2D) + ggml_tensor * quant_first_out_w = nullptr; + ggml_tensor * quant_first_cb_w = nullptr; // codebook (1 layer) + ggml_tensor * quant_rest_in_w = nullptr; // acoustic RVQ + ggml_tensor * quant_rest_out_w = nullptr; + ggml_tensor * quant_rest_cb_w = nullptr; // codebooks, merged 3D [15, vocab, dim] + + ggml_tensor * pre_conv_w = nullptr; + ggml_tensor * pre_conv_b = nullptr; + + ggml_tensor * tfm_in_proj_w = nullptr; + ggml_tensor * tfm_in_proj_b = nullptr; + ggml_tensor * tfm_out_proj_w = nullptr; + ggml_tensor * tfm_out_proj_b = nullptr; + ggml_tensor * tfm_output_norm_w = nullptr; + std::vector tfm_layers; // reuses the generic block fields (ln_1/attn/ln_2/ffn/ls_1/ls_2) + + std::vector upsample; + + ggml_tensor * dac_entry_w = nullptr; + ggml_tensor * dac_entry_b = nullptr; + std::vector dac; + ggml_tensor * dac_post_snake_alpha = nullptr; + ggml_tensor * dac_post_snake_beta = nullptr; + ggml_tensor * dac_post_conv_w = nullptr; + ggml_tensor * dac_post_conv_b = nullptr; +}; + struct clip_model { clip_modality modality = CLIP_MODALITY_VISION; projector_type proj_type = PROJECTOR_TYPE_MLP; @@ -577,6 +665,24 @@ struct clip_model { ggml_tensor * conv2d_3_w = nullptr; ggml_tensor * conv2d_3_b = nullptr; + // qwen3tts speaker encoder (ECAPA-TDNN) + // reused tensors: stem conv is conv1d_1_w/b, feature aggregation is conv_out_w/b, output proj is mm_fc_w/b + ggml_tensor * spk_asp_attn_w = nullptr; + ggml_tensor * spk_asp_attn_b = nullptr; + ggml_tensor * spk_asp_tdnn_w = nullptr; + ggml_tensor * spk_asp_tdnn_b = nullptr; + + // qwen3tts code_predictor + ggml_tensor * gen_code_proj_in_w = nullptr; // small_to_mtp_projection + ggml_tensor * gen_code_proj_in_b = nullptr; + ggml_tensor * gen_code_embd_w = nullptr; // per-codebook embedding, merged 3D + ggml_tensor * gen_code_head_w = nullptr; // per-codebook output head, merged 3D + ggml_tensor * gen_code_out_embd_w = nullptr; // codebook-0 embedding, fed back into the talker + ggml_tensor * gen_code_norm_w = nullptr; // final norm + + // qwen3tts code2wav: RVQ codes -> raw PCM + clip_code2wav c2w; + // cogvlm ggml_tensor * mm_post_fc_norm_w = nullptr; ggml_tensor * mm_post_fc_norm_b = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index c1870813fb93..d6670030ff79 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -269,6 +270,29 @@ clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) : gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false); } +clip_graph::clip_graph(const clip_graph & parent) : + model(parent.model), + hparams(parent.hparams), + proj_type(parent.proj_type), + img(parent.img), + patch_size(parent.patch_size), + n_patches_x(parent.n_patches_x), + n_patches_y(parent.n_patches_y), + n_patches(parent.n_patches), + n_embd(parent.n_embd), + n_head(parent.n_head), + n_head_kv(parent.n_head_kv), + d_head(parent.d_head), + n_layer(parent.n_layer), + n_mmproj_embd(parent.n_mmproj_embd), + eps(parent.eps), + kq_scale(parent.kq_scale), + flash_attn_type(parent.flash_attn_type) { + // reuse from parent + ctx0 = parent.ctx0; + gf = parent.gf; +} + ggml_tensor * clip_graph::build_mm(ggml_tensor * w, ggml_tensor * x) const { return ggml_mul_mat(ctx0, w, x); } @@ -873,7 +897,8 @@ ggml_tensor * clip_graph::build_patch_merge_permute(ggml_tensor * cur, int scale return cur; } -static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const clip_image_f32_batch & imgs) { +static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const clip_image_f32_batch & imgs, + const clip_encode_params * params = nullptr) { const clip_image_f32 & img = imgs.entries[0]; std::unique_ptr builder; @@ -1025,6 +1050,17 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; + const int top_k = params ? params->top_k : 50; + const float top_p = params ? params->top_p : 1.0f; + builder = std::make_unique(ctx, img, gen_process, top_k, top_p); + } break; case PROJECTOR_TYPE_YOUTUVL: { builder = std::make_unique(ctx, img); @@ -1065,8 +1101,9 @@ struct clip_model_loader { size_t model_size = 0; // in bytes - bool has_vision = false; - bool has_audio = false; + bool has_vision = false; + bool has_audio = false; + bool has_gen_audio = false; mtmd_progress_callback progress_callback = nullptr; void * progress_callback_user_data = nullptr; @@ -1112,8 +1149,9 @@ struct clip_model_loader { // modalities { - get_bool(KEY_HAS_VISION_ENC, has_vision, false); - get_bool(KEY_HAS_AUDIO_ENC, has_audio, false); + get_bool(KEY_HAS_VISION_ENC, has_vision, false); + get_bool(KEY_HAS_AUDIO_ENC, has_audio, false); + get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false); if (has_vision) { LOG_INF("%s: has vision encoder\n", __func__); @@ -1121,6 +1159,9 @@ struct clip_model_loader { if (has_audio) { LOG_INF("%s: has audio encoder\n", __func__); } + if (has_gen_audio) { + LOG_INF("%s: has audio generation (gen) encoder\n", __func__); + } } // tensors @@ -1147,6 +1188,8 @@ struct clip_model_loader { GGML_ASSERT(has_vision); } else if (modality == CLIP_MODALITY_AUDIO) { GGML_ASSERT(has_audio); + } else if (modality == CLIP_MODALITY_GEN_AUDIO) { + GGML_ASSERT(has_gen_audio); } model.modality = modality; @@ -1163,6 +1206,8 @@ struct clip_model_loader { get_string(KEY_VISION_PROJ_TYPE, proj_type, false); } else if (modality == CLIP_MODALITY_AUDIO) { get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false); + } else if (modality == CLIP_MODALITY_GEN_AUDIO) { + get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false); } else { GGML_ABORT("unknown modality"); } @@ -1182,12 +1227,13 @@ struct clip_model_loader { } } - const bool is_vision = model.modality == CLIP_MODALITY_VISION; - const bool is_audio = model.modality == CLIP_MODALITY_AUDIO; + const bool is_vision = model.modality == CLIP_MODALITY_VISION; + const bool is_audio = model.modality == CLIP_MODALITY_AUDIO; + const bool is_gen_audio = model.modality == CLIP_MODALITY_GEN_AUDIO; // other hparams { - const char * prefix = is_vision ? "vision" : "audio"; + const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio"); get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd); get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head); get_u32(string_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false); @@ -1198,6 +1244,7 @@ struct clip_model_loader { // n_head_kv is optional (for GQA), default to n_head hparams.n_head_kv = hparams.n_head; + get_u32(string_format(KEY_N_HEAD_KV, prefix), hparams.n_head_kv, false); if (is_vision) { get_u32(KEY_IMAGE_SIZE, hparams.image_size); @@ -1226,6 +1273,11 @@ struct clip_model_loader { hparams.image_size = 0; hparams.patch_size = 1; + } else if (is_gen_audio) { + // these are unused, but still need to be set to avoid issues + hparams.image_size = 0; + hparams.patch_size = 1; + } else { GGML_ASSERT(false && "unknown modality"); } @@ -1647,6 +1699,33 @@ struct clip_model_loader { "%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE)); } } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2) + hparams.audio_sample_rate = 24000; + hparams.audio_n_fft = 1024; + hparams.audio_window_len = 1024; + hparams.audio_hop_len = 256; + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + // TODO: hardcoded for now, read from code_predictor_config instead + hparams.rope_theta = 1000000.0f; + + // code2wav params + hparams.wav_tfm_n_layer = 8; + hparams.wav_tfm_n_embd = 512; + hparams.wav_tfm_n_ff = 1024; + hparams.wav_tfm_n_head = 16; + hparams.wav_tfm_n_head_kv = 16; + hparams.wav_tfm_eps = 1e-5f; + hparams.wav_tfm_rope_theta = 10000.0f; + hparams.wav_upsample_n_block = 2; + hparams.wav_dac_n_block = 4; + hparams.wav_dac_n_res = 3; + // matches the reference decoder's sliding_window (speech_tokenizer/config.json) + hparams.wav_tfm_swa = 72; + } break; case PROJECTOR_TYPE_PADDLEOCR: { hparams.n_merge = 2; @@ -1871,7 +1950,9 @@ struct clip_model_loader { } // TODO @ngxson : support both audio and video in the future - const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a" : "v"; + const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a" + : model.modality == CLIP_MODALITY_GEN_AUDIO ? "a.gen.code" + : "v"; // get offsets for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { @@ -1973,7 +2054,8 @@ struct clip_model_loader { model.position_embeddings = get_tensor(string_format(TN_POS_EMBD, prefix), false); const bool has_standard_layers = ( - model.proj_type != PROJECTOR_TYPE_GEMMA3NV); + model.proj_type != PROJECTOR_TYPE_GEMMA3NV && + model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC); // layers const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0; @@ -2599,6 +2681,144 @@ struct clip_model_loader { model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // stem TDNN (block 0) + model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 0, "weight")); + model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 0, "bias")); + + // SE-Res2Net blocks (GGUF bid 1..3, one per hparams.n_layer) + model.layers.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; il++) { + auto & layer = model.layers[il]; + int bid = il + 1; + layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "weight")); + layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "bias")); + layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "weight")); + layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "bias")); + layer.se_conv1_w = get_tensor(string_format(TN_A_SE_CONV1, bid, "weight")); + layer.se_conv1_b = get_tensor(string_format(TN_A_SE_CONV1, bid, "bias")); + layer.se_conv2_w = get_tensor(string_format(TN_A_SE_CONV2, bid, "weight")); + layer.se_conv2_b = get_tensor(string_format(TN_A_SE_CONV2, bid, "bias")); + layer.res2_conv_w.resize(7); + layer.res2_conv_b.resize(7); + for (int xid = 0; xid < 7; xid++) { + layer.res2_conv_w[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "weight")); + layer.res2_conv_b[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "bias")); + } + } + + // multi-layer feature aggregation + model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); + model.conv_out_b = get_tensor(string_format(TN_CONV_OUT, "bias")); + + // attentive statistics pooling + model.spk_asp_attn_w = get_tensor(string_format(TN_A_ASP_ATTN, "weight")); + model.spk_asp_attn_b = get_tensor(string_format(TN_A_ASP_ATTN, "bias")); + model.spk_asp_tdnn_w = get_tensor(string_format(TN_A_ASP_TDNN, "weight")); + model.spk_asp_tdnn_b = get_tensor(string_format(TN_A_ASP_TDNN, "bias")); + + // final speaker embedding projection + model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight")); + model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias")); + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + // code_predictor + model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight")); + model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias")); + model.gen_code_embd_w = get_tensor(string_format(TN_A_GEN_CODE_EMBD, "weight")); + model.gen_code_head_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD, "weight")); + model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); + model.gen_code_norm_w = get_tensor(string_format(TN_A_GEN_CODE_NORM, "weight")); + + // code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor + { + auto & c2w = model.c2w; + + c2w.quant_first_in_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_IN, "weight")); + c2w.quant_first_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight")); + c2w.quant_first_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight")); + c2w.quant_rest_in_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_IN, "weight")); + c2w.quant_rest_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_OUT, "weight")); + c2w.quant_rest_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight")); + + c2w.pre_conv_w = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "weight")); + c2w.pre_conv_b = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "bias")); + + c2w.tfm_in_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight")); + c2w.tfm_in_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias")); + c2w.tfm_out_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight")); + c2w.tfm_out_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias")); + c2w.tfm_output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight")); + + // loaded manually, the generic model.layers loop is taken by code_predictor + c2w.tfm_layers.resize(hparams.wav_tfm_n_layer); + for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { + auto & layer = c2w.tfm_layers[il]; + const char * p = "a.gen.wav.tfm"; + layer.q_w = get_tensor(string_format(TN_ATTN_Q, p, il, "weight")); + layer.k_w = get_tensor(string_format(TN_ATTN_K, p, il, "weight")); + layer.v_w = get_tensor(string_format(TN_ATTN_V, p, il, "weight")); + layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, p, il, "weight")); + layer.ln_1_w = get_tensor(string_format(TN_LN_1, p, il, "weight")); + layer.ln_2_w = get_tensor(string_format(TN_LN_2, p, il, "weight")); + layer.ls_1_w = get_tensor(string_format(TN_LS_1, p, il, "weight")); + layer.ls_2_w = get_tensor(string_format(TN_LS_2, p, il, "weight")); + layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, p, il, "weight")); + layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, p, il, "weight")); + layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, p, il, "weight")); + } + + // upsample: 2x (causal ConvTranspose1d + ConvNeXt block) + c2w.upsample.resize(hparams.wav_upsample_n_block); + for (int il = 0; il < hparams.wav_upsample_n_block; il++) { + auto & up = c2w.upsample[il]; + up.conv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV, il, "weight")); + up.conv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV, il, "bias")); + up.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight")); + up.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias")); + up.norm_w = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight")); + up.norm_b = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias")); + up.pw1_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight")); + up.pw1_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias")); + up.pw2_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight")); + up.pw2_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias")); + up.gamma = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA, il)); + } + + // DAC decoder: conv_pre + n upsample blocks (each with n_res residual units) + conv_post + c2w.dac_entry_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "weight")); + c2w.dac_entry_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "bias")); + + c2w.dac.resize(hparams.wav_dac_n_block); + for (int il = 0; il < hparams.wav_dac_n_block; il++) { + auto & blk = c2w.dac[il]; + blk.snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "alpha")); + blk.snake_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "beta")); + blk.conv_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV, il, "weight")); + blk.conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV, il, "bias")); + + blk.res.resize(hparams.wav_dac_n_res); + for (int ir = 0; ir < hparams.wav_dac_n_res; ir++) { + auto & res = blk.res[ir]; + res.act1_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "alpha")); + res.act1_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "beta")); + res.conv1_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "weight")); + res.conv1_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "bias")); + res.act2_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "alpha")); + res.act2_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "beta")); + res.conv2_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "weight")); + res.conv2_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "bias")); + } + } + + c2w.dac_post_snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "alpha")); + c2w.dac_post_snake_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "beta")); + c2w.dac_post_conv_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "weight")); + c2w.dac_post_conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias")); + } + } break; case PROJECTOR_TYPE_VOXTRAL: { model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); @@ -3427,6 +3647,7 @@ struct clip_model_loader { struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params) { clip_ctx * ctx_vision = nullptr; clip_ctx * ctx_audio = nullptr; + clip_ctx * ctx_gen_audio = nullptr; try { clip_model_loader loader(fname, @@ -3459,16 +3680,25 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params } } + if (loader.has_gen_audio) { + ctx_gen_audio = new clip_ctx(ctx_params); + loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO); + loader.load_tensors(*ctx_gen_audio); + // TODO: fix warmup + ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() + ggml_graph_overhead()); + } + } catch (const std::exception & e) { LOG_ERR("%s: failed to load model '%s': %s\n", __func__, fname, e.what()); delete ctx_vision; delete ctx_audio; + delete ctx_gen_audio; - return {nullptr, nullptr}; + return {nullptr, nullptr, nullptr}; } - return {ctx_vision, ctx_audio}; + return {ctx_vision, ctx_audio, ctx_gen_audio}; } struct clip_cap clip_get_cap(const char * fname) { @@ -3784,6 +4014,16 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { const int ds = ctx->model.hparams.audio_proj_downsample_rate; n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds); } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // pooling gives one speaker embedding, whatever the clip length is + n_patches = 1; + } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + // one hidden-state vector fed back to the talker per call + n_patches = 1; + } break; case PROJECTOR_TYPE_GRANITE4_VISION: { // Per-tile output token count: each projector block outputs @@ -3817,7 +4057,16 @@ bool clip_image_encode(struct clip_ctx * ctx, int n_threads, const clip_image_f3 } bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32_batch * imgs_c_ptr, std::vector & out_batch_embd) { - const clip_image_f32_batch & imgs = *imgs_c_ptr; + clip_encode_params params; + params.imgs = imgs_c_ptr; + params.n_threads = n_threads; + params.out_embd = &out_batch_embd; + + return clip_encode(ctx, ¶ms); +} + +bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { + const clip_image_f32_batch & imgs = *params->imgs; int n_batch_cur = imgs.entries.size(); // [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames @@ -3828,12 +4077,12 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 // if buffers are not allocated, we need to do a warmup run to allocate them if (!ctx->is_allocated) { - clip_model_loader::warmup(*ctx, *imgs_c_ptr); + clip_model_loader::warmup(*ctx, *params->imgs); } // build the inference graph ggml_backend_sched_reset(ctx->sched.get()); - ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs)->build(); + ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs, params)->build(); ggml_backend_sched_alloc_graph(ctx->sched.get(), gf); // set inputs @@ -3918,8 +4167,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 } set_input_f32("inp_raw", inp_raw); - } else { - // audio input + } else if (!(ctx->proj_type() == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == CLIP_GEN_PROCESS_GEN_WAV)) { + // audio input, code2wav is not here: its only input is "inp_codes", set in the switch below GGML_ASSERT(imgs.entries.size() == 1); const auto & mel_inp = imgs.entries[0]; @@ -4475,9 +4724,77 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 case PROJECTOR_TYPE_COGVLM: case PROJECTOR_TYPE_YASA2: case PROJECTOR_TYPE_GEMMA4UA: + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { // do nothing } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { + GGML_ASSERT(params->codes != nullptr); + + // frame-major input to group-major, rear-padded with code 0 up to one window + const int64_t n_codes = model.gen_code_head_w->ne[2] + 1; + const int64_t n_frames_w = hparams.wav_tfm_swa; + const int64_t n_frames = (int64_t) params->codes->size() / n_codes; + GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w); + + // codes are used as ggml_get_rows indices, so check them against the codebook vocab + const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1]; + const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1]; + for (int64_t f = 0; f < n_frames; f++) { + for (int64_t g = 0; g < n_codes; g++) { + const int32_t c = (*params->codes)[f * n_codes + g]; + const int64_t vocab = (g == 0) ? vocab_first : vocab_rest; + if (c < 0 || (int64_t) c >= vocab) { + LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n", + __func__, (long long) f, (long long) g, c, (long long) vocab); + return false; + } + } + } + + std::vector codes(n_frames_w * n_codes, 0); + for (int64_t f = 0; f < n_frames; f++) { + for (int64_t g = 0; g < n_codes; g++) { + codes[g * n_frames_w + f] = (*params->codes)[f * n_codes + g]; + } + } + set_input_i32("inp_codes", codes); + + // upload the state from the previous call, or zero-fill on a cold start + size_t offset = 0; + for (const auto & slot : list_c2w_state_slots(hparams, model)) { + ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str()); + const size_t nb = ggml_nbytes(t); + if (params->state_in && params->state_in->size() >= offset + nb) { + ggml_backend_tensor_set(t, params->state_in->data() + offset, 0, nb); + } else { + std::vector zeros(nb, 0); + ggml_backend_tensor_set(t, zeros.data(), 0, nb); + } + offset += nb; + } + } else { + // code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it + const int64_t vocab0 = model.gen_code_out_embd_w->ne[1]; + if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) { + LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0); + return false; + } + std::vector code0 = { params->code0 }; + set_input_i32("inp_code0", code0); + + // one uniform(0,1) draw per codebook, used by do_sampling() + static std::mt19937 rng{ std::random_device{}() }; + std::uniform_real_distribution dist(0.0f, 1.0f); + const int64_t n_acoustic = model.gen_code_head_w->ne[2]; + for (int64_t g = 0; g < n_acoustic; g++) { + std::vector r = { dist(rng) }; + set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r); + } + } + } break; case PROJECTOR_TYPE_HUNYUANVL: { // Compute the HunyuanVL 2D position embedding on CPU (with the @@ -4883,7 +5200,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 if (reg) { auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); if (ggml_backend_set_n_threads_fn) { - ggml_backend_set_n_threads_fn(ctx->backend_cpu, n_threads); + ggml_backend_set_n_threads_fn(ctx->backend_cpu, params->n_threads); } } @@ -4893,34 +5210,90 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 return false; } - // the last node is the embedding tensor - ggml_tensor * embeddings = ggml_graph_node(gf, -1); + // the last node is the embedding tensor, code2wav has no out_embd + ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr; + + if (embeddings != nullptr) { + // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one) + const int n_tokens_out = embeddings->ne[1]; + const int expected_n_tokens_out = clip_n_output_tokens(ctx, &imgs.entries[0]); + if (n_tokens_out != expected_n_tokens_out) { + LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out); + GGML_ABORT("Invalid number of output tokens"); + } + + LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__, + (int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]); - // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one) - const int n_tokens_out = embeddings->ne[1]; - const int expected_n_tokens_out = clip_n_output_tokens(ctx, &imgs.entries[0]); - if (n_tokens_out != expected_n_tokens_out) { - LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out); - GGML_ABORT("Invalid number of output tokens"); + // copy output to user buffer if provided + // if output is empty, skip the copy + auto & out_batch_embd = *params->out_embd; + if (!out_batch_embd.empty()) { + if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) { + LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings)); + GGML_ABORT("Output buffer size mismatch"); + } + ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings)); + } else { + LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__); + } } - LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__, - (int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]); + // + // for audio gen models + // - // copy output to user buffer if provided - // if output is empty, skip the copy - if (!out_batch_embd.empty()) { - if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) { - LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings)); - GGML_ABORT("Output buffer size mismatch"); + if (params->out_codes != nullptr) { + ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes"); + if (codes == nullptr) { + GGML_ABORT("out_codes requested but graph has no \"out_codes\" tensor"); + } + auto & out_codes = *params->out_codes; + out_codes.resize(ggml_nelements(codes)); + ggml_backend_tensor_get(codes, out_codes.data(), 0, ggml_nbytes(codes)); + } + if (params->out_audio != nullptr) { + ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio"); + if (audio == nullptr) { + GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor"); + } + auto & out_audio = *params->out_audio; + out_audio.resize(ggml_nelements(audio)); + ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio)); + + // drop the tail audio that comes from the code-0 rear padding + const int64_t n_codes = model.gen_code_head_w->ne[2] + 1; + const int64_t n_frames_w = hparams.wav_tfm_swa; + const int64_t n_frames = (int64_t) params->codes->size() / n_codes; + if (n_frames < n_frames_w) { + const size_t hop = out_audio.size() / n_frames_w; + out_audio.resize((size_t) n_frames * hop); + } + } + if (params->state_out != nullptr) { + auto & state_out = *params->state_out; + size_t total = 0; + for (const auto & slot : list_c2w_state_slots(hparams, model)) { + total += (size_t) (slot.ne0 * slot.ne1) * sizeof(float); + } + state_out.resize(total); + size_t offset = 0; + for (const auto & slot : list_c2w_state_slots(hparams, model)) { + ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); + if (t == nullptr) { + GGML_ABORT("state_out requested but graph has no \"state_out_%s\" tensor", slot.name.c_str()); + } + const size_t nb = ggml_nbytes(t); + ggml_backend_tensor_get(t, state_out.data() + offset, 0, nb); + offset += nb; } - ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings)); - } else { - LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__); } + // // Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set - if (ctx->debug_output_embeddings) { + // + + if (ctx->debug_output_embeddings && embeddings != nullptr) { const int64_t n_embd = embeddings->ne[0]; const int64_t n_tokens = embeddings->ne[1]; std::vector emb_data(ggml_nelements(embeddings)); @@ -5047,6 +5420,10 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.mm_ffn_down_w->ne[1]; case PROJECTOR_TYPE_MIMO_AUDIO: return ctx->model.mm_2_w->ne[1]; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + return ctx->model.mm_fc_w->ne[2]; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + return ctx->model.gen_code_out_embd_w->ne[0]; case PROJECTOR_TYPE_PARAKEET: return ctx->model.mm_1_w->ne[1]; default: diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 967093a812d6..7f706d976eb3 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -37,6 +37,7 @@ struct clip_image_f32_batch; enum clip_modality { CLIP_MODALITY_VISION, CLIP_MODALITY_AUDIO, + CLIP_MODALITY_GEN_AUDIO, }; enum clip_flash_attn_type { @@ -61,6 +62,7 @@ struct clip_context_params { struct clip_init_result { struct clip_ctx * ctx_v; // vision context struct clip_ctx * ctx_a; // audio context + struct clip_ctx * ctx_gen_a; // audio generation context }; struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params); @@ -84,6 +86,33 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx); bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector & out_vec); bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector & out_batch_embd); +enum clip_gen_process_type { + CLIP_GEN_PROCESS_GEN_UNKNOWN, + CLIP_GEN_PROCESS_GEN_CODE, // h_state to codes + CLIP_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio +}; +struct clip_encode_params { + int n_threads = 1; + const clip_image_f32_batch * imgs = nullptr; + std::vector * out_embd = nullptr; + + // for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV) + clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_UNKNOWN; + + // GEN_CODE: out_embd receives the embd to feed back to the backbone + int32_t code0 = 0; // semantic code sampled by the backbone + int32_t top_k = 50; + float top_p = 1.0f; + std::vector * out_codes = nullptr; // this frame's 16 sampled codes + + // GEN_WAV + const std::vector * codes = nullptr; // this frame's 16 RVQ codes + std::vector * out_audio = nullptr; // decoded PCM samples, F32 + const std::vector * state_in = nullptr; // state from previous call, null or wrong size means cold start + std::vector * state_out = nullptr; // state for the next call +}; +bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params); + bool clip_is_llava(const struct clip_ctx * ctx); // note for contributor: this clip_is_(model) pattern is deprecated // do NOT add new functions like this diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index e54366a086f4..eb924972bf5c 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -2,6 +2,11 @@ #include "../clip-graph.h" +#include +#include +#include +#include + /* * IMPORTANT: The mtmd module does NOT accept pull requests that are fully or predominantly AI-generated. * We encourage human contributors to ensure the quality and reliability of the codebase. @@ -215,6 +220,111 @@ struct clip_graph_mimo_audio : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_qwen3tts_spkenc : clip_graph { + clip_graph_qwen3tts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; + + ggml_tensor * conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const; + ggml_tensor * res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const; + ggml_tensor * se_block(ggml_tensor * x, const clip_layer & layer) const; + ggml_tensor * se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const; + ggml_tensor * attentive_stats_pool(ggml_tensor * x) const; +}; + +struct clip_graph_qwen3tts_gen : clip_graph { + clip_graph_qwen3tts_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int top_k, float top_p) + : clip_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {} + ggml_cgraph * build() override; + + // which sub-graph build() constructs, fixed at graph-build time + clip_gen_process_type gen_process; + + // sampling params, fixed at graph-build time (GEN_CODE only) + int top_k; + float top_p; + + // + // code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes + // MTP-style code predictor, one token per codebook + // + struct code_gen : clip_graph { + code_gen(const clip_graph & parent, int top_k, float top_p) + : clip_graph(parent), top_k(top_k), top_p(top_p) {} + ggml_cgraph * build() override { GGML_ABORT("call prefill()/step() instead"); } + + int top_k; + float top_p; + + ggml_tensor * cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const; + ggml_tensor * do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const; + + ggml_tensor * const_i32(ggml_tensor * anchor, float value) const; + ggml_tensor * causal_mask_row(int64_t n_kv_pad, int pos) const; + ggml_tensor * project_in(ggml_tensor * cur) const; + + ggml_tensor * layer_forward( + ggml_tensor * cur, + const clip_layer & layer, + ggml_tensor * inp_pos, + ggml_tensor * kq_mask, + ggml_tensor *& k_cache_layer, + ggml_tensor *& v_cache_layer, + int64_t n_kv_pad, + int pos, + int il) const; + + void prefill( + std::vector & k_cache, + std::vector & v_cache, + ggml_tensor *& out_code_cache, + ggml_tensor * h_state, + ggml_tensor * code0_embd, + ggml_tensor * inp_rand) const; + + ggml_tensor * step( + std::vector & k_cache, + std::vector & v_cache, + ggml_tensor * out_code_cache, + ggml_tensor * inp_rand, + int step_idx) const; + }; + + // + // code2wav: RVQ codes -> raw PCM (quantizer + pre_conv + pre_transformer + upsample + DAC). + // + struct code2wav : clip_graph { + code2wav(const clip_graph & parent) : clip_graph(parent) {} + ggml_cgraph * build() override { GGML_ABORT("call decode() instead"); } + + // state_in: previous call's persisted state, by slot name (see list_c2w_state_slots()) + std::map state_in; + // state_out: this call's state to persist, added to the graph outputs by build() + mutable std::vector> state_out; + + // stateful conv ops: read/update their state via state_in/state_out[state_name] + ggml_tensor * causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const; + ggml_tensor * causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const; + ggml_tensor * causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const; + ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const; + + ggml_tensor * quant_decode(ggml_tensor * inp_codes) const; + ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const; + ggml_tensor * convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const; + ggml_tensor * dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const; + + // inp_codes [1, n_codes] I32 -> this frame's audio samples [n_samples] F32, clamped to [-1, 1] + ggml_tensor * decode(ggml_tensor * inp_codes) const; + }; +}; + +// one persisted state buffer used by code2wav, see qwen3tts-gen.cpp +struct c2w_state_slot { + std::string name; + int64_t ne0; + int64_t ne1; +}; +std::vector list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model); + struct clip_graph_kimik25 : clip_graph { clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp new file mode 100644 index 000000000000..b6c95efa941e --- /dev/null +++ b/tools/mtmd/models/qwen3tts-gen.cpp @@ -0,0 +1,766 @@ +#include "models.h" + +#include + +// on-device sampling: top-k, top-p, then a random draw +ggml_tensor * clip_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const { + logits = ggml_reshape_1d(ctx0, logits, ggml_nelements(logits)); + const int64_t n_vocab = logits->ne[0]; + + // sort a's rows by idx + auto sort_by = [this](ggml_tensor * a, ggml_tensor * idx) { + ggml_tensor * a2d = ggml_reshape_2d(ctx0, a, 1, a->ne[0]); + return ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, a2d, idx), idx->ne[0]); + }; + + ggml_tensor * cur = logits; + ggml_tensor * candidates = nullptr; // maps row index back to vocab id + + if (top_k > 0 && top_k < n_vocab) { + ggml_tensor * idx = ggml_top_k(ctx0, cur, top_k); + candidates = idx; + cur = sort_by(cur, idx); + cb(cur, "sample_top_k_logits", -1); + } + + if (top_p < 1.0f) { + ggml_tensor * sorted_idx = ggml_argsort(ctx0, cur, GGML_SORT_ORDER_DESC); + ggml_tensor * sorted_logits = sort_by(cur, sorted_idx); + candidates = candidates ? sort_by(candidates, sorted_idx) : sorted_idx; + + ggml_tensor * probs = ggml_soft_max(ctx0, sorted_logits); + ggml_tensor * cdf = ggml_cumsum(ctx0, probs); + + // keep_mask[i] = 1 once cdf[i] crosses top_p + ggml_tensor * cdf_scaled = ggml_scale_bias(ctx0, cdf, -1.0f, top_p); + ggml_tensor * keep_mask = ggml_step(ctx0, cdf_scaled); + ggml_tensor * idxf = ggml_sum(ctx0, keep_mask); + idxf = ggml_clamp(ctx0, idxf, 0.0f, (float) keep_mask->ne[0] - 1); + ggml_tensor * ones = ggml_scale_bias(ctx0, idxf, 0.0f, 1.0f); + + // top-p must include the crossing element, so force it to 1 + ggml_tensor * keep_mask_2d = ggml_reshape_2d(ctx0, keep_mask, 1, keep_mask->ne[0]); + keep_mask_2d = ggml_set_rows(ctx0, keep_mask_2d, ones, ggml_cast(ctx0, idxf, GGML_TYPE_I32)); + keep_mask = ggml_reshape_1d(ctx0, keep_mask_2d, keep_mask->ne[0]); + + // log(1) = 0 (keep), log(0) = -inf (drop) + ggml_tensor * bias = ggml_log(ctx0, keep_mask); + cur = ggml_add(ctx0, sorted_logits, bias); + cb(cur, "sample_top_p_logits", -1); + } + + // draw one token: find where the cdf crosses inp_rand + ggml_tensor * probs = ggml_soft_max(ctx0, cur); + ggml_tensor * cumsum = ggml_cumsum(ctx0, probs); + + ggml_tensor * diff = ggml_sub(ctx0, cumsum, inp_rand); + ggml_tensor * cross_mask = ggml_step(ctx0, diff); + ggml_tensor * idxf = ggml_sum(ctx0, cross_mask); + ggml_tensor * idx = ggml_cast(ctx0, ggml_scale_bias(ctx0, idxf, -1.0f, (float) cross_mask->ne[0]), GGML_TYPE_I32); + + if (candidates) { + ggml_tensor * cand_2d = ggml_reshape_2d(ctx0, candidates, 1, candidates->ne[0]); + idx = ggml_get_rows(ctx0, cand_2d, idx); + } + cb(idx, "sample_token_id", -1); + + return idx; +} + +// returns a new cache with row row_idx set to value +ggml_tensor * clip_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const { + const int64_t n_embd = cache->ne[0]; + const int64_t n_cache = cache->ne[1]; + GGML_ASSERT(row_idx >= 0 && row_idx < n_cache); + + // append value as the last row, then gather it back into place + ggml_tensor * value_2d = ggml_reshape_2d(ctx0, value, n_embd, 1); + ggml_tensor * cache_ext = ggml_concat(ctx0, cache, value_2d, 1); // [n_embd, n_cache + 1] + + // gather indices [0..row_idx-1, n_cache, row_idx+1..n_cache-1] + // built via concat, since ggml_set_rows needs F32/F16 values, not an I32 index array + ggml_tensor * idx = const_i32(cache, (float) n_cache); + if (row_idx > 0) { + ggml_tensor * prefix = ggml_cast(ctx0, ggml_arange(ctx0, 0.0f, (float) row_idx, 1.0f), GGML_TYPE_I32); + idx = ggml_concat(ctx0, prefix, idx, 0); + } + if (row_idx < n_cache - 1) { + ggml_tensor * suffix = ggml_cast(ctx0, ggml_arange(ctx0, (float) (row_idx + 1), (float) n_cache, 1.0f), GGML_TYPE_I32); + idx = ggml_concat(ctx0, idx, suffix, 0); + } + + ggml_tensor * result = ggml_get_rows(ctx0, cache_ext, idx); + cb(result, "cache_set_out", -1); + return result; +} + +// builds a const i32 with no host upload: view a tensor, zero it via scale, add value, cast to i32 +ggml_tensor * clip_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor, float value) const { + ggml_tensor * v = ggml_view_1d(ctx0, anchor, 1, 0); + if (v->type != GGML_TYPE_F32) { + v = ggml_cast(ctx0, v, GGML_TYPE_F32); + } + return ggml_cast(ctx0, ggml_scale_bias(ctx0, v, 0.0f, value), GGML_TYPE_I32); +} + +// causal keep-mask row for a query at position pos, window size n_kv_pad +ggml_tensor * clip_graph_qwen3tts_gen::code_gen::causal_mask_row(int64_t n_kv_pad, int pos) const { + ggml_tensor * ones = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv_pad, n_kv_pad), 1.0f); + ggml_tensor * keep = ggml_tri(ctx0, ones, GGML_TRI_TYPE_LOWER_DIAG); + ggml_tensor * row = ggml_view_1d(ctx0, keep, n_kv_pad, (size_t) pos * keep->nb[1]); + ggml_tensor * mask = ggml_log(ctx0, row); // 0 = keep, -inf = masked + return ggml_reshape_4d(ctx0, mask, n_kv_pad, 1, 1, 1); +} + +// talker hidden size -> predictor hidden size (small_to_mtp_projection) +ggml_tensor * clip_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) const { + if (!model.gen_code_proj_in_w) { + return cur; + } + cur = ggml_mul_mat(ctx0, model.gen_code_proj_in_w, cur); + if (model.gen_code_proj_in_b) { + cur = ggml_add(ctx0, cur, model.gen_code_proj_in_b); + } + return cur; +} + +// one transformer layer at position pos; writes k/v into k_cache_layer/v_cache_layer at row pos +ggml_tensor * clip_graph_qwen3tts_gen::code_gen::layer_forward( + ggml_tensor * cur, + const clip_layer & layer, + ggml_tensor * inp_pos, + ggml_tensor * kq_mask, + ggml_tensor *& k_cache_layer, + ggml_tensor *& v_cache_layer, + int64_t n_kv_pad, + int pos, + int il) const { + const int n_head = hparams.n_head; + const int n_head_kv = hparams.n_head_kv; + const int64_t d_head = layer.q_w->ne[1] / n_head; // real head_dim, not n_embd / n_head + const float kq_scale = 1.0f / sqrtf((float) d_head); + + ggml_tensor * residual = cur; + + ggml_tensor * h = ggml_rms_norm(ctx0, cur, hparams.eps); + h = ggml_mul(ctx0, h, layer.ln_1_w); + + ggml_tensor * q = ggml_mul_mat(ctx0, layer.q_w, h); + ggml_tensor * k = ggml_mul_mat(ctx0, layer.k_w, h); + ggml_tensor * v = ggml_mul_mat(ctx0, layer.v_w, h); + + q = ggml_reshape_3d(ctx0, q, d_head, n_head, 1); + k = ggml_reshape_3d(ctx0, k, d_head, n_head_kv, 1); + + q = ggml_rms_norm(ctx0, q, hparams.eps); + q = ggml_mul(ctx0, q, layer.q_norm); + k = ggml_rms_norm(ctx0, k, hparams.eps); + k = ggml_mul(ctx0, k, layer.k_norm); + + q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0, + hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0, + hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + + // write k/v into the cache at row pos, flat layout + ggml_tensor * k_flat = ggml_reshape_1d(ctx0, k, d_head * n_head_kv); + k_cache_layer = cache_set(k_cache_layer, pos, k_flat); + v_cache_layer = cache_set(v_cache_layer, pos, v); + + ggml_tensor * q_cur = ggml_reshape_4d(ctx0, q, d_head, n_head, 1, 1); + ggml_tensor * k_cur = ggml_reshape_4d(ctx0, k_cache_layer, d_head, n_head_kv, n_kv_pad, 1); + ggml_tensor * v_cur = ggml_reshape_4d(ctx0, v_cache_layer, d_head, n_head_kv, n_kv_pad, 1); + + ggml_tensor * attn_out = build_attn(layer.o_w, layer.o_b, q_cur, k_cur, v_cur, kq_mask, kq_scale, il); + + cur = ggml_add(ctx0, residual, attn_out); + + ggml_tensor * h2 = ggml_rms_norm(ctx0, cur, hparams.eps); + h2 = ggml_mul(ctx0, h2, layer.ln_2_w); + + ggml_tensor * gate = ggml_mul_mat(ctx0, layer.ff_gate_w, h2); + ggml_tensor * up = ggml_mul_mat(ctx0, layer.ff_up_w, h2); + ggml_tensor * gu = ggml_swiglu_split(ctx0, gate, up); + ggml_tensor * down = ggml_mul_mat(ctx0, layer.ff_down_w, gu); + + return ggml_add(ctx0, cur, down); +} + +// position 0: hidden bridge, seeds the k/v cache, no sampling +// position 1: embed(code0), sample with lm_head[0], write out_code_cache[1] +void clip_graph_qwen3tts_gen::code_gen::prefill( + std::vector & k_cache, + std::vector & v_cache, + ggml_tensor *& out_code_cache, + ggml_tensor * h_state, + ggml_tensor * code0_embd, + ggml_tensor * inp_rand) const { + const int64_t n_kv_pad = k_cache[0]->ne[1]; + + { + ggml_tensor * cur = project_in(h_state); + ggml_tensor * kq_mask = causal_mask_row(n_kv_pad, 0); + ggml_tensor * inp_pos = const_i32(k_cache[0], 0.0f); + for (size_t il = 0; il < model.layers.size(); il++) { + cur = layer_forward(cur, model.layers[il], inp_pos, kq_mask, k_cache[il], v_cache[il], n_kv_pad, 0, (int) il); + } + // position 0's output is unused, it only seeded the cache + } + + { + ggml_tensor * cur = project_in(code0_embd); + ggml_tensor * kq_mask = causal_mask_row(n_kv_pad, 1); + ggml_tensor * inp_pos = const_i32(k_cache[0], 1.0f); + for (size_t il = 0; il < model.layers.size(); il++) { + cur = layer_forward(cur, model.layers[il], inp_pos, kq_mask, k_cache[il], v_cache[il], n_kv_pad, 1, (int) il); + } + + cur = ggml_rms_norm(ctx0, cur, hparams.eps); + cur = ggml_mul(ctx0, cur, model.gen_code_norm_w); + + ggml_tensor * head_w = model.gen_code_head_w; + ggml_tensor * head_g = ggml_view_2d(ctx0, head_w, head_w->ne[0], head_w->ne[1], head_w->nb[1], 0); // lm_head[0] + ggml_tensor * logits = ggml_mul_mat(ctx0, head_g, cur); + + ggml_tensor * sampled = do_sampling(logits, inp_rand); + out_code_cache = cache_set(out_code_cache, 1, sampled); + } +} + +// one decode step of code_predictor +// at step_idx g: +// - read code from out_code_cache[g], then embed it with codebook table g-1 +// - write new kv at cache row g+1, sample with lm_head[g] +// - write result to out_code_cache[g+1] +// step_idx must be in [1, n_acoustic - 1] +ggml_tensor * clip_graph_qwen3tts_gen::code_gen::step( + std::vector & k_cache, + std::vector & v_cache, + ggml_tensor * out_code_cache, + ggml_tensor * inp_rand, + int step_idx) const { + const int64_t n_acoustic = model.gen_code_head_w->ne[2]; + GGML_ASSERT(step_idx >= 1 && step_idx < n_acoustic); + GGML_ASSERT(k_cache.size() == model.layers.size()); + GGML_ASSERT(v_cache.size() == model.layers.size()); + + const int64_t n_kv_pad = k_cache[0]->ne[1]; + const int pos = step_idx + 1; // new cache row and RoPE position + + // embed the previous code via this step's codebook table (rows are already scalars) + ggml_tensor * code_in = ggml_view_1d(ctx0, out_code_cache, 1, (size_t) step_idx * out_code_cache->nb[1]); + + ggml_tensor * embd_w = model.gen_code_embd_w; // [n_embd_talker, vocab, n_acoustic] + ggml_tensor * embd_g = ggml_view_2d(ctx0, embd_w, embd_w->ne[0], embd_w->ne[1], embd_w->nb[1], + (size_t) (step_idx - 1) * embd_w->nb[2]); + ggml_tensor * cur = ggml_get_rows(ctx0, embd_g, code_in); + cur = ggml_reshape_1d(ctx0, cur, cur->ne[0]); + cb(cur, "step_embd_in", step_idx); + + cur = project_in(cur); + cb(cur, "step_proj_in", step_idx); + + ggml_tensor * kq_mask = causal_mask_row(n_kv_pad, pos); + ggml_tensor * inp_pos = const_i32(k_cache[0], (float) pos); + + for (size_t il = 0; il < model.layers.size(); il++) { + cur = layer_forward(cur, model.layers[il], inp_pos, kq_mask, k_cache[il], v_cache[il], n_kv_pad, pos, (int) il); + cb(cur, "step_layer_out", (int) il); + } + + // final norm, this step's lm_head, sample, write the result + cur = ggml_rms_norm(ctx0, cur, hparams.eps); + cur = ggml_mul(ctx0, cur, model.gen_code_norm_w); + + ggml_tensor * head_w = model.gen_code_head_w; // [n_embd_pred, vocab, n_acoustic] + ggml_tensor * head_g = ggml_view_2d(ctx0, head_w, head_w->ne[0], head_w->ne[1], head_w->nb[1], + (size_t) step_idx * head_w->nb[2]); + ggml_tensor * logits = ggml_mul_mat(ctx0, head_g, cur); + cb(logits, "step_logits", step_idx); + + ggml_tensor * sampled = do_sampling(logits, inp_rand); + cb(sampled, "step_sampled", step_idx); + + return cache_set(out_code_cache, pos, sampled); +} + +// causal conv1d, stride 1: prepend persisted left-context instead of zero-padding, then a plain conv +// x: [T, IC] (T-first). w: [K, IC, OC]. state_name empty means K == 1 (no left-context). returns [T, OC] +ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const { + const int K = (int) w->ne[0]; + const int pad = (K - 1) * dilation; + + ggml_tensor * x_full = x; + if (pad > 0) { + ggml_tensor * left = state_in.at(state_name); // [pad, IC] + x_full = ggml_concat(ctx0, left, x, 0); + } + ggml_tensor * y = ggml_conv_1d(ctx0, w, x_full, 1, 0, dilation); // [T, OC, 1] + y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]); + if (b) { + y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0])); + } + if (pad > 0) { + ggml_tensor * new_left = ggml_cont(ctx0, ggml_view_2d(ctx0, x_full, pad, x_full->ne[1], x_full->nb[1], + (size_t) (x_full->ne[0] - pad) * x_full->nb[0])); + state_out.push_back({state_name, new_left}); + } + return y; +} + +// causal depthwise conv1d, stride 1, dilation 1, kernel from w's shape. +// x: [T, C]. w: [K, 1, C]. returns [T, C]. see causal_conv1d for the state contract. +ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const { + const int K = (int) w->ne[0]; + const int pad = K - 1; + + ggml_tensor * x_full = x; + if (pad > 0) { + ggml_tensor * left = state_in.at(state_name); // [pad, C] + x_full = ggml_concat(ctx0, left, x, 0); + } + ggml_tensor * y = ggml_conv_1d_dw(ctx0, w, x_full, 1, 0, 1); // [T, C, 1] + y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]); + if (b) { + y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0])); + } + if (pad > 0) { + ggml_tensor * new_left = ggml_cont(ctx0, ggml_view_2d(ctx0, x_full, pad, x_full->ne[1], x_full->nb[1], + (size_t) (x_full->ne[0] - pad) * x_full->nb[0])); + state_out.push_back({state_name, new_left}); + } + return y; +} + +// causal ConvTranspose1d, the (kernel - stride) overlap tail is kept as state for the next call +// x: [T, IC], w: [K, OC, IC]. state_name empty means K == stride (no overlap). returns [T * stride, OC] +ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const { + const int K = (int) w->ne[0]; + const int OC = (int) w->ne[1]; + const int trim = K - stride; + const int64_t emit_len = x->ne[0] * stride; + + // transposed conv as GEMM + col2im scatter-add, y: [emit_len + trim, OC] + ggml_tensor * w2 = ggml_reshape_2d(ctx0, w, (int64_t) K * OC, w->ne[2]); + w2 = ggml_cont(ctx0, ggml_transpose(ctx0, w2)); + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); + ggml_tensor * col = ggml_mul_mat(ctx0, w2, xt); + ggml_tensor * y = ggml_col2im_1d(ctx0, col, stride, OC, 0); + + ggml_tensor * out = y; + if (trim > 0) { + ggml_tensor * tail = state_in.at(state_name); // [trim, OC] + ggml_tensor * head = ggml_add(ctx0, ggml_view_2d(ctx0, y, trim, y->ne[1], y->nb[1], 0), tail); + if (emit_len > trim) { + ggml_tensor * middle = ggml_view_2d(ctx0, y, emit_len - trim, y->ne[1], y->nb[1], (size_t) trim * y->nb[0]); + out = ggml_concat(ctx0, head, middle, 0); + } else { + out = head; + } + ggml_tensor * new_tail = ggml_cont(ctx0, ggml_view_2d(ctx0, y, trim, y->ne[1], y->nb[1], (size_t) emit_len * y->nb[0])); + state_out.push_back({state_name, new_tail}); + } + if (b) { + out = ggml_add(ctx0, out, ggml_reshape_2d(ctx0, b, 1, b->ne[0])); + } + return out; +} + +// SnakeBeta activation: y = x + sin(alpha*x)^2 * inv_beta (alpha/inv_beta folded via exp/reciprocal at conversion time) +// x: [T, C]. alpha/beta: [C], broadcasts over T +ggml_tensor * clip_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const { + ggml_tensor * a = ggml_reshape_2d(ctx0, alpha, 1, alpha->ne[0]); + ggml_tensor * b = ggml_reshape_2d(ctx0, beta, 1, beta->ne[0]); + + // expand reshapes first so mul/sin/sqr/mul/add lands as consecutive nodes, letting backends fuse them + ggml_build_forward_expand(gf, a); + ggml_build_forward_expand(gf, b); + + ggml_tensor * s = ggml_sin(ctx0, ggml_mul(ctx0, x, a)); + s = ggml_sqr(ctx0, s); + s = ggml_mul(ctx0, s, b); + return ggml_add(ctx0, x, s); +} + +// RVQ codebook decode: T frames of 16 codes -> 512-dim hidden (C-first, [512, T]) +// codebook 0 (semantic) and 1..15 (acoustic) sum within their group, project separately, then add +ggml_tensor * clip_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_codes) const { + const auto & c2w = model.c2w; + const int64_t T = inp_codes->ne[0]; + + // ids for codebook group g over all T frames, [T] I32 + auto group_ids = [&](int g) { + return ggml_view_1d(ctx0, inp_codes, T, (size_t) g * inp_codes->nb[1]); + }; + + ggml_tensor * sem = ggml_get_rows(ctx0, c2w.quant_first_cb_w, group_ids(0)); // [256, T] + ggml_tensor * sem_out = ggml_mul_mat(ctx0, c2w.quant_first_out_w, sem); // [512, T] + + ggml_tensor * acc = nullptr; + const int64_t n_acoustic = c2w.quant_rest_cb_w->ne[2]; + for (int g = 1; g <= n_acoustic; g++) { + ggml_tensor * cb_g = ggml_view_2d(ctx0, c2w.quant_rest_cb_w, c2w.quant_rest_cb_w->ne[0], c2w.quant_rest_cb_w->ne[1], + c2w.quant_rest_cb_w->nb[1], (size_t) (g - 1) * c2w.quant_rest_cb_w->nb[2]); + ggml_tensor * embd = ggml_get_rows(ctx0, cb_g, group_ids(g)); // [256, T] + acc = acc ? ggml_add(ctx0, acc, embd) : embd; + } + ggml_tensor * ac_out = ggml_mul_mat(ctx0, c2w.quant_rest_out_w, acc); // [512, T] + + ggml_tensor * hidden = ggml_add(ctx0, sem_out, ac_out); + cb(hidden, "wav_quant_hidden", -1); + return hidden; +} + +// one pre_transformer layer over a batch of N = sliding_window new frames +// attention runs over [(W-1)-frame prefix from the last batch] + [N new frames] +// RoPE positions come from a persisted counter, so phases line up across batches +ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const { + const int n_head = hparams.wav_tfm_n_head; + const int n_head_kv = hparams.wav_tfm_n_head_kv; + const int64_t d_head = layer.q_w->ne[1] / n_head; + const float kq_scale = 1.0f / sqrtf((float) d_head); + const int64_t W = hparams.wav_tfm_swa; // == N, frames per batch + const int64_t N = cur->ne[1]; + const int64_t prefix = W - 1; + const int64_t total_kv = prefix + N; + + ggml_tensor * residual = cur; + ggml_tensor * h = ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps); + h = ggml_mul(ctx0, h, layer.ln_1_w); + + ggml_tensor * q = ggml_mul_mat(ctx0, layer.q_w, h); // [n_head*d_head, N] + ggml_tensor * k = ggml_mul_mat(ctx0, layer.k_w, h); // [n_head_kv*d_head, N] + ggml_tensor * v = ggml_mul_mat(ctx0, layer.v_w, h); // [n_head_kv*d_head, N] + + q = ggml_reshape_3d(ctx0, q, d_head, n_head, N); + k = ggml_reshape_3d(ctx0, k, d_head, n_head_kv, N); + + // real, ever-increasing positions: base (persisted) .. base+N-1 + ggml_tensor * base = ggml_reshape_1d(ctx0, state_in.at("tfm_pos"), 1); + ggml_tensor * offset = ggml_arange(ctx0, 0.0f, (float) N, 1.0f); + ggml_tensor * pos = ggml_cast(ctx0, ggml_add(ctx0, offset, base), GGML_TYPE_I32); + + q = ggml_rope_ext(ctx0, q, pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0, + hparams.wav_tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + k = ggml_rope_ext(ctx0, k, pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0, + hparams.wav_tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + + // the position counter is the same for all layers, push it once from layer 0 + if (il == 0) { + state_out.push_back({"tfm_pos", ggml_scale_bias(ctx0, state_in.at("tfm_pos"), 1.0f, (float) N)}); + } + + ggml_tensor * k_new = ggml_reshape_2d(ctx0, k, d_head * n_head_kv, N); + ggml_tensor * v_new = ggml_reshape_2d(ctx0, v, d_head * n_head_kv, N); + + ggml_tensor * old_k = state_in.at("tfm_k_" + std::to_string(il)); // [d_head*n_head_kv, W-1] + ggml_tensor * old_v = state_in.at("tfm_v_" + std::to_string(il)); + + ggml_tensor * k_full = ggml_concat(ctx0, old_k, k_new, 1); // [.., prefix+N] + ggml_tensor * v_full = ggml_concat(ctx0, old_v, v_new, 1); + + // next batch's prefix: the last (W-1) frames of this batch + state_out.push_back({"tfm_k_" + std::to_string(il), + ggml_cont(ctx0, ggml_view_2d(ctx0, k_full, k_full->ne[0], prefix, k_full->nb[1], (size_t) N * k_full->nb[1]))}); + state_out.push_back({"tfm_v_" + std::to_string(il), + ggml_cont(ctx0, ggml_view_2d(ctx0, v_full, v_full->ne[0], prefix, v_full->nb[1], (size_t) N * v_full->nb[1]))}); + + // banded causal mask: key j is visible to query i iff 0 <= (prefix+i) - j < W + ggml_tensor * pos_k = ggml_reshape_2d(ctx0, ggml_arange(ctx0, 0.0f, (float) total_kv, 1.0f), total_kv, 1); + ggml_tensor * pos_q = ggml_reshape_2d(ctx0, ggml_arange(ctx0, (float) prefix, (float) (prefix + N), 1.0f), 1, N); + ggml_tensor * pos_q_grid = ggml_repeat_4d(ctx0, pos_q, total_kv, N, 1, 1); + ggml_tensor * diff = ggml_sub(ctx0, pos_q_grid, pos_k); // [total_kv, N] + + ggml_tensor * causal_keep = ggml_step(ctx0, ggml_scale_bias(ctx0, diff, 1.0f, 0.5f)); // diff >= 0 + ggml_tensor * in_window = ggml_step(ctx0, ggml_scale_bias(ctx0, diff, -1.0f, (float) W - 0.5f)); // diff < W + ggml_tensor * keep = ggml_mul(ctx0, causal_keep, in_window); + + // on a cold start, key j is real state only when j >= prefix - tfm_pos, mask out the rest + ggml_tensor * warm = ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), + 1.0f, 0.5f - (float) prefix)); // j + pos > prefix - 0.5 + keep = ggml_mul(ctx0, keep, warm); + + ggml_tensor * mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), total_kv, N, 1, 1); // 0 = keep, -inf = masked + + ggml_tensor * q_cur = ggml_reshape_4d(ctx0, q, d_head, n_head, N, 1); + ggml_tensor * k_cur = ggml_reshape_4d(ctx0, k_full, d_head, n_head_kv, total_kv, 1); + ggml_tensor * v_cur = ggml_reshape_4d(ctx0, v_full, d_head, n_head_kv, total_kv, 1); + + ggml_tensor * attn_out = build_attn(layer.o_w, layer.o_b, q_cur, k_cur, v_cur, mask, kq_scale, il); + if (layer.ls_1_w) { + attn_out = ggml_mul(ctx0, attn_out, layer.ls_1_w); + } + cur = ggml_add(ctx0, residual, attn_out); + + ggml_tensor * residual2 = cur; + ggml_tensor * h2 = ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps); + h2 = ggml_mul(ctx0, h2, layer.ln_2_w); + + ggml_tensor * gate = ggml_mul_mat(ctx0, layer.ff_gate_w, h2); + ggml_tensor * up = ggml_mul_mat(ctx0, layer.ff_up_w, h2); + ggml_tensor * gu = ggml_swiglu_split(ctx0, gate, up); + ggml_tensor * down = ggml_mul_mat(ctx0, layer.ff_down_w, gu); + if (layer.ls_2_w) { + down = ggml_mul(ctx0, down, layer.ls_2_w); + } + return ggml_add(ctx0, residual2, down); +} + +// dwconv -> LayerNorm -> pwconv1 -> GELU -> pwconv2 -> layer scale -> residual +// x: [T, C] T-first; LayerNorm/pwconv need C on ne0, so this transposes in and back out +ggml_tensor * clip_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const { + ggml_tensor * residual = x; + + ggml_tensor * h = causal_conv1d_dw(x, blk.dwconv_w, blk.dwconv_b, state_prefix + "_dwconv"); // [T, C] + ggml_tensor * hc = ggml_cont(ctx0, ggml_transpose(ctx0, h)); // [C, T] + + hc = ggml_norm(ctx0, hc, 1e-6f); + hc = ggml_mul(ctx0, hc, blk.norm_w); + hc = ggml_add(ctx0, hc, blk.norm_b); + + ggml_tensor * g = ggml_mul_mat(ctx0, blk.pw1_w, hc); + g = ggml_add(ctx0, g, blk.pw1_b); + g = ggml_gelu(ctx0, g); + g = ggml_mul_mat(ctx0, blk.pw2_w, g); + g = ggml_add(ctx0, g, blk.pw2_b); + g = ggml_mul(ctx0, g, blk.gamma); + + ggml_tensor * g_t = ggml_cont(ctx0, ggml_transpose(ctx0, g)); // back to [T, C] + return ggml_add(ctx0, residual, g_t); +} + +// SnakeBeta -> dilated causal conv (k=7) -> SnakeBeta -> pointwise causal conv (k=1) -> residual. +// x: [T, C]. returns [T, C]. +ggml_tensor * clip_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const { + ggml_tensor * residual = x; + ggml_tensor * h = snake(x, res.act1_alpha, res.act1_beta); + h = causal_conv1d(h, res.conv1_w, res.conv1_b, dilation, state_name); + h = snake(h, res.act2_alpha, res.act2_beta); + h = causal_conv1d(h, res.conv2_w, res.conv2_b, 1, ""); // k=1, no left-context needed + return ggml_add(ctx0, residual, h); +} + +// RVQ codes -> raw PCM for a batch of N = sliding_window frames +ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) const { + const auto & c2w = model.c2w; + + // 1. quantizer decode: N frames of 16 codes -> [512, N] (C-first) + ggml_tensor * hidden = quant_decode(inp_codes); + + // 2. pre_conv: [512, N] -> T-first [N, 512] -> causal conv k=3 -> [N, 1024] + ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, hidden)); // [N, 512] + x = causal_conv1d(x, c2w.pre_conv_w, c2w.pre_conv_b, 1, "pre_conv"); // [N, 1024] + cb(x, "wav_pre_conv_out", -1); + + // 3. pre_transformer: back to C-first [1024, N], project down, run the layers, project back up + ggml_tensor * cur = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [1024, N] + cur = ggml_mul_mat(ctx0, c2w.tfm_in_proj_w, cur); + cur = ggml_add(ctx0, cur, c2w.tfm_in_proj_b); // [512 (tfm hidden), N] + + for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { + cur = tfm_layer_forward(cur, c2w.tfm_layers[il], il); + } + + cur = ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps); + cur = ggml_mul(ctx0, cur, c2w.tfm_output_norm_w); + cur = ggml_mul_mat(ctx0, c2w.tfm_out_proj_w, cur); + cur = ggml_add(ctx0, cur, c2w.tfm_out_proj_b); // [1024, N] + cb(cur, "wav_tfm_out", -1); + + // 4. upsample: 2x (causal ConvTranspose1d, stride 2 + ConvNeXt block), back to T-first + // kernel == stride here, so there is no overlap tail to persist + x = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [N, 1024] + for (size_t il = 0; il < c2w.upsample.size(); il++) { + const auto & up = c2w.upsample[il]; + x = causal_conv_transpose1d(x, up.conv_w, up.conv_b, 2, ""); + x = convnext_block(x, up, "up" + std::to_string(il)); + cb(x, "wav_upsample_out", (int) il); + } + + // 5. DAC decoder: conv_pre -> n blocks (SnakeBeta -> ConvTranspose1d -> 3 res units) -> conv_post + static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 }; + + x = causal_conv1d(x, c2w.dac_entry_w, c2w.dac_entry_b, 1, "dac_entry"); + cb(x, "wav_dac_entry_out", -1); + + for (size_t il = 0; il < c2w.dac.size(); il++) { + const auto & blk = c2w.dac[il]; + const int stride = (int) (blk.conv_w->ne[0] / 2); // kernel == 2*stride for all 4 blocks + const std::string blk_name = "dac" + std::to_string(il); + x = snake(x, blk.snake_alpha, blk.snake_beta); + x = causal_conv_transpose1d(x, blk.conv_w, blk.conv_b, stride, blk_name + "_tail"); + for (size_t ir = 0; ir < blk.res.size(); ir++) { + x = dac_res_unit(x, blk.res[ir], DAC_DILATIONS[ir], blk_name + "_res" + std::to_string(ir)); + } + cb(x, "wav_dac_block_out", (int) il); + } + + x = snake(x, c2w.dac_post_snake_alpha, c2w.dac_post_snake_beta); + x = causal_conv1d(x, c2w.dac_post_conv_w, c2w.dac_post_conv_b, 1, "dac_post_conv"); // [n_samples, 1] + + x = ggml_clamp(ctx0, x, -1.0f, 1.0f); + x = ggml_reshape_1d(ctx0, x, x->ne[0]); + cb(x, "wav_audio_out", -1); + return x; +} + +// code2wav's persisted state buffers: RoPE position counter, K/V per pre_transformer layer, +// left-context/tail per stateful conv. shape lookup only, no graph needed +std::vector list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model) { + const auto & c2w = model.c2w; + std::vector slots; + + slots.push_back({"tfm_pos", 1, 1}); + + // prefix is (W-1) frames, the batch itself gives the other N=W frames (see tfm_layer_forward) + const int64_t d_head = c2w.tfm_layers[0].q_w->ne[1] / hparams.wav_tfm_n_head; + const int64_t kv_ch = d_head * hparams.wav_tfm_n_head_kv; + const int64_t prefix = hparams.wav_tfm_swa - 1; + for (int il = 0; il < hparams.wav_tfm_n_layer; il++) { + slots.push_back({"tfm_k_" + std::to_string(il), kv_ch, prefix}); + slots.push_back({"tfm_v_" + std::to_string(il), kv_ch, prefix}); + } + + slots.push_back({"pre_conv", c2w.pre_conv_w->ne[0] - 1, c2w.pre_conv_w->ne[1]}); + + for (size_t il = 0; il < c2w.upsample.size(); il++) { + const auto & up = c2w.upsample[il]; + slots.push_back({"up" + std::to_string(il) + "_dwconv", up.dwconv_w->ne[0] - 1, up.dwconv_w->ne[2]}); + } + + slots.push_back({"dac_entry", c2w.dac_entry_w->ne[0] - 1, c2w.dac_entry_w->ne[1]}); + + static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 }; + for (size_t il = 0; il < c2w.dac.size(); il++) { + const auto & blk = c2w.dac[il]; + const int64_t stride = blk.conv_w->ne[0] / 2; // kernel == 2*stride for all 4 blocks + const std::string blk_name = "dac" + std::to_string(il); + slots.push_back({blk_name + "_tail", stride, blk.conv_w->ne[1]}); + for (size_t ir = 0; ir < blk.res.size(); ir++) { + const auto & res = blk.res[ir]; + slots.push_back({blk_name + "_res" + std::to_string(ir), + (res.conv1_w->ne[0] - 1) * DAC_DILATIONS[ir], res.conv1_w->ne[1]}); + } + } + + slots.push_back({"dac_post_conv", c2w.dac_post_conv_w->ne[0] - 1, c2w.dac_post_conv_w->ne[1]}); + + return slots; +} + +// both sub-graphs are always built, so the topology stays constant +// ggml_build_forward_select() then picks the one that actually runs +ggml_cgraph * clip_graph_qwen3tts_gen::build() { + GGML_ASSERT(n_batch == 1); // this module only ever processes one frame at a time + + int idx; + switch (gen_process) { + case CLIP_GEN_PROCESS_GEN_CODE: idx = 0; break; + case CLIP_GEN_PROCESS_GEN_WAV: idx = 1; break; + default: GGML_ABORT("unknown gen_process"); + } + + // ---- CLIP_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ---- + // not build_inp_raw(), a GEN_WAV call's `img` has no hidden-state data + ggml_tensor * h_state = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd); + ggml_set_name(h_state, "inp_raw"); // must keep this exact name, clip_encode() sets it by name + ggml_set_input(h_state); + + ggml_tensor * code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); + ggml_set_name(code0, "inp_code0"); + ggml_set_input(code0); + + ggml_tensor * code0_embd = ggml_get_rows(ctx0, model.gen_code_out_embd_w, code0); + code0_embd = ggml_reshape_1d(ctx0, code0_embd, code0_embd->ne[0]); + cb(code0_embd, "code0_embd", -1); + + const int64_t n_acoustic = model.gen_code_head_w->ne[2]; // 15 + const int n_codes = (int) n_acoustic + 1; // 16 + const int64_t n_kv_pad = n_codes; + const int n_layer = (int) model.layers.size(); + const int n_head = hparams.n_head; + const int n_head_kv = hparams.n_head_kv; + const int64_t d_head = model.layers[0].q_w->ne[1] / n_head; + + // zero-filled per layer k/v caches, so masked-out rows can't hold garbage + std::vector k_cache(n_layer), v_cache(n_layer); + for (int il = 0; il < n_layer; il++) { + k_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, d_head * n_head_kv, n_kv_pad), 0.0f); + v_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, d_head * n_head_kv, n_kv_pad), 0.0f); + } + + code_gen cg(*this, top_k, top_p); + + ggml_tensor * out_code_cache = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_codes); + out_code_cache = cg.cache_set(out_code_cache, 0, code0); + + ggml_tensor * inp_rand0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1); + ggml_set_name(inp_rand0, "inp_rand_0"); + ggml_set_input(inp_rand0); + + cg.prefill(k_cache, v_cache, out_code_cache, h_state, code0_embd, inp_rand0); + + for (int g = 1; g < n_acoustic; g++) { + ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1); + ggml_set_name(inp_rand, ("inp_rand_" + std::to_string(g)).c_str()); + ggml_set_input(inp_rand); + out_code_cache = cg.step(k_cache, v_cache, out_code_cache, inp_rand, g); + } + + // output 1: this frame's 16 sampled codes, for the caller's code2wav window + ggml_tensor * out_codes = ggml_cont(ctx0, out_code_cache); + ggml_set_name(out_codes, "out_codes"); + ggml_set_output(out_codes); + + // output 2: sum of all 16 codebook embeddings, fed back to the talker for the next frame + ggml_tensor * out_embd = code0_embd; + for (int g = 1; g <= n_acoustic; g++) { + ggml_tensor * code_g = ggml_view_1d(ctx0, out_code_cache, 1, (size_t) g * out_code_cache->nb[1]); + + ggml_tensor * embd_g = ggml_view_2d(ctx0, model.gen_code_embd_w, model.gen_code_embd_w->ne[0], model.gen_code_embd_w->ne[1], + model.gen_code_embd_w->nb[1], (size_t) (g - 1) * model.gen_code_embd_w->nb[2]); + ggml_tensor * e = ggml_get_rows(ctx0, embd_g, code_g); + e = ggml_reshape_1d(ctx0, e, e->ne[0]); + + out_embd = ggml_add(ctx0, out_embd, e); + } + out_embd = ggml_reshape_2d(ctx0, out_embd, out_embd->ne[0], 1); + cb(out_embd, "gen_audio_out", -1); + + // ---- CLIP_GEN_PROCESS_GEN_WAV: 16 RVQ codes -> raw PCM ---- + const int n_frames = hparams.wav_tfm_swa; // frames per batch, == the attention window + + ggml_tensor * inp_codes = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_frames, n_codes); + ggml_set_name(inp_codes, "inp_codes"); + ggml_set_input(inp_codes); + + code2wav c2w(*this); + for (const auto & slot : list_c2w_state_slots(hparams, model)) { + ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_set_name(t, ("state_in_" + slot.name).c_str()); + ggml_set_input(t); + c2w.state_in[slot.name] = t; + } + + ggml_tensor * out_audio = c2w.decode(inp_codes); + ggml_set_name(out_audio, "out_audio"); + ggml_set_output(out_audio); + + for (auto & slot : c2w.state_out) { + ggml_set_name(slot.second, ("state_out_" + slot.first).c_str()); + ggml_set_output(slot.second); + } + + // out_embd goes last, clip_encode() reads it back via ggml_graph_node(gf, -1) + ggml_tensor * outs[2]; + outs[0] = out_codes; outs[1] = out_audio; + ggml_build_forward_select(gf, outs, 2, idx); + for (auto & slot : c2w.state_out) { + outs[0] = out_codes; outs[1] = slot.second; + ggml_build_forward_select(gf, outs, 2, idx); + } + outs[0] = out_embd; outs[1] = out_audio; + ggml_build_forward_select(gf, outs, 2, idx); + + return gf; +} diff --git a/tools/mtmd/models/qwen3tts-spkenc.cpp b/tools/mtmd/models/qwen3tts-spkenc.cpp new file mode 100644 index 000000000000..d4659fd63dc4 --- /dev/null +++ b/tools/mtmd/models/qwen3tts-spkenc.cpp @@ -0,0 +1,197 @@ +#include "models.h" + +static constexpr int SPK_RES2NET_SCALE = 8; // enc_res2net_scale +static constexpr int SPK_DILATIONS[3] = { 2, 3, 4 }; // enc_dilations[1..3] + +// conv1d, kernel K, padding "same" (reflect), dilation d +// x: [C, T] (ne[0]=C, ne[1]=T) -> [out_c, T] +ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const { + const int K = (int) w->ne[0]; + const int IC = (int) w->ne[1]; + const int OC = (int) w->ne[2]; + const int pad = ((K - 1) * dilation) / 2; + + // ggml_pad_reflect_1d pads ne[0], so bring T onto ne[0] first, same layout as im2col wants + ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, IC] + if (pad > 0) { + x_t = ggml_pad_reflect_1d(ctx0, x_t, pad, pad); // [T + 2*pad, IC] + } + ggml_tensor * x4d = ggml_reshape_4d(ctx0, x_t, x_t->ne[0], IC, 1, 1); + + // dummy F32 kernel, im2col only reads its shape, so a quantized w does not assert + ggml_tensor * dummy = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, K, IC, 1, 1); + + ggml_tensor * col = ggml_im2col(ctx0, dummy, x4d, 1, 1, 0, 0, dilation, 1, false, GGML_TYPE_F32); + const int64_t T_out = col->ne[1]; + col = ggml_reshape_2d(ctx0, col, (int64_t) K * IC, T_out); + + ggml_tensor * w2d = ggml_reshape_2d(ctx0, w, (int64_t) K * IC, OC); + ggml_tensor * y = ggml_mul_mat(ctx0, w2d, col); // [OC, T_out] + ggml_mul_mat_set_prec(y, GGML_PREC_F32); + + ggml_tensor * b2d = ggml_reshape_2d(ctx0, b, OC, 1); + y = ggml_add(ctx0, y, b2d); + return y; +} + +// Res2Net: split channel axis into `scale` chunks, chain dilated conv1d branches +// x: [C, T] -> [C, T] +ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const { + const int64_t C = x->ne[0]; + const int64_t T = x->ne[1]; + const int64_t Cs = C / scale; + + std::vector outs; + outs.reserve(scale); + + auto chunk = [&](int i) -> ggml_tensor * { + return ggml_view_2d(ctx0, x, Cs, T, x->nb[1], (size_t) i * Cs * x->nb[0]); + }; + + ggml_tensor * prev = nullptr; + for (int i = 0; i < scale; i++) { + ggml_tensor * c = ggml_cont(ctx0, chunk(i)); + if (i == 0) { + outs.push_back(c); + continue; + } + ggml_tensor * inp = (i >= 2) ? ggml_add(ctx0, c, prev) : c; + ggml_tensor * y = conv1d_same(inp, layer.res2_conv_w[i - 1], layer.res2_conv_b[i - 1], dilation); + y = ggml_relu(ctx0, y); + outs.push_back(y); + prev = y; + } + + ggml_tensor * acc = outs[0]; + for (int i = 1; i < scale; i++) { + acc = ggml_concat(ctx0, acc, outs[i], 0); + } + return acc; +} + +// squeeze-and-excitation gate. x: [C, T] -> [C, T] +ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_layer & layer) const { + // temporal mean, keepdim: transpose so T is on ne[0], reduce, transpose back + ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C] + ggml_tensor * mean = ggml_mean(ctx0, x_t); // [1, C] + mean = ggml_cont(ctx0, ggml_transpose(ctx0, mean)); // [C, 1] + + ggml_tensor * h = conv1d_same(mean, layer.se_conv1_w, layer.se_conv1_b, 1); + h = ggml_relu(ctx0, h); + h = conv1d_same(h, layer.se_conv2_w, layer.se_conv2_b, 1); + h = ggml_sigmoid(ctx0, h); // [C, 1] + + return ggml_mul(ctx0, x, h); // broadcast gate over T +} + +// tdnn1 -> res2net -> tdnn2 -> se, plus residual. x: [C, T] -> [C, T] +ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const { + ggml_tensor * residual = x; + ggml_tensor * h = conv1d_same(x, layer.conv_pw1_w, layer.conv_pw1_b, 1); // tdnn1 + h = ggml_relu(ctx0, h); + h = res2net(h, layer, dilation, scale); + h = conv1d_same(h, layer.conv_pw2_w, layer.conv_pw2_b, 1); // tdnn2 + h = ggml_relu(ctx0, h); + h = se_block(h, layer); + return ggml_add(ctx0, h, residual); +} + +// attentive statistics pooling. x: [C, T] -> [2*C, 1] +ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const { + const int64_t T = x->ne[1]; + + // mean over T: [C, 1] + ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); + ggml_tensor * mean = ggml_mean(ctx0, x_t); + mean = ggml_cont(ctx0, ggml_transpose(ctx0, mean)); + + // std over T: sqrt(clamp(mean((x - mean)^2), eps)) + ggml_tensor * mean_rep = ggml_repeat(ctx0, mean, x); + ggml_tensor * centered = ggml_sub(ctx0, x, mean_rep); + ggml_tensor * var_t = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_sqr(ctx0, centered))); + ggml_tensor * var = ggml_mean(ctx0, var_t); + var = ggml_cont(ctx0, ggml_transpose(ctx0, var)); + var = ggml_scale_bias(ctx0, var, 1.0f, 1e-12f); + ggml_tensor * std = ggml_sqrt(ctx0, var); + + // attention input: cat([x, mean, std]) along channel axis -> [3C, T] + ggml_tensor * std_rep = ggml_repeat(ctx0, std, x); + ggml_tensor * cat = ggml_concat(ctx0, x, mean_rep, 0); + cat = ggml_concat(ctx0, cat, std_rep, 0); + + // attention TDNN (3C -> attn_c) + ReLU, tanh, then 1x1 conv (attn_c -> C) + ggml_tensor * a = conv1d_same(cat, model.spk_asp_tdnn_w, model.spk_asp_tdnn_b, 1); + a = ggml_relu(ctx0, a); + a = ggml_tanh(ctx0, a); + a = conv1d_same(a, model.spk_asp_attn_w, model.spk_asp_attn_b, 1); + + // softmax over T + ggml_tensor * a_t = ggml_cont(ctx0, ggml_transpose(ctx0, a)); // [T, C] + ggml_tensor * w_t = ggml_soft_max(ctx0, a_t); + ggml_tensor * w = ggml_cont(ctx0, ggml_transpose(ctx0, w_t)); // [C, T] + + // weighted mean: sum(w * x) over T, multiply by T to undo ggml_mean's 1/T scaling + ggml_tensor * wx = ggml_mul(ctx0, w, x); + ggml_tensor * wx_t = ggml_cont(ctx0, ggml_transpose(ctx0, wx)); + ggml_tensor * w_mean = ggml_mean(ctx0, wx_t); + w_mean = ggml_scale(ctx0, w_mean, (float) T); + w_mean = ggml_cont(ctx0, ggml_transpose(ctx0, w_mean)); // [C, 1] + + // weighted std: sum(w * (x - w_mean)^2) over T + ggml_tensor * w_mean_rep = ggml_repeat(ctx0, w_mean, x); + ggml_tensor * dev = ggml_sub(ctx0, x, w_mean_rep); + ggml_tensor * w_var_in = ggml_mul(ctx0, w, ggml_sqr(ctx0, dev)); + ggml_tensor * w_var_t = ggml_cont(ctx0, ggml_transpose(ctx0, w_var_in)); + ggml_tensor * w_var = ggml_mean(ctx0, w_var_t); + w_var = ggml_scale(ctx0, w_var, (float) T); + w_var = ggml_cont(ctx0, ggml_transpose(ctx0, w_var)); + w_var = ggml_scale_bias(ctx0, w_var, 1.0f, 1e-12f); + ggml_tensor * w_std = ggml_sqrt(ctx0, w_var); + + return ggml_concat(ctx0, w_mean, w_std, 0); // [2C, 1] +} + +ggml_cgraph * clip_graph_qwen3tts_spkenc::build() { + // inp_raw: [T, n_mel, 1, 1], from mtmd_audio_preprocessor_qwen3tts_spk + ggml_tensor * inp = build_inp_raw(1); + inp = ggml_reshape_2d(ctx0, inp, inp->ne[0], inp->ne[1]); + + // this file's convention is [C, T]; the preprocessor delivers [T, C] + ggml_tensor * mel = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [n_mel, T] + cb(mel, "mel", -1); + + // frontend conv0 TDNN k=5, dilation=1: 128 -> 512 + ggml_tensor * cur = conv1d_same(mel, model.conv1d_1_w, model.conv1d_1_b, 1); + cur = ggml_relu(ctx0, cur); + cb(cur, "frontend", -1); + + // 3 SE-Res2Net blocks at dilations 2, 3, 4 + GGML_ASSERT((int) model.layers.size() == 3); + std::vector blk_out(3); + for (int il = 0; il < 3; il++) { + cur = se_res2net_block(cur, model.layers[il], SPK_DILATIONS[il], SPK_RES2NET_SCALE); + blk_out[il] = cur; + cb(cur, "block_out", il); + } + + // multi-layer feature aggregation: cat blk[0..2] then TDNN k=1 + ReLU + ggml_tensor * cat = ggml_concat(ctx0, blk_out[0], blk_out[1], 0); + cat = ggml_concat(ctx0, cat, blk_out[2], 0); // [1536, T] + ggml_tensor * mfa = conv1d_same(cat, model.conv_out_w, model.conv_out_b, 1); + mfa = ggml_relu(ctx0, mfa); + cb(mfa, "mfa", -1); + + // attentive statistics pooling: [1536, T] -> [3072, 1] + ggml_tensor * stats = attentive_stats_pool(mfa); + cb(stats, "asp", -1); + + // final FC k=1: [3072, 1] -> [enc_dim, 1] + ggml_tensor * emb = conv1d_same(stats, model.mm_fc_w, model.mm_fc_b, 1); + + emb = ggml_reshape_1d(ctx0, emb, emb->ne[0]); + emb = ggml_cont(ctx0, emb); + cb(emb, "spk_embedding", -1); + + ggml_build_forward_expand(gf, emb); + return gf; +} diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index fea03557d05c..7fbc18ea9396 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -791,6 +791,66 @@ bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * return true; } +// +// mtmd_audio_preprocessor_qwen3tts_spk +// +// same as mel_spectrogram() in modeling_qwen3_tts.py +// ECAPA-TDNN takes the whole clip in one pass, so no Whisper-style chunking or normalization +// + +void mtmd_audio_preprocessor_qwen3tts_spk::initialize() { + cache.fill_sin_cos_table(hparams.audio_n_fft); + cache.fill_hann_window(hparams.audio_window_len, true); + cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate); +} + +bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * samples, + size_t n_samples, + std::vector & output) { + if (n_samples == 0) { + return false; + } + + GGML_ASSERT(!cache.sin_vals.empty()); + GGML_ASSERT(!cache.cos_vals.empty()); + GGML_ASSERT(!cache.filters.data.empty()); + + // reflect pad by (n_fft - hop) / 2 = 384, matching center=False STFT framing + const int pad = (hparams.audio_n_fft - hparams.audio_hop_len) / 2; + if (n_samples < (size_t) pad + 1) { + return false; + } + + std::vector padded(n_samples + 2 * pad, 0.0f); + for (int i = 0; i < pad; i++) { + padded[i] = samples[pad - i]; + } + std::copy(samples, samples + n_samples, padded.begin() + pad); + for (int i = 0; i < pad; i++) { + padded[n_samples + pad + i] = samples[n_samples - 2 - i]; + } + + filter_params params; + params.n_mel = hparams.n_mel_bins; + params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + params.hann_window_size = hparams.audio_window_len; + params.hop_length = hparams.audio_hop_len; + params.sample_rate = hparams.audio_sample_rate; + params.no_padding = true; // reflect padding already applied above + params.use_natural_log = true; + params.use_magnitude = true; + params.mel_floor = 1e-5f; + + mtmd_audio_mel out; + bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, out); + if (!ok) { + return false; + } + + output.push_back(std::move(out)); + return true; +} + // // mtmd_audio_preprocessor_conformer // diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index f65f282d96e2..b4d6f7259808 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -120,6 +120,15 @@ struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { mtmd_audio_cache cache; }; +struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + void initialize() override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + + private: + mtmd_audio_cache cache; +}; + struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { } void initialize() override; diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp index 08288c868139..07b45b644073 100644 --- a/tools/mtmd/mtmd-cli.cpp +++ b/tools/mtmd/mtmd-cli.cpp @@ -116,6 +116,14 @@ struct mtmd_cli_context { exit(1); } + init_vision_context(params); + + if (!mtmd_helper_model_can_chat(lctx, ctx_vision.get())) { + LOG_ERR("Model does not support chat mode\n"); + LOG_ERR("Hint: for TTS models, please use llama-tts\n"); + exit(1); + } + if (!llama_model_chat_template(model, nullptr) && params.chat_template.empty()) { LOG_ERR("Model does not have chat template.\n"); LOG_ERR(" For old llava models, you may need to use '--chat-template vicuna'\n"); @@ -129,8 +137,6 @@ struct mtmd_cli_context { chat_history.clear(); LOG_INF("%s: chat template example:\n%s\n", __func__, common_chat_format_example(tmpls.get(), params.use_jinja, params.default_template_kwargs).c_str()); - init_vision_context(params); - // load antiprompt tokens for legacy templates if (params.chat_template == "vicuna") { antiprompt_tokens = common_tokenize(lctx, "ASSISTANT:", false, true); diff --git a/tools/mtmd/mtmd-helper-common.h b/tools/mtmd/mtmd-helper-common.h new file mode 100644 index 000000000000..968b4df9c8f7 --- /dev/null +++ b/tools/mtmd/mtmd-helper-common.h @@ -0,0 +1,180 @@ +#pragma once + +// shared internal utilities for the mtmd-helper-*.cpp translation units +// (mtmd-helper.cpp, mtmd-helper-gen.cpp) +// NOT part of the public mtmd-helper.h API + +#include "ggml.h" +#include "llama.h" +#include "mtmd.h" + +#include +#include +#include +#include + +// +// logging +// + +struct mtmd_helper_logger { + ggml_log_callback default_callback = [](ggml_log_level level, const char * text, void * user_data) { + (void) level; + (void) user_data; + fputs(text, stderr); + fflush(stderr); + }; + + ggml_log_callback log_callback = default_callback; + void * log_callback_user_data; + + void log_v(enum ggml_log_level level, const char * format, va_list args) { + if (format == NULL) { + return; + } + va_list args_copy; + va_copy(args_copy, args); + char buffer[128]; + int len = vsnprintf(buffer, 128, format, args); + if (len < 128) { + log_callback(level, buffer, log_callback_user_data); + } else { + char * buffer2 = (char *) calloc(len + 1, sizeof(char)); + vsnprintf(buffer2, len + 1, format, args_copy); + buffer2[len] = 0; + log_callback(level, buffer2, log_callback_user_data); + free(buffer2); + } + va_end(args_copy); + } + + void log(enum ggml_log_level level, const char * format, ...) { + va_list args; + va_start(args, format); + log_v(level, format, args); + va_end(args); + } +}; + +// inline, so all TUs including this header share one instance +inline mtmd_helper_logger g_logger; + +#define LOG_DBG(...) g_logger.log(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) +#define LOG_INF(...) g_logger.log(GGML_LOG_LEVEL_INFO, __VA_ARGS__) +#define LOG_WRN(...) g_logger.log(GGML_LOG_LEVEL_WARN, __VA_ARGS__) +#define LOG_ERR(...) g_logger.log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) + +// +// embd batch +// + +// helper struct to make working with embd batch easier +// note: this will be removed after llama_batch_ext refactoring +struct decode_embd_batch { + int n_pos_per_embd; + int n_mmproj_embd; + std::vector pos; + std::vector pos_view; // used by mrope + std::vector n_seq_id; + std::vector seq_id_0; + std::vector seq_ids; + std::vector logits; + llama_batch batch; + decode_embd_batch(float * embd, int32_t n_tokens, int n_pos_per_embd, int n_mmproj_embd) : n_pos_per_embd(n_pos_per_embd), n_mmproj_embd(n_mmproj_embd) { + GGML_ASSERT(n_tokens > 0 && n_pos_per_embd > 0 && n_mmproj_embd > 0); + pos .resize(n_tokens * n_pos_per_embd); + n_seq_id.resize(n_tokens); + seq_ids .resize(n_tokens + 1); + logits .resize(n_tokens); + seq_id_0.resize(1); + seq_ids [n_tokens] = nullptr; + batch = { + /*n_tokens =*/ n_tokens, + /*tokens =*/ nullptr, + /*embd =*/ embd, + /*pos =*/ pos.data(), + /*n_seq_id =*/ n_seq_id.data(), + /*seq_id =*/ seq_ids.data(), + /*logits =*/ logits.data(), + }; + } + + void set_position_normal(llama_pos pos_0, llama_seq_id seq_id) { + seq_id_0[0] = seq_id; + for (int i = 0; i < batch.n_tokens; i++) { + batch.pos [i] = pos_0 + i; + batch.n_seq_id[i] = 1; + batch.seq_id [i] = seq_id_0.data(); + batch.logits [i] = false; + } + } + + // M-RoPE for image + void set_position_mrope_2d(const std::vector & rel_pos, llama_seq_id seq_id) { + GGML_ASSERT(n_pos_per_embd == 4); + GGML_ASSERT(!rel_pos.empty() && (int32_t)rel_pos.size() == batch.n_tokens); + seq_id_0[0] = seq_id; + for (int32_t i = 0; i < batch.n_tokens; i++) { + pos[i ] = rel_pos[i].t; + pos[i + batch.n_tokens ] = rel_pos[i].y; + pos[i + batch.n_tokens * 2] = rel_pos[i].x; + pos[i + batch.n_tokens * 3] = rel_pos[i].z; + } + for (int i = 0; i < batch.n_tokens; i++) { + batch.n_seq_id[i] = 1; + batch.seq_id [i] = seq_id_0.data(); + batch.logits [i] = false; + } + } + + // M-RoPE for audio + void set_position_mrope_1d(llama_pos pos_0, llama_seq_id seq_id) { + GGML_ASSERT(n_pos_per_embd == 4); + seq_id_0[0] = seq_id; + for (int i = 0; i < batch.n_tokens; i++) { + pos[i ] = pos_0 + i; + pos[i + batch.n_tokens ] = pos_0 + i; + pos[i + batch.n_tokens * 2] = pos_0 + i; + pos[i + batch.n_tokens * 3] = pos_0 + i; + } + for (int i = 0; i < batch.n_tokens; i++) { + batch.n_seq_id[i] = 1; + batch.seq_id [i] = seq_id_0.data(); + batch.logits [i] = false; + } + } + + llama_batch get_view(int offset, int n_tokens) { + GGML_ASSERT(offset >= 0 && n_tokens > 0 && offset + n_tokens <= batch.n_tokens); + llama_pos * pos_ptr; + pos_view.clear(); + pos_view.reserve(n_tokens * n_pos_per_embd); + if (n_pos_per_embd > 1) { + // mrope + // for example, with layout of src: 1234...1234...1234...1234... + // offset 2 will give us dst: 34...34...34...34... + for (int i = 0; i < n_pos_per_embd; i++) { + // assume n_tokens is less than or equal to batch.n_tokens + // batch.n_tokens is number of **total** tokens + // n_tokens is number of viewed token + size_t src_idx = i * batch.n_tokens + offset; + pos_view.insert(pos_view.end(), + pos.data() + src_idx, + pos.data() + src_idx + n_tokens); + } + pos_ptr = pos_view.data(); + } else { + // normal + pos_ptr = pos.data() + offset; + } + return { + /*n_tokens =*/ n_tokens, + /*tokens =*/ nullptr, + /*embd =*/ batch.embd + offset * n_mmproj_embd, + /*pos =*/ pos_ptr, + /*n_seq_id =*/ batch.n_seq_id + offset, + /*seq_id =*/ batch.seq_id + offset, + /*logits =*/ batch.logits + offset, + }; + } +}; diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp new file mode 100644 index 000000000000..b52dc8e5a34e --- /dev/null +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -0,0 +1,505 @@ +#include "mtmd.h" +#include "mtmd-helper.h" +#include "mtmd-helper-common.h" +#include "llama.h" +#include "../src/llama-ext.h" + +#include +#include +#include +#include +#include +#include + +#ifdef MTMD_INTERNAL_HEADER +#error "mtmd-helper is a public library outside of mtmd. it must not include internal headers" +#endif + +// +// Audio generation helpers +// + +// --tts-lang codes -> language names used by the codec_language special tokens +static const std::unordered_map tts_lang_codes = { + { "zh", "chinese" }, + { "en", "english" }, + { "de", "german" }, + { "it", "italian" }, + { "pt", "portuguese" }, + { "es", "spanish" }, + { "ja", "japanese" }, + { "ko", "korean" }, + { "fr", "french" }, + { "ru", "russian" }, +}; + +static std::string tts_resolve_lang(const std::string & lang) { + auto it = tts_lang_codes.find(lang); + return it != tts_lang_codes.end() ? it->second : lang; +} + +static llama_token find_special_token(const llama_vocab * vocab, const std::string & piece) { + const int32_t n = llama_vocab_n_tokens(vocab); + for (llama_token t = 0; t < n; t++) { + if (piece == llama_vocab_get_text(vocab, t)) { + return t; + } + } + return LLAMA_TOKEN_NULL; +} + +static bool write_wav16(std::vector & buf, const std::vector & pcm, int32_t rate) { + // RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header + if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) { + return false; + } + const uint32_t data_sz = (uint32_t) (pcm.size() * 2); + const uint32_t riff_sz = 36 + data_sz; + const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2; + const uint16_t fmt = 1, ch = 1, align = 2, bits = 16; + const uint32_t rate32 = (uint32_t) rate; + auto put = [&](const void * p, size_t n) { + const char * c = (const char *) p; + buf.insert(buf.end(), c, c + n); + }; + put("RIFF", 4); put(&riff_sz, 4); put("WAVE", 4); + put("fmt ", 4); put(&fmt_sz, 4); + put(&fmt, 2); put(&ch, 2); put(&rate32, 4); + put(&byte_rate, 4); put(&align, 2); put(&bits, 2); + put("data", 4); put(&data_sz, 4); + for (float v : pcm) { + int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, v)) * 32767.0f); + put(&s, 2); + } + return true; +} + +class mtmd_gen_audio_pipeline { +public: + mtmd_gen_audio_pipeline(llama_context * lctx, mtmd_context * mctx) + : lctx(lctx), mctx(mctx), model(llama_get_model(lctx)), vocab(llama_model_get_vocab(model)), + n_embd(llama_model_n_embd(model)), info(mtmd_gen_audio_get_info(mctx)) {} + virtual ~mtmd_gen_audio_pipeline() = default; + + virtual void reset() = 0; + virtual int32_t set_input(const mtmd_helper_gen_audio_inp * inp) = 0; + // decodes at most n_batch prompt tokens; returns remaining count (0 = done), <0 on error + virtual int32_t step_prompt(int32_t n_batch) = 0; + // sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token, + // those read what they need from h_state_in instead + virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out) = 0; + virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0; + +protected: + llama_context * lctx; + mtmd_context * mctx; + const llama_model * model; + const llama_vocab * vocab; + int n_embd; + mtmd_gen_audio_info info; +}; + +// Qwen3-TTS: backbone samples codec_0, code_predictor gives the other 15 codebooks, +// then code2wav decodes them to PCM +class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { +public: + using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; + + void reset() override { + seq_id = 0; + pos = 0; + codes_buf.clear(); + c2w_state.clear(); + audio_pcm.clear(); + overlay.clear(); + overlay_idx = 0; + h_state_buf.clear(); + out_buf.clear(); + prompt_embd_buf.clear(); + prompt_batch.reset(); + n_prompt = 0; + prompt_pos = 0; + } + + int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { + reset(); + seq_id = inp->seq_id; + + if (!ensure_cache()) { + return 1; + } + + const std::string lang = tts_resolve_lang((inp->lang && inp->lang[0]) ? inp->lang : "english"); + const llama_token c_lang = find_special_token(vocab, ("<|codec_language_" + lang + "|>").c_str()); + if (c_lang == LLAMA_TOKEN_NULL) { + LOG_ERR("mtmd_helper_gen_audio: unknown language '%s'\n", lang.c_str()); + return 1; + } + + std::vector speaker_embd; + if (inp->speaker_ref) { + if (!encode_speaker(inp->speaker_ref, speaker_embd)) { + return 1; + } + } + + const int n_e = n_embd; + auto row = [&](llama_token t) { + return std::vector(tok_embd.begin() + (size_t) t * n_e, + tok_embd.begin() + (size_t) (t + 1) * n_e); + }; + auto sum_row = [&](llama_token a, llama_token b) { + std::vector va = row(a), vb = row(b); + for (int i = 0; i < n_e; i++) va[(size_t) i] += vb[(size_t) i]; + return va; + }; + auto sum_vec = [&](llama_token a, const std::vector & vb) { + std::vector va = row(a); + for (int i = 0; i < n_e; i++) va[(size_t) i] += vb[(size_t) i]; + return va; + }; + + // upstream chat wrap, then slices: [0:3] role, [3:-5] utterance body + const std::string full = "<|im_start|>assistant\n" + std::string(inp->prompt, inp->prompt_len) + + "<|im_end|>\n<|im_start|>assistant\n"; + std::vector ids(full.size() + 16); + int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), ids.data(), (int32_t) ids.size(), + false, true); + if (n_ids < 8) { + LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); + return 1; + } + ids.resize((size_t) n_ids); + + std::vector> prompt; + for (int i = 0; i < 3; i++) prompt.push_back(row(ids[(size_t) i])); + prompt.push_back(sum_row(tts_pad, c_think)); + prompt.push_back(sum_row(tts_pad, c_think_b)); + prompt.push_back(sum_row(tts_pad, c_lang)); + prompt.push_back(sum_row(tts_pad, c_think_e)); + if (!speaker_embd.empty()) prompt.push_back(sum_vec(tts_pad, speaker_embd)); + prompt.push_back(sum_row(tts_bos, codec_pad)); + for (int i = 3; i < n_ids - 5; i++) prompt.push_back(sum_row(ids[(size_t) i], codec_pad)); + prompt.push_back(sum_row(tts_eos, codec_pad)); + prompt.push_back(sum_row(tts_pad, codec_bos)); + + n_prompt = (int) prompt.size(); + + // the talker uses the qwen3vl interleaved mrope, all sections are equal for a text/codec stream + mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE || + llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE; + const int n_pos_per_embd = mrope ? 4 : 1; + + prompt_embd_buf.resize((size_t) n_prompt * (size_t) n_e); + for (int i = 0; i < n_prompt; i++) { + memcpy(prompt_embd_buf.data() + (size_t) i * n_e, prompt[(size_t) i].data(), (size_t) n_e * sizeof(float)); + } + + prompt_batch.reset(new decode_embd_batch(prompt_embd_buf.data(), n_prompt, n_pos_per_embd, n_e)); + if (mrope) prompt_batch->set_position_mrope_1d(0, seq_id); + else prompt_batch->set_position_normal (0, seq_id); + prompt_pos = 0; + + pos = 0; + top_k = inp->top_k > 0 ? inp->top_k : 50; + top_p = inp->top_p > 0 ? inp->top_p : 1.0f; + out_type = inp->out_type; + + // the text stream keeps flowing during generation: after frame k, the input adds + // trailing text row k on top of the codes embedding, then tts_eos, then tts_pad + for (int i = 3; i < n_ids - 5; i++) overlay.push_back(row(ids[(size_t) i])); + overlay.push_back(row(tts_eos)); + overlay.push_back(row(tts_pad)); + + return 0; + } + + int32_t step_prompt(int32_t n_batch) override { + GGML_ASSERT(n_batch > 0); + if (prompt_pos >= n_prompt) { + return 0; + } + const int32_t n_tokens_batch = std::min(n_batch, n_prompt - prompt_pos); + llama_batch batch_view = prompt_batch->get_view(prompt_pos, n_tokens_batch); + + const bool is_last_batch = (prompt_pos + n_tokens_batch) == n_prompt; + if (is_last_batch) { + batch_view.logits[n_tokens_batch - 1] = 1; + } + + if (llama_decode(lctx, batch_view) != 0) { + LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); + return -1; + } + + pos += n_tokens_batch; + prompt_pos += n_tokens_batch; + + if (prompt_pos >= n_prompt) { + // prompt fully processed, its embedding buffer is no longer needed + prompt_batch.reset(); + prompt_embd_buf.clear(); + return 0; + } + return n_prompt - prompt_pos; + } + + int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out) override { + mtmd_gen_inp inp{}; + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; + inp.code0 = sampled - codec_0; + inp.embd = const_cast(h_state_in); + inp.top_k = top_k; + inp.top_p = top_p; + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) { + LOG_ERR("mtmd_helper_gen_audio: gen_code process failed\n"); + return 1; + } + + codes_buf.insert(codes_buf.end(), out.codes, out.codes + out.n_codes); + if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= window_frames) { + if (!flush_gen_wav()) { + return 1; + } + } + + std::vector fb(out.embd, out.embd + n_embd); + const auto & ov = overlay[std::min(overlay_idx, overlay.size() - 1)]; + for (int i = 0; i < n_embd; i++) fb[(size_t) i] += ov[(size_t) i]; + overlay_idx++; + + const int n_pos_per_embd = mrope ? 4 : 1; + decode_embd_batch batch_embd(fb.data(), 1, n_pos_per_embd, n_embd); + if (mrope) batch_embd.set_position_mrope_1d(pos, seq_id); + else batch_embd.set_position_normal (pos, seq_id); + batch_embd.batch.logits[0] = 1; + pos++; + + if (llama_decode(lctx, batch_embd.batch) != 0) { + LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); + return 1; + } + + const float * he = llama_get_embeddings_ith(lctx, -1); + h_state_buf.assign(he, he + n_embd); + *h_state_out = h_state_buf.data(); + + return 0; + } + + int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { + if (!flush_gen_wav()) { + return 1; + } + + *out_sample_rate = info.sample_rate; + if (out_n_samples) { + *out_n_samples = (int64_t) audio_pcm.size(); + } + + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) audio_pcm.data(); + *out_data_len = audio_pcm.size() * sizeof(float); + return 0; + } + + out_buf.clear(); + if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { + LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); + return 1; + } + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; + } + +private: + bool ensure_cache() { + if (specials_ok) { + return true; + } + codec_0 = find_special_token(vocab, "<|codec_0|>"); + codec_bos = find_special_token(vocab, "<|codec_bos|>"); + codec_eos = find_special_token(vocab, "<|codec_eos_token|>"); + codec_pad = find_special_token(vocab, "<|codec_pad|>"); + c_think = find_special_token(vocab, "<|codec_think|>"); + c_think_b = find_special_token(vocab, "<|codec_think_bos|>"); + c_think_e = find_special_token(vocab, "<|codec_think_eos|>"); + tts_pad = find_special_token(vocab, ""); + tts_bos = find_special_token(vocab, ""); + tts_eos = find_special_token(vocab, ""); + for (llama_token t : { codec_0, codec_bos, codec_eos, codec_pad, + c_think, c_think_b, c_think_e, + tts_pad, tts_bos, tts_eos }) { + if (t == LLAMA_TOKEN_NULL) { + LOG_ERR("mtmd_helper_gen_audio: missing a required special token in vocab\n"); + return false; + } + } + const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr); + if (n_tok_embd == 0) { + LOG_ERR("mtmd_helper_gen_audio: model has no token embeddings\n"); + return false; + } + tok_embd.resize(n_tok_embd); + if (llama_model_get_tok_embd(model, tok_embd.data()) != n_tok_embd) { + LOG_ERR("mtmd_helper_gen_audio: token embedding copy failed\n"); + return false; + } + specials_ok = true; + return true; + } + + // runs the reference wav through the speaker encoder, returns one x-vector embedding row + bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { + if (!mtmd_support_audio(mctx)) { + LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); + return false; + } + const std::string marker = mtmd_default_marker(); + mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; + mtmd_input_chunks * chunks = mtmd_input_chunks_init(); + const mtmd_bitmap * bptr = bitmap; + bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0; + if (ok) { + ok = false; + for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) { + const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i); + if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) { + continue; + } + if (mtmd_encode_chunk(mctx, chunk) != 0) { + LOG_ERR("mtmd_helper_gen_audio: speaker encode failed\n"); + break; + } + const float * embd = mtmd_get_output_embd(mctx); + const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk); + out.assign(embd, embd + n); + ok = true; + break; + } + } + mtmd_input_chunks_free(chunks); + return ok; + } + + // one GEN_WAV process() call over the buffered codes, state is carried across batches + bool flush_gen_wav() { + if (codes_buf.empty()) { + return true; + } + mtmd_gen_inp inp{}; + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; + inp.codes = codes_buf.data(); + inp.n_codes = codes_buf.size(); + inp.state_data = c2w_state.empty() ? nullptr : (const char *) c2w_state.data(); + inp.state_size = c2w_state.size(); + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) { + LOG_ERR("mtmd_helper_gen_audio: gen_wav process failed\n"); + return false; + } + audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples); + c2w_state.assign(out.state_data, out.state_data + out.state_size); + codes_buf.clear(); + return true; + } + + // vocab specials fixed across the whole session, looked up once + bool specials_ok = false; + llama_token codec_0 = LLAMA_TOKEN_NULL; + llama_token codec_bos = LLAMA_TOKEN_NULL; + llama_token codec_eos = LLAMA_TOKEN_NULL; + llama_token codec_pad = LLAMA_TOKEN_NULL; + llama_token c_think = LLAMA_TOKEN_NULL; + llama_token c_think_b = LLAMA_TOKEN_NULL; + llama_token c_think_e = LLAMA_TOKEN_NULL; + llama_token tts_pad = LLAMA_TOKEN_NULL; + llama_token tts_bos = LLAMA_TOKEN_NULL; + llama_token tts_eos = LLAMA_TOKEN_NULL; + std::vector tok_embd; // whole token embedding matrix, n_vocab * n_embd + + // must match hparams.wav_tfm_swa hardcoded in clip.cpp + size_t window_frames = 72; + + // per-generation state, cleared by reset() + llama_seq_id seq_id = 0; + bool mrope = false; + int pos = 0; + // prompt decode state, consumed batch-by-batch by step_prompt() + std::vector prompt_embd_buf; + std::unique_ptr prompt_batch; + int n_prompt = 0; + int prompt_pos = 0; + int32_t top_k = 50; + float top_p = 1.0f; + std::vector codes_buf; + std::vector c2w_state; + std::vector audio_pcm; + std::vector> overlay; + size_t overlay_idx = 0; + std::vector h_state_buf; + mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + std::vector out_buf; +}; + +static std::unique_ptr make_pipeline(llama_context * lctx, mtmd_context * mctx) { + switch (mtmd_gen_audio_get_info(mctx).type) { + case MTMD_GEN_AUDIO_TYPE_QWEN3TTS: + return std::unique_ptr(new qwen3tts_gen_audio_pipeline(lctx, mctx)); + default: + return nullptr; + } +} + +struct mtmd_helper_gen_audio { + std::unique_ptr pipeline; +}; + +mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(struct llama_context * lctx, struct mtmd_context * mctx) { + auto * ctx = new mtmd_helper_gen_audio(); + ctx->pipeline = make_pipeline(lctx, mctx); + return ctx; +} + +void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx) { + delete ctx; +} + +void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) { + if (ctx->pipeline) { + ctx->pipeline->reset(); + } +} + +int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) { + if (!ctx->pipeline) { + LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n"); + return 1; + } + return ctx->pipeline->set_input(inp); +} + +int32_t mtmd_helper_gen_audio_step_prompt(mtmd_helper_gen_audio * ctx, int32_t n_batch) { + if (!ctx->pipeline) { + return -1; + } + return ctx->pipeline->step_prompt(n_batch); +} + +int32_t mtmd_helper_gen_audio_step_gen(mtmd_helper_gen_audio * ctx, llama_token sampled, + const float * h_state_in, const float ** h_state_out) { + if (!ctx->pipeline) { + return 1; + } + return ctx->pipeline->step_gen(sampled, h_state_in, h_state_out); +} + +int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * out_sample_rate, + const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) { + if (!ctx->pipeline) { + return 1; + } + return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples); +} diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index 90451d02ebd8..d77c93966471 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -9,6 +9,7 @@ #include "mtmd.h" #include "mtmd-helper.h" +#include "mtmd-helper-common.h" #include "llama.h" #include @@ -45,50 +46,6 @@ // internal logging functions // -struct mtmd_helper_logger { - ggml_log_callback default_callback = [](ggml_log_level level, const char * text, void * user_data) { - (void) level; - (void) user_data; - fputs(text, stderr); - fflush(stderr); - }; - - ggml_log_callback log_callback = default_callback; - void * log_callback_user_data; - - void log_v(enum ggml_log_level level, const char * format, va_list args) { - if (format == NULL) { - return; - } - va_list args_copy; - va_copy(args_copy, args); - char buffer[128]; - int len = vsnprintf(buffer, 128, format, args); - if (len < 128) { - log_callback(level, buffer, log_callback_user_data); - } else { - char * buffer2 = (char *) calloc(len + 1, sizeof(char)); - vsnprintf(buffer2, len + 1, format, args_copy); - buffer2[len] = 0; - log_callback(level, buffer2, log_callback_user_data); - free(buffer2); - } - va_end(args_copy); - } - - void log(enum ggml_log_level level, const char * format, ...) { - va_list args; - va_start(args, format); - log_v(level, format, args); - va_end(args); - } -} g_logger; - -#define LOG_DBG(...) g_logger.log(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__) -#define LOG_INF(...) g_logger.log(GGML_LOG_LEVEL_INFO, __VA_ARGS__) -#define LOG_WRN(...) g_logger.log(GGML_LOG_LEVEL_WARN, __VA_ARGS__) -#define LOG_ERR(...) g_logger.log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) - void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data) { if (log_callback == nullptr) { log_callback = g_logger.default_callback; @@ -127,117 +84,6 @@ void mtmd_helper_image_get_decoder_pos(const mtmd_image_tokens * chunks, llama_p } } -// helper struct to make working with embd batch easier -// note: this will be removed after llama_batch_ext refactoring -struct decode_embd_batch { - int n_pos_per_embd; - int n_mmproj_embd; - std::vector pos; - std::vector pos_view; // used by mrope - std::vector n_seq_id; - std::vector seq_id_0; - std::vector seq_ids; - std::vector logits; - llama_batch batch; - decode_embd_batch(float * embd, int32_t n_tokens, int n_pos_per_embd, int n_mmproj_embd) : n_pos_per_embd(n_pos_per_embd), n_mmproj_embd(n_mmproj_embd) { - GGML_ASSERT(n_tokens > 0 && n_pos_per_embd > 0 && n_mmproj_embd > 0); - pos .resize(n_tokens * n_pos_per_embd); - n_seq_id.resize(n_tokens); - seq_ids .resize(n_tokens + 1); - logits .resize(n_tokens); - seq_id_0.resize(1); - seq_ids [n_tokens] = nullptr; - batch = { - /*n_tokens =*/ n_tokens, - /*tokens =*/ nullptr, - /*embd =*/ embd, - /*pos =*/ pos.data(), - /*n_seq_id =*/ n_seq_id.data(), - /*seq_id =*/ seq_ids.data(), - /*logits =*/ logits.data(), - }; - } - - void set_position_normal(llama_pos pos_0, llama_seq_id seq_id) { - seq_id_0[0] = seq_id; - for (int i = 0; i < batch.n_tokens; i++) { - batch.pos [i] = pos_0 + i; - batch.n_seq_id[i] = 1; - batch.seq_id [i] = seq_id_0.data(); - batch.logits [i] = false; - } - } - - // M-RoPE for image - void set_position_mrope_2d(const std::vector & rel_pos, llama_seq_id seq_id) { - GGML_ASSERT(n_pos_per_embd == 4); - GGML_ASSERT(!rel_pos.empty() && (int32_t)rel_pos.size() == batch.n_tokens); - seq_id_0[0] = seq_id; - for (int32_t i = 0; i < batch.n_tokens; i++) { - pos[i ] = rel_pos[i].t; - pos[i + batch.n_tokens ] = rel_pos[i].y; - pos[i + batch.n_tokens * 2] = rel_pos[i].x; - pos[i + batch.n_tokens * 3] = rel_pos[i].z; - } - for (int i = 0; i < batch.n_tokens; i++) { - batch.n_seq_id[i] = 1; - batch.seq_id [i] = seq_id_0.data(); - batch.logits [i] = false; - } - } - - // M-RoPE for audio - void set_position_mrope_1d(llama_pos pos_0, llama_seq_id seq_id) { - GGML_ASSERT(n_pos_per_embd == 4); - seq_id_0[0] = seq_id; - for (int i = 0; i < batch.n_tokens; i++) { - pos[i ] = pos_0 + i; - pos[i + batch.n_tokens ] = pos_0 + i; - pos[i + batch.n_tokens * 2] = pos_0 + i; - pos[i + batch.n_tokens * 3] = pos_0 + i; - } - for (int i = 0; i < batch.n_tokens; i++) { - batch.n_seq_id[i] = 1; - batch.seq_id [i] = seq_id_0.data(); - batch.logits [i] = false; - } - } - - llama_batch get_view(int offset, int n_tokens) { - GGML_ASSERT(offset >= 0 && n_tokens > 0 && offset + n_tokens <= batch.n_tokens); - llama_pos * pos_ptr; - pos_view.clear(); - pos_view.reserve(n_tokens * n_pos_per_embd); - if (n_pos_per_embd > 1) { - // mrope - // for example, with layout of src: 1234...1234...1234...1234... - // offset 2 will give us dst: 34...34...34...34... - for (int i = 0; i < n_pos_per_embd; i++) { - // assume n_tokens is less than or equal to batch.n_tokens - // batch.n_tokens is number of **total** tokens - // n_tokens is number of viewed token - size_t src_idx = i * batch.n_tokens + offset; - pos_view.insert(pos_view.end(), - pos.data() + src_idx, - pos.data() + src_idx + n_tokens); - } - pos_ptr = pos_view.data(); - } else { - // normal - pos_ptr = pos.data() + offset; - } - return { - /*n_tokens =*/ n_tokens, - /*tokens =*/ nullptr, - /*embd =*/ batch.embd + offset * n_mmproj_embd, - /*pos =*/ pos_ptr, - /*n_seq_id =*/ batch.n_seq_id + offset, - /*seq_id =*/ batch.seq_id + offset, - /*logits =*/ batch.logits + offset, - }; - } -}; - // Helper class to set non-causal attention via RAII class scope_non_causal { public: @@ -1084,3 +930,18 @@ int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, GGML_ASSERT(false && "video is not supported in this build (MTMD_VIDEO is set to OFF)"); #endif } + +bool mtmd_helper_model_can_chat(llama_context * lctx, mtmd_context * mctx) { + if (!mctx) { + return true; + } + + auto * model = llama_get_model(lctx); + auto * tmpl = llama_model_chat_template(model, nullptr); + auto info = mtmd_gen_audio_get_info(mctx); + + // tts-only model cannot be used for chat (no chat template) + bool is_tts_only = info.type != MTMD_GEN_AUDIO_TYPE_NONE && tmpl == nullptr; + + return !is_tts_only; +} diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 680a2317df0e..7e5cf9b5098c 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -157,6 +157,73 @@ MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, mtmd_bitmap ** out_bitmap, char ** out_text); +// return true if model can be used for chat +MTMD_API bool mtmd_helper_model_can_chat(struct llama_context * lctx, struct mtmd_context * mctx); + +// +// Audio generation helpers +// (early-stage experimental, subjected to breaking changes) +// + +// audio generation helper context +// contains accumulator for generated audio features and PCM audio +struct mtmd_helper_gen_audio; +typedef struct mtmd_helper_gen_audio mtmd_helper_gen_audio; + +enum mtmd_helper_gen_audio_outtype { + MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM, // raw PCM + MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono +}; +struct mtmd_helper_gen_audio_inp { + llama_seq_id seq_id; + + const char * prompt; + size_t prompt_len; + + mtmd_bitmap * speaker_ref; // optional, can be NULL + const char * lang; // optional, can be NULL + + int32_t top_k; + float top_p; + + enum mtmd_helper_gen_audio_outtype out_type; +}; + +MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init( + struct llama_context * lctx, + struct mtmd_context * mctx); + +MTMD_API void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx); + +MTMD_API void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx); + +MTMD_API int32_t mtmd_helper_gen_audio_set_input( + mtmd_helper_gen_audio * ctx, + const struct mtmd_helper_gen_audio_inp * inp); + +// processes at most n_batch prompt tokens per call +// returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error +MTMD_API int32_t mtmd_helper_gen_audio_step_prompt( + mtmd_helper_gen_audio * ctx, + int32_t n_batch); + +// generates one frame; must only be called after step_prompt() has returned 0 +// h_state_out is valid until next step_gen() or reset() call +MTMD_API int32_t mtmd_helper_gen_audio_step_gen( + mtmd_helper_gen_audio * ctx, + llama_token sampled, + const float * h_state_in, + const float ** h_state_out); + +// out_data valid until next get_output() or reset() call +// out_n_samples (optional, can be NULL) receives the number of generated PCM samples +MTMD_API int32_t mtmd_helper_gen_audio_get_output( + mtmd_helper_gen_audio * ctx, + int32_t * out_sample_rate, + const char ** out_data, + size_t * out_data_len, + int64_t * out_n_samples); + #ifdef __cplusplus } // extern "C" #endif @@ -177,6 +244,31 @@ struct mtmd_helper_video_deleter { }; using video_ptr = std::unique_ptr; +// audio generation-related C++ wrappers +struct mtmd_helper_gen_audio_deleter { + void operator()(mtmd_helper_gen_audio * val) { mtmd_helper_gen_audio_free(val); } +}; +using gen_audio_ptr = std::unique_ptr; +struct gen_audio { + gen_audio_ptr ctx; + gen_audio(struct llama_context * lctx, struct mtmd_context * mctx) : ctx(mtmd_helper_gen_audio_init(lctx, mctx)) {} + void reset() { + mtmd_helper_gen_audio_reset(ctx.get()); + } + int32_t set_input(const struct mtmd_helper_gen_audio_inp * inp) { + return mtmd_helper_gen_audio_set_input(ctx.get(), inp); + } + int32_t step_prompt(int32_t n_batch) { + return mtmd_helper_gen_audio_step_prompt(ctx.get(), n_batch); + } + int32_t step_gen(llama_token sampled, const float * h_state, const float ** h_state_out) { + return mtmd_helper_gen_audio_step_gen(ctx.get(), sampled, h_state, h_state_out); + } + int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) { + return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples); + } +}; + } // namespace mtmd_helper #endif diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index d3899f5c853d..ff90d6818cb5 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -262,6 +262,13 @@ struct mtmd_context { struct clip_ctx * ctx_a; // audio std::vector out_embd; // image embedding vector + // generation context + struct clip_ctx * ctx_gen_a; // audio + std::vector gen_out_codes; // this frame's 16 sampled codes (GEN_CODE) + std::vector gen_out_embd; // next-step hidden state fed back to backbone (GEN_CODE) + std::vector gen_out_audio; // decoded PCM samples for the current frame (GEN_WAV) + std::vector gen_out_state; // state to feed into the next GEN_WAV call + bool print_timings; int n_threads; std::string media_marker; @@ -354,6 +361,7 @@ struct mtmd_context { auto res = clip_init(mmproj_fname, ctx_clip_params); ctx_v = res.ctx_v; ctx_a = res.ctx_a; + ctx_gen_a = res.ctx_gen_a; if (!ctx_v && !ctx_a) { throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname)); } @@ -378,6 +386,15 @@ struct mtmd_context { "hint: you may be using wrong mmproj\n", n_embd_text, n_embd_clip)); } + if (ctx_gen_a) { + int n_embd_gen = clip_n_mmproj_embd(ctx_gen_a); + if (n_embd_text > 0 && n_embd_text != n_embd_gen) { + throw std::runtime_error(string_format( + "mismatch between text model (n_embd = %d) and gen-audio mmproj (n_embd = %d)\n" + "hint: you may be using wrong mmproj\n", + n_embd_text, n_embd_gen)); + } + } if (ctx_v) { init_vision(); } @@ -740,6 +757,10 @@ struct mtmd_context { aud_end = "<|mimo_audio_end|>"; audio_preproc = std::make_unique(ctx_a); } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + audio_preproc = std::make_unique(ctx_a); + } break; default: throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj)); } @@ -780,6 +801,7 @@ struct mtmd_context { ~mtmd_context() { clip_free(ctx_a); clip_free(ctx_v); + clip_free(ctx_gen_a); } private: @@ -1553,6 +1575,125 @@ float * mtmd_get_output_embd(mtmd_context * ctx) { return ctx->out_embd.data(); } +// +// audio generation +// + +mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { + mtmd_gen_audio_info info; + if (!ctx->ctx_gen_a) { + info.type = MTMD_GEN_AUDIO_TYPE_NONE; + return info; + } + switch (clip_get_projector_type(ctx->ctx_gen_a)) { + case PROJECTOR_TYPE_QWEN3TTS_GEN: + info.type = MTMD_GEN_AUDIO_TYPE_QWEN3TTS; + info.sample_rate = 24000; + break; + default: + info.type = MTMD_GEN_AUDIO_TYPE_NONE; + break; + } + return info; +} + +static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_inp * inp, mtmd_gen_out * out) { + clip_ctx * ctx_clip = ctx->ctx_gen_a; + if (!ctx_clip) { + LOG_ERR("%s: model does not support audio generation\n", __func__); + return 1; + } + + if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) { + const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip); + + clip_image_f32 hidden_state; + hidden_state.set_size({(int) n_embd, 1}, false, true); + hidden_state.cpy_buf(std::vector(inp->embd, inp->embd + n_embd)); + + clip_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(hidden_state)); + + std::vector out_embd(n_embd); + std::vector out_codes; + + clip_encode_params params; + params.imgs = &batch; + params.n_threads = ctx->n_threads; + params.gen_process = CLIP_GEN_PROCESS_GEN_CODE; + params.out_embd = &out_embd; + params.out_codes = &out_codes; + params.code0 = inp->code0; + params.top_k = inp->top_k; + params.top_p = inp->top_p; + + if (!clip_encode(ctx_clip, ¶ms)) { + LOG_ERR("%s: clip_encode failed (gen_code)\n", __func__); + return 1; + } + + ctx->gen_out_embd = std::move(out_embd); + ctx->gen_out_codes = std::move(out_codes); + + out->embd = ctx->gen_out_embd.data(); + out->codes = ctx->gen_out_codes.data(); + out->n_codes = ctx->gen_out_codes.size(); + return 0; + } + + // MTMD_GEN_PROCESS_TYPE_GEN_WAV + if (!inp->codes || inp->n_codes == 0) { + LOG_ERR("%s: codes required for gen_wav\n", __func__); + return 1; + } + std::vector in_codes(inp->codes, inp->codes + inp->n_codes); + std::vector in_state; + if (inp->state_data) { + in_state.assign(inp->state_data, inp->state_data + inp->state_size); + } + + // gen_wav has no hidden-state input, the batch entry is an unused placeholder + // TODO @ngxson : some models in the future may require hidden-state input, need to update this code later + clip_image_f32 dummy; + dummy.set_size({1, 1}, false, true); + dummy.cpy_buf(std::vector(1, 0.0f)); + + clip_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(dummy)); + + clip_encode_params params; + params.imgs = &batch; + params.n_threads = ctx->n_threads; + params.gen_process = CLIP_GEN_PROCESS_GEN_WAV; + params.codes = &in_codes; + params.out_audio = &ctx->gen_out_audio; + params.state_in = inp->state_data ? &in_state : nullptr; + params.state_out = &ctx->gen_out_state; + + if (!clip_encode(ctx_clip, ¶ms)) { + LOG_ERR("%s: clip_encode failed (code2wav)\n", __func__); + return 1; + } + + out->audio = ctx->gen_out_audio.data(); + out->n_samples = ctx->gen_out_audio.size(); + out->state_data = (const char *) ctx->gen_out_state.data(); + out->state_size = ctx->gen_out_state.size(); + + return 0; +} + +int32_t mtmd_gen_audio_process(mtmd_context * ctx, const struct mtmd_gen_inp * inp, struct mtmd_gen_out * out) { + try { + return mtmd_gen_audio_process_impl(ctx, inp, out); + } catch (const std::exception & e) { + LOG_ERR("%s: error: %s\n", __func__, e.what()); + return 1; + } +} + mtmd_batch * mtmd_batch_init(mtmd_context * ctx) { return new mtmd_batch(ctx); } diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 3b8c1200b566..84651f8dcd00 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -327,6 +327,60 @@ struct mtmd_caps { }; MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname); +///////////////////////////////////////// +// EXPERIMENTAL API for audio generation, subjected to breaking changes + +// represent the pipeline type +enum mtmd_gen_audio_type { + MTMD_GEN_AUDIO_TYPE_NONE, // not supported + MTMD_GEN_AUDIO_TYPE_QWEN3TTS, +}; +struct mtmd_gen_audio_info { + enum mtmd_gen_audio_type type; + int32_t sample_rate; // in Hz, for example 24000 for qwen3tts +}; +MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); + +enum mtmd_gen_process_type { + MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.) + MTMD_GEN_PROCESS_TYPE_GEN_WAV, // convert semantic to PCM audio + // for qwen3tts, this is code2wav +}; +struct mtmd_gen_inp { + enum mtmd_gen_process_type type; + + // for MTMD_GEN_PROCESS_TYPE_GEN_CODE + int32_t code0; // the sampled codebook 0 entry from backbone + float * embd; // the hidden state from backbone, must have n_text_embd elements + int32_t top_k; + float top_p; + + // for MTMD_GEN_PROCESS_TYPE_GEN_WAV + int32_t * codes; + size_t n_codes; + const char * state_data; + size_t state_size; +}; +struct mtmd_gen_out { + // note: output memory is allocated by the context, valid until next process() call + + // for MTMD_GEN_PROCESS_TYPE_GEN_CODE + const int32_t * codes; + size_t n_codes; + const float * embd; // the generated hidden state, to be fed back to backbone + // it must have n_text_embd elements + + // for MTMD_GEN_PROCESS_TYPE_GEN_WAV + const float * audio; + size_t n_samples; + const char * state_data; + size_t state_size; +}; +// note: this API is stateless, caller must handle state management and audio frame accumulation +MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, + const struct mtmd_gen_inp * inp, + struct mtmd_gen_out * out); + ///////////////////////////////////////// // test function, to be used in test-mtmd-c-api.c diff --git a/tools/tts/CMakeLists.txt b/tools/tts/CMakeLists.txt index 26a8bb8f2d1f..0a0b5730eaa6 100644 --- a/tools/tts/CMakeLists.txt +++ b/tools/tts/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET llama-tts) add_executable(${TARGET} tts.cpp) -target_link_libraries(${TARGET} PRIVATE llama llama-common ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${TARGET} PRIVATE llama llama-common mtmd ${CMAKE_THREAD_LIBS_INIT}) target_compile_features(${TARGET} PRIVATE cxx_std_17) if(LLAMA_TOOLS_INSTALL) diff --git a/tools/tts/README.md b/tools/tts/README.md index 4749bb9f5a72..dd84336c3990 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -1,117 +1,34 @@ -# llama.cpp/example/tts -This example demonstrates the Text To Speech feature. It uses a -[model](https://www.outeai.com/blog/outetts-0.2-500m) from -[outeai](https://www.outeai.com/). +# llama.cpp TTS -## Quickstart -If you have built llama.cpp with SSL support you can simply run the -following command and the required models will be downloaded automatically: -```console -$ build/bin/llama-tts --tts-oute-default -p "Hello world" && aplay output.wav -``` -For details about the models and how to convert them to the required format -see the following sections. +This is a tool to demonstrate audio generation capability in llama.cpp via `libmtmd`. It was added via PR [#26254](https://github.com/ggml-org/llama.cpp/pull/26254) -### Model conversion -Checkout or download the model that contains the LLM model: -```console -$ pushd models -$ git clone --branch main --single-branch --depth 1 https://huggingface.co/OuteAI/OuteTTS-0.2-500M -$ cd OuteTTS-0.2-500M && git lfs install && git lfs pull -$ popd -``` -Convert the model to .gguf format: -```console -(venv) python convert_hf_to_gguf.py models/OuteTTS-0.2-500M \ - --outfile models/outetts-0.2-0.5B-f16.gguf --outtype f16 -``` -The generated model will be `models/outetts-0.2-0.5B-f16.gguf`. +Note: this tool used to serve as a demo for OuteTTS, but it was converted to a more model-agnostic tool. -We can optionally quantize this to Q8_0 using the following command: -```console -$ build/bin/llama-quantize models/outetts-0.2-0.5B-f16.gguf \ - models/outetts-0.2-0.5B-q8_0.gguf q8_0 -``` -The quantized model will be `models/outetts-0.2-0.5B-q8_0.gguf`. - -Next we do something similar for the audio decoder. First download or checkout -the model for the voice decoder: -```console -$ pushd models -$ git clone --branch main --single-branch --depth 1 https://huggingface.co/novateur/WavTokenizer-large-speech-75token -$ cd WavTokenizer-large-speech-75token && git lfs install && git lfs pull -$ popd -``` -This model file is a PyTorch checkpoint (.ckpt) and we first need to convert it to -huggingface format: -```console -(venv) python tools/tts/convert_pt_to_hf.py \ - models/WavTokenizer-large-speech-75token/wavtokenizer_large_speech_320_24k.ckpt -... -Model has been successfully converted and saved to models/WavTokenizer-large-speech-75token/model.safetensors -Metadata has been saved to models/WavTokenizer-large-speech-75token/index.json -Config has been saved to models/WavTokenizer-large-speech-75tokenconfig.json -``` -Then we can convert the huggingface format to gguf: -```console -(venv) python convert_hf_to_gguf.py models/WavTokenizer-large-speech-75token \ - --outfile models/wavtokenizer-large-75-f16.gguf --outtype f16 -... -INFO:hf-to-gguf:Model successfully exported to models/wavtokenizer-large-75-f16.gguf -``` +## Common usage -### Running the example +Simple usage: -With both of the models generated, the LLM model and the voice decoder model, -we can run the example: -```console -$ build/bin/llama-tts -m ./models/outetts-0.2-0.5B-q8_0.gguf \ - -mv ./models/wavtokenizer-large-75-f16.gguf \ - -p "Hello world" -... -main: audio written to file 'output.wav' -``` -The output.wav file will contain the audio of the prompt. This can be heard -by playing the file with a media player. On Linux the following command will -play the audio: -```console -$ aplay output.wav +```sh +llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF -p "Hello world" --output out.wav ``` -### Running the example with llama-server -Running this example with `llama-server` is also possible and requires two -server instances to be started. One will serve the LLM model and the other -will serve the voice decoder model. +Common params: +- Sampling params such as `--top-k`, `--top-p`, `--temp`, etc. +- `-n ` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model +- Core inference params such as `-ngl`, `-b`, `-ub`, etc. -The LLM model server can be started with the following command: -```console -$ ./build/bin/llama-server -m ./models/outetts-0.2-0.5B-q8_0.gguf --port 8020 -``` +## Qwen3-TTS -And the voice decoder model server can be started using: -```console -./build/bin/llama-server -m ./models/wavtokenizer-large-75-f16.gguf --port 8021 --embeddings --pooling none -``` +Available params: +- `--tts-lang` can be `zh`, `en`, `de`, `it`, `pt`, `es`, `ja`, `ko`, `fr`, `ru` (default: `en`) +- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3) -Then we can run [tts-outetts.py](tts-outetts.py) to generate the audio. +Example usage: -First create a virtual environment for python and install the required -dependencies (this in only required to be done once): -```console -$ python3 -m venv venv -$ source venv/bin/activate -(venv) pip install requests numpy -``` - -And then run the python script using: -```conole -(venv) python ./tools/tts/tts-outetts.py http://localhost:8020 http://localhost:8021 "Hello world" -spectrogram generated: n_codes: 90, n_embd: 1282 -converting to audio ... -audio generated: 28800 samples -audio written to file "output.wav" -``` -And to play the audio we can again use aplay or any other media player: -```console -$ aplay output.wav +```sh +llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \ + -p "Hello world" \ + --tts-lang english \ + --tts-speaker-file speaker.mp3 \ + --output out.wav ``` diff --git a/tools/tts/convert_pt_to_hf.py b/tools/tts/convert_pt_to_hf.py deleted file mode 100644 index ebd55d9657b2..000000000000 --- a/tools/tts/convert_pt_to_hf.py +++ /dev/null @@ -1,180 +0,0 @@ -# convert the https://huggingface.co/novateur/WavTokenizer-large-speech-75token to HF format -# the goal is to be able to reuse the convert_hf_to_gguf.py after that to create a GGUF file with the WavTokenizer decoder -# -# TODO: this script is LLM-generated and probably very inefficient and should be rewritten - -import torch -import json -import os -import sys -import re - -from safetensors.torch import save_file - -# default -model_path = './model.pt' - -# read from CLI -if len(sys.argv) > 1: - model_path = sys.argv[1] - -# get the directory of the input model -path_dst = os.path.dirname(model_path) - -print(f"Loading model from {model_path}") - -model = torch.load(model_path, map_location='cpu') - -#print(model) - -# print all keys -for key in model.keys(): - print(key) - if key == 'hyper_parameters': - #print(model[key]) - # dump as json pretty - print(json.dumps(model[key], indent=4)) - #if key != 'state_dict' and key != 'optimizer_states': - # print(model[key]) - -# Check if the loaded model is a state_dict or a model instance -if isinstance(model, torch.nn.Module): - state_dict = model.state_dict() -else: - state_dict = model - -# Print the structure of the state_dict to understand its format -print("State dictionary keys:") -for key in state_dict.keys(): - print(key) - -# Ensure the state_dict is flat and contains only torch.Tensor objects -def flatten_state_dict(state_dict, parent_key='', sep='.'): - items = [] - items_new = [] - - for k, v in state_dict.items(): - new_key = f"{parent_key}{sep}{k}" if parent_key else k - if isinstance(v, torch.Tensor): - items.append((new_key, v)) - elif isinstance(v, dict): - items.extend(flatten_state_dict(v, new_key, sep=sep).items()) - return dict(items) - - size_total_mb = 0 - - for key, value in list(items): - # keep only what we need for inference - if not key.startswith('state_dict.feature_extractor.encodec.quantizer.') and \ - not key.startswith('state_dict.backbone.') and \ - not key.startswith('state_dict.head.out'): - print('Skipping key: ', key) - continue - - new_key = key - - new_key = new_key.replace('state_dict.', '') - new_key = new_key.replace('pos_net', 'posnet') - - # check if matches "backbone.posnet.%d.bias" or "backbone.posnet.%d.weight" - if new_key.startswith("backbone.posnet."): - match = re.match(r"backbone\.posnet\.(\d+)\.(bias|weight)", new_key) - if match: - new_key = f"backbone.posnet.{match.group(1)}.norm.{match.group(2)}" - - # "feature_extractor.encodec.quantizer.vq.layers.0._codebook.embed" -> "backbone.embedding.weight" - if new_key == "feature_extractor.encodec.quantizer.vq.layers.0._codebook.embed": - new_key = "backbone.embedding.weight" - - # these are the only rows used - # ref: https://github.com/edwko/OuteTTS/blob/a613e79c489d8256dd657ea9168d78de75895d82/outetts/wav_tokenizer/audio_codec.py#L100 - if new_key.endswith("norm.scale.weight"): - new_key = new_key.replace("norm.scale.weight", "norm.weight") - value = value[0] - - if new_key.endswith("norm.shift.weight"): - new_key = new_key.replace("norm.shift.weight", "norm.bias") - value = value[0] - - if new_key.endswith("gamma"): - new_key = new_key.replace("gamma", "gamma.weight") - - # convert from 1D [768] to 2D [768, 1] so that ggml_add can broadcast the bias - if (new_key.endswith("norm.weight") or new_key.endswith("norm1.weight") or new_key.endswith("norm2.weight") or new_key.endswith(".bias")) and (new_key.startswith("backbone.posnet") or new_key.startswith("backbone.embed.bias")): - value = value.unsqueeze(1) - - if new_key.endswith("dwconv.bias"): - value = value.unsqueeze(1) - - size_mb = value.element_size() * value.nelement() / (1024 * 1024) - print(f"{size_mb:8.2f} MB - {new_key}: {value.shape}") - - size_total_mb += size_mb - - #print(key, '->', new_key, ': ', value) - #print(key, '->', new_key) - - items_new.append((new_key, value)) - - print(f"Total size: {size_total_mb:8.2f} MB") - - return dict(items_new) - -flattened_state_dict = flatten_state_dict(state_dict) - - -# Convert the model to the safetensors format -output_path = path_dst + '/model.safetensors' -save_file(flattened_state_dict, output_path) - -print(f"Model has been successfully converted and saved to {output_path}") - -# Calculate the total size of the .safetensors file -total_size = os.path.getsize(output_path) - -# Create the weight map -weight_map = { - "model.safetensors": ["*"] # Assuming all weights are in one file -} - -# Create metadata for the index.json file -metadata = { - "total_size": total_size, - "weight_map": weight_map -} - -# Save the metadata to index.json -index_path = path_dst + '/index.json' -with open(index_path, 'w') as f: - json.dump(metadata, f, indent=4) - -print(f"Metadata has been saved to {index_path}") - -config = { - "architectures": [ - "WavTokenizerDec" - ], - "hidden_size": 1282, - "n_embd_features": 512, - "n_ff": 2304, - "vocab_size": 4096, - "n_head": 1, - "layer_norm_epsilon": 1e-6, - "group_norm_epsilon": 1e-6, - "group_norm_groups": 32, - "max_position_embeddings": 8192, # ? - "n_layer": 12, - "posnet": { - "n_embd": 768, - "n_layer": 6 - }, - "convnext": { - "n_embd": 768, - "n_layer": 12 - }, -} - -with open(path_dst + '/config.json', 'w') as f: - json.dump(config, f, indent=4) - -print(f"Config has been saved to {path_dst + 'config.json'}") diff --git a/tools/tts/tts-outetts.py b/tools/tts/tts-outetts.py deleted file mode 100644 index 3791f9fc3ebc..000000000000 --- a/tools/tts/tts-outetts.py +++ /dev/null @@ -1,299 +0,0 @@ -import sys -#import json -#import struct -import requests -import re -import struct -import numpy as np -from concurrent.futures import ThreadPoolExecutor - - -def fill_hann_window(size, periodic=True): - if periodic: - return np.hanning(size + 1)[:-1] - return np.hanning(size) - - -def irfft(n_fft, complex_input): - return np.fft.irfft(complex_input, n=n_fft) - - -def fold(buffer, n_out, n_win, n_hop, n_pad): - result = np.zeros(n_out) - n_frames = len(buffer) // n_win - - for i in range(n_frames): - start = i * n_hop - end = start + n_win - result[start:end] += buffer[i * n_win:(i + 1) * n_win] - - return result[n_pad:-n_pad] if n_pad > 0 else result - - -def process_frame(args): - l, n_fft, ST, hann = args - frame = irfft(n_fft, ST[l]) - frame = frame * hann - hann2 = hann * hann - return frame, hann2 - - -def embd_to_audio(embd, n_codes, n_embd, n_thread=4): - embd = np.asarray(embd, dtype=np.float32).reshape(n_codes, n_embd) - - n_fft = 1280 - n_hop = 320 - n_win = 1280 - n_pad = (n_win - n_hop) // 2 - n_out = (n_codes - 1) * n_hop + n_win - - hann = fill_hann_window(n_fft, True) - - E = np.zeros((n_embd, n_codes), dtype=np.float32) - for l in range(n_codes): - for k in range(n_embd): - E[k, l] = embd[l, k] - - half_embd = n_embd // 2 - S = np.zeros((n_codes, half_embd + 1), dtype=np.complex64) - - for k in range(half_embd): - for l in range(n_codes): - mag = E[k, l] - phi = E[k + half_embd, l] - - mag = np.clip(np.exp(mag), 0, 1e2) - S[l, k] = mag * np.exp(1j * phi) - - res = np.zeros(n_codes * n_fft) - hann2_buffer = np.zeros(n_codes * n_fft) - - with ThreadPoolExecutor(max_workers=n_thread) as executor: - args = [(l, n_fft, S, hann) for l in range(n_codes)] - results = list(executor.map(process_frame, args)) - - for l, (frame, hann2) in enumerate(results): - res[l*n_fft:(l+1)*n_fft] = frame - hann2_buffer[l*n_fft:(l+1)*n_fft] = hann2 - - audio = fold(res, n_out, n_win, n_hop, n_pad) - env = fold(hann2_buffer, n_out, n_win, n_hop, n_pad) - - mask = env > 1e-10 - audio[mask] /= env[mask] - - return audio - - -def save_wav(filename, audio_data, sample_rate): - num_channels = 1 - bits_per_sample = 16 - bytes_per_sample = bits_per_sample // 8 - data_size = len(audio_data) * bytes_per_sample - byte_rate = sample_rate * num_channels * bytes_per_sample - block_align = num_channels * bytes_per_sample - chunk_size = 36 + data_size # 36 = size of header minus first 8 bytes - - header = struct.pack( - '<4sI4s4sIHHIIHH4sI', - b'RIFF', - chunk_size, - b'WAVE', - b'fmt ', - 16, # fmt chunk size - 1, # audio format (PCM) - num_channels, - sample_rate, - byte_rate, - block_align, - bits_per_sample, - b'data', - data_size - ) - - audio_data = np.clip(audio_data * 32767, -32768, 32767) - pcm_data = audio_data.astype(np.int16) - - with open(filename, 'wb') as f: - f.write(header) - f.write(pcm_data.tobytes()) - - -def process_text(text: str): - text = re.sub(r'\d+(\.\d+)?', lambda x: x.group(), text.lower()) # TODO this needs to be fixed - text = re.sub(r'[-_/,\.\\]', ' ', text) - text = re.sub(r'[^a-z\s]', '', text) - text = re.sub(r'\s+', ' ', text).strip() - return text.split() - -# usage: -# python tts-outetts.py http://server-llm:port http://server-dec:port "text" - -if len(sys.argv) <= 3: - print("usage: python tts-outetts.py http://server-llm:port http://server-dec:port \"text\"") - exit(1) - -host_llm = sys.argv[1] -host_dec = sys.argv[2] -text = sys.argv[3] - -prefix = """<|im_start|> -<|text_start|>the<|text_sep|>overall<|text_sep|>package<|text_sep|>from<|text_sep|>just<|text_sep|>two<|text_sep|>people<|text_sep|>is<|text_sep|>pretty<|text_sep|>remarkable<|text_sep|>sure<|text_sep|>i<|text_sep|>have<|text_sep|>some<|text_sep|>critiques<|text_sep|>about<|text_sep|>some<|text_sep|>of<|text_sep|>the<|text_sep|>gameplay<|text_sep|>aspects<|text_sep|>but<|text_sep|>its<|text_sep|>still<|text_sep|>really<|text_sep|>enjoyable<|text_sep|>and<|text_sep|>it<|text_sep|>looks<|text_sep|>lovely<|text_sep|>""" - -words = process_text(text) -words = "<|text_sep|>".join([i.strip() for i in words]) -words += "<|text_end|>\n" - -# voice data -# TODO: load from json -#suffix = """<|audio_start|> -#the<|t_0.08|><|code_start|><|257|><|740|><|636|><|913|><|788|><|1703|><|code_end|> -#overall<|t_0.36|><|code_start|><|127|><|201|><|191|><|774|><|700|><|532|><|1056|><|557|><|798|><|298|><|1741|><|747|><|1662|><|1617|><|1702|><|1527|><|368|><|1588|><|1049|><|1008|><|1625|><|747|><|1576|><|728|><|1019|><|1696|><|1765|><|code_end|> -#package<|t_0.56|><|code_start|><|935|><|584|><|1319|><|627|><|1016|><|1491|><|1344|><|1117|><|1526|><|1040|><|239|><|1435|><|951|><|498|><|723|><|1180|><|535|><|789|><|1649|><|1637|><|78|><|465|><|1668|><|901|><|595|><|1675|><|117|><|1009|><|1667|><|320|><|840|><|79|><|507|><|1762|><|1508|><|1228|><|1768|><|802|><|1450|><|1457|><|232|><|639|><|code_end|> -#from<|t_0.19|><|code_start|><|604|><|782|><|1682|><|872|><|1532|><|1600|><|1036|><|1761|><|647|><|1554|><|1371|><|653|><|1595|><|950|><|code_end|> -#just<|t_0.25|><|code_start|><|1782|><|1670|><|317|><|786|><|1748|><|631|><|599|><|1155|><|1364|><|1524|><|36|><|1591|><|889|><|1535|><|541|><|440|><|1532|><|50|><|870|><|code_end|> -#two<|t_0.24|><|code_start|><|1681|><|1510|><|673|><|799|><|805|><|1342|><|330|><|519|><|62|><|640|><|1138|><|565|><|1552|><|1497|><|1552|><|572|><|1715|><|1732|><|code_end|> -#people<|t_0.39|><|code_start|><|593|><|274|><|136|><|740|><|691|><|633|><|1484|><|1061|><|1138|><|1485|><|344|><|428|><|397|><|1562|><|645|><|917|><|1035|><|1449|><|1669|><|487|><|442|><|1484|><|1329|><|1832|><|1704|><|600|><|761|><|653|><|269|><|code_end|> -#is<|t_0.16|><|code_start|><|566|><|583|><|1755|><|646|><|1337|><|709|><|802|><|1008|><|485|><|1583|><|652|><|10|><|code_end|> -#pretty<|t_0.32|><|code_start|><|1818|><|1747|><|692|><|733|><|1010|><|534|><|406|><|1697|><|1053|><|1521|><|1355|><|1274|><|816|><|1398|><|211|><|1218|><|817|><|1472|><|1703|><|686|><|13|><|822|><|445|><|1068|><|code_end|> -#remarkable<|t_0.68|><|code_start|><|230|><|1048|><|1705|><|355|><|706|><|1149|><|1535|><|1787|><|1356|><|1396|><|835|><|1583|><|486|><|1249|><|286|><|937|><|1076|><|1150|><|614|><|42|><|1058|><|705|><|681|><|798|><|934|><|490|><|514|><|1399|><|572|><|1446|><|1703|><|1346|><|1040|><|1426|><|1304|><|664|><|171|><|1530|><|625|><|64|><|1708|><|1830|><|1030|><|443|><|1509|><|1063|><|1605|><|1785|><|721|><|1440|><|923|><|code_end|> -#sure<|t_0.36|><|code_start|><|792|><|1780|><|923|><|1640|><|265|><|261|><|1525|><|567|><|1491|><|1250|><|1730|><|362|><|919|><|1766|><|543|><|1|><|333|><|113|><|970|><|252|><|1606|><|133|><|302|><|1810|><|1046|><|1190|><|1675|><|code_end|> -#i<|t_0.08|><|code_start|><|123|><|439|><|1074|><|705|><|1799|><|637|><|code_end|> -#have<|t_0.16|><|code_start|><|1509|><|599|><|518|><|1170|><|552|><|1029|><|1267|><|864|><|419|><|143|><|1061|><|0|><|code_end|> -#some<|t_0.16|><|code_start|><|619|><|400|><|1270|><|62|><|1370|><|1832|><|917|><|1661|><|167|><|269|><|1366|><|1508|><|code_end|> -#critiques<|t_0.60|><|code_start|><|559|><|584|><|1163|><|1129|><|1313|><|1728|><|721|><|1146|><|1093|><|577|><|928|><|27|><|630|><|1080|><|1346|><|1337|><|320|><|1382|><|1175|><|1682|><|1556|><|990|><|1683|><|860|><|1721|><|110|><|786|><|376|><|1085|><|756|><|1523|><|234|><|1334|><|1506|><|1578|><|659|><|612|><|1108|><|1466|><|1647|><|308|><|1470|><|746|><|556|><|1061|><|code_end|> -#about<|t_0.29|><|code_start|><|26|><|1649|><|545|><|1367|><|1263|><|1728|><|450|><|859|><|1434|><|497|><|1220|><|1285|><|179|><|755|><|1154|><|779|><|179|><|1229|><|1213|><|922|><|1774|><|1408|><|code_end|> -#some<|t_0.23|><|code_start|><|986|><|28|><|1649|><|778|><|858|><|1519|><|1|><|18|><|26|><|1042|><|1174|><|1309|><|1499|><|1712|><|1692|><|1516|><|1574|><|code_end|> -#of<|t_0.07|><|code_start|><|197|><|716|><|1039|><|1662|><|64|><|code_end|> -#the<|t_0.08|><|code_start|><|1811|><|1568|><|569|><|886|><|1025|><|1374|><|code_end|> -#gameplay<|t_0.48|><|code_start|><|1269|><|1092|><|933|><|1362|><|1762|><|1700|><|1675|><|215|><|781|><|1086|><|461|><|838|><|1022|><|759|><|649|><|1416|><|1004|><|551|><|909|><|787|><|343|><|830|><|1391|><|1040|><|1622|><|1779|><|1360|><|1231|><|1187|><|1317|><|76|><|997|><|989|><|978|><|737|><|189|><|code_end|> -#aspects<|t_0.56|><|code_start|><|1423|><|797|><|1316|><|1222|><|147|><|719|><|1347|><|386|><|1390|><|1558|><|154|><|440|><|634|><|592|><|1097|><|1718|><|712|><|763|><|1118|><|1721|><|1311|><|868|><|580|><|362|><|1435|><|868|><|247|><|221|><|886|><|1145|><|1274|><|1284|><|457|><|1043|><|1459|><|1818|><|62|><|599|><|1035|><|62|><|1649|><|778|><|code_end|> -#but<|t_0.20|><|code_start|><|780|><|1825|><|1681|><|1007|><|861|><|710|><|702|><|939|><|1669|><|1491|><|613|><|1739|><|823|><|1469|><|648|><|code_end|> -#its<|t_0.09|><|code_start|><|92|><|688|><|1623|><|962|><|1670|><|527|><|599|><|code_end|> -#still<|t_0.27|><|code_start|><|636|><|10|><|1217|><|344|><|713|><|957|><|823|><|154|><|1649|><|1286|><|508|><|214|><|1760|><|1250|><|456|><|1352|><|1368|><|921|><|615|><|5|><|code_end|> -#really<|t_0.36|><|code_start|><|55|><|420|><|1008|><|1659|><|27|><|644|><|1266|><|617|><|761|><|1712|><|109|><|1465|><|1587|><|503|><|1541|><|619|><|197|><|1019|><|817|><|269|><|377|><|362|><|1381|><|507|><|1488|><|4|><|1695|><|code_end|> -#enjoyable<|t_0.49|><|code_start|><|678|><|501|><|864|><|319|><|288|><|1472|><|1341|><|686|><|562|><|1463|><|619|><|1563|><|471|><|911|><|730|><|1811|><|1006|><|520|><|861|><|1274|><|125|><|1431|><|638|><|621|><|153|><|876|><|1770|><|437|><|987|><|1653|><|1109|><|898|><|1285|><|80|><|593|><|1709|><|843|><|code_end|> -#and<|t_0.15|><|code_start|><|1285|><|987|><|303|><|1037|><|730|><|1164|><|502|><|120|><|1737|><|1655|><|1318|><|code_end|> -#it<|t_0.09|><|code_start|><|848|><|1366|><|395|><|1601|><|1513|><|593|><|1302|><|code_end|> -#looks<|t_0.27|><|code_start|><|1281|><|1266|><|1755|><|572|><|248|><|1751|><|1257|><|695|><|1380|><|457|><|659|><|585|><|1315|><|1105|><|1776|><|736|><|24|><|736|><|654|><|1027|><|code_end|> -#lovely<|t_0.56|><|code_start|><|634|><|596|><|1766|><|1556|><|1306|><|1285|><|1481|><|1721|><|1123|><|438|><|1246|><|1251|><|795|><|659|><|1381|><|1658|><|217|><|1772|><|562|><|952|><|107|><|1129|><|1112|><|467|><|550|><|1079|><|840|><|1615|><|1469|><|1380|><|168|><|917|><|836|><|1827|><|437|><|583|><|67|><|595|><|1087|><|1646|><|1493|><|1677|><|code_end|>""" - -# TODO: tokenization is slow for some reason - here is pre-tokenized input -suffix = [ 151667, 198, 1782, 155780, 151669, 151929, 152412, 152308, 152585, 152460, 153375, 151670, 198, 74455, - 155808, 151669, 151799, 151873, 151863, 152446, 152372, 152204, 152728, 152229, 152470, 151970, 153413, - 152419, 153334, 153289, 153374, 153199, 152040, 153260, 152721, 152680, 153297, 152419, 153248, 152400, - 152691, 153368, 153437, 151670, 198, 1722, 155828, 151669, 152607, 152256, 152991, 152299, 152688, 153163, - 153016, 152789, 153198, 152712, 151911, 153107, 152623, 152170, 152395, 152852, 152207, 152461, 153321, - 153309, 151750, 152137, 153340, 152573, 152267, 153347, 151789, 152681, 153339, 151992, 152512, 151751, - 152179, 153434, 153180, 152900, 153440, 152474, 153122, 153129, 151904, 152311, 151670, 198, 1499, 155791, - 151669, 152276, 152454, 153354, 152544, 153204, 153272, 152708, 153433, 152319, 153226, 153043, 152325, - 153267, 152622, 151670, 198, 4250, 155797, 151669, 153454, 153342, 151989, 152458, 153420, 152303, 152271, - 152827, 153036, 153196, 151708, 153263, 152561, 153207, 152213, 152112, 153204, 151722, 152542, 151670, 198, - 19789, 155796, 151669, 153353, 153182, 152345, 152471, 152477, 153014, 152002, 152191, 151734, 152312, 152810, - 152237, 153224, 153169, 153224, 152244, 153387, 153404, 151670, 198, 16069, 155811, 151669, 152265, 151946, - 151808, 152412, 152363, 152305, 153156, 152733, 152810, 153157, 152016, 152100, 152069, 153234, 152317, - 152589, 152707, 153121, 153341, 152159, 152114, 153156, 153001, 153504, 153376, 152272, 152433, 152325, - 151941, 151670, 198, 285, 155788, 151669, 152238, 152255, 153427, 152318, 153009, 152381, 152474, 152680, - 152157, 153255, 152324, 151682, 151670, 198, 32955, 155804, 151669, 153490, 153419, 152364, 152405, 152682, - 152206, 152078, 153369, 152725, 153193, 153027, 152946, 152488, 153070, 151883, 152890, 152489, 153144, - 153375, 152358, 151685, 152494, 152117, 152740, 151670, 198, 37448, 480, 155840, 151669, 151902, 152720, - 153377, 152027, 152378, 152821, 153207, 153459, 153028, 153068, 152507, 153255, 152158, 152921, 151958, - 152609, 152748, 152822, 152286, 151714, 152730, 152377, 152353, 152470, 152606, 152162, 152186, 153071, - 152244, 153118, 153375, 153018, 152712, 153098, 152976, 152336, 151843, 153202, 152297, 151736, 153380, - 153502, 152702, 152115, 153181, 152735, 153277, 153457, 152393, 153112, 152595, 151670, 198, 19098, 155808, - 151669, 152464, 153452, 152595, 153312, 151937, 151933, 153197, 152239, 153163, 152922, 153402, 152034, - 152591, 153438, 152215, 151673, 152005, 151785, 152642, 151924, 153278, 151805, 151974, 153482, 152718, - 152862, 153347, 151670, 198, 72, 155780, 151669, 151795, 152111, 152746, 152377, 153471, 152309, 151670, 198, - 19016, 155788, 151669, 153181, 152271, 152190, 152842, 152224, 152701, 152939, 152536, 152091, 151815, 152733, - 151672, 151670, 198, 14689, 155788, 151669, 152291, 152072, 152942, 151734, 153042, 153504, 152589, 153333, - 151839, 151941, 153038, 153180, 151670, 198, 36996, 8303, 155832, 151669, 152231, 152256, 152835, 152801, - 152985, 153400, 152393, 152818, 152765, 152249, 152600, 151699, 152302, 152752, 153018, 153009, 151992, - 153054, 152847, 153354, 153228, 152662, 153355, 152532, 153393, 151782, 152458, 152048, 152757, 152428, - 153195, 151906, 153006, 153178, 153250, 152331, 152284, 152780, 153138, 153319, 151980, 153142, 152418, - 152228, 152733, 151670, 198, 9096, 155801, 151669, 151698, 153321, 152217, 153039, 152935, 153400, 152122, - 152531, 153106, 152169, 152892, 152957, 151851, 152427, 152826, 152451, 151851, 152901, 152885, 152594, - 153446, 153080, 151670, 198, 14689, 155795, 151669, 152658, 151700, 153321, 152450, 152530, 153191, 151673, - 151690, 151698, 152714, 152846, 152981, 153171, 153384, 153364, 153188, 153246, 151670, 198, 1055, 155779, - 151669, 151869, 152388, 152711, 153334, 151736, 151670, 198, 1782, 155780, 151669, 153483, 153240, 152241, - 152558, 152697, 153046, 151670, 198, 5804, 1363, 155820, 151669, 152941, 152764, 152605, 153034, 153434, - 153372, 153347, 151887, 152453, 152758, 152133, 152510, 152694, 152431, 152321, 153088, 152676, 152223, - 152581, 152459, 152015, 152502, 153063, 152712, 153294, 153451, 153032, 152903, 152859, 152989, 151748, - 152669, 152661, 152650, 152409, 151861, 151670, 198, 300, 7973, 155828, 151669, 153095, 152469, 152988, - 152894, 151819, 152391, 153019, 152058, 153062, 153230, 151826, 152112, 152306, 152264, 152769, 153390, - 152384, 152435, 152790, 153393, 152983, 152540, 152252, 152034, 153107, 152540, 151919, 151893, 152558, - 152817, 152946, 152956, 152129, 152715, 153131, 153490, 151734, 152271, 152707, 151734, 153321, 152450, - 151670, 198, 8088, 155792, 151669, 152452, 153497, 153353, 152679, 152533, 152382, 152374, 152611, 153341, - 153163, 152285, 153411, 152495, 153141, 152320, 151670, 198, 1199, 155781, 151669, 151764, 152360, 153295, - 152634, 153342, 152199, 152271, 151670, 198, 43366, 155799, 151669, 152308, 151682, 152889, 152016, 152385, - 152629, 152495, 151826, 153321, 152958, 152180, 151886, 153432, 152922, 152128, 153024, 153040, 152593, - 152287, 151677, 151670, 198, 53660, 155808, 151669, 151727, 152092, 152680, 153331, 151699, 152316, 152938, - 152289, 152433, 153384, 151781, 153137, 153259, 152175, 153213, 152291, 151869, 152691, 152489, 151941, - 152049, 152034, 153053, 152179, 153160, 151676, 153367, 151670, 198, 268, 4123, 480, 155821, 151669, 152350, - 152173, 152536, 151991, 151960, 153144, 153013, 152358, 152234, 153135, 152291, 153235, 152143, 152583, - 152402, 153483, 152678, 152192, 152533, 152946, 151797, 153103, 152310, 152293, 151825, 152548, 153442, - 152109, 152659, 153325, 152781, 152570, 152957, 151752, 152265, 153381, 152515, 151670, 198, 437, 155787, - 151669, 152957, 152659, 151975, 152709, 152402, 152836, 152174, 151792, 153409, 153327, 152990, 151670, 198, - 275, 155781, 151669, 152520, 153038, 152067, 153273, 153185, 152265, 152974, 151670, 198, 94273, 155799, - 151669, 152953, 152938, 153427, 152244, 151920, 153423, 152929, 152367, 153052, 152129, 152331, 152257, - 152987, 152777, 153448, 152408, 151696, 152408, 152326, 152699, 151670, 198, 385, 16239, 155828, 151669, - 152306, 152268, 153438, 153228, 152978, 152957, 153153, 153393, 152795, 152110, 152918, 152923, 152467, - 152331, 153053, 153330, 151889, 153444, 152234, 152624, 151779, 152801, 152784, 152139, 152222, 152751, - 152512, 153287, 153141, 153052, 151840, 152589, 152508, 153499, 152109, 152255, 151739, 152267, 152759, - 153318, 153165, 153349, 151670, ] - -response = requests.post( - host_llm + "/completion", - json={ - "prompt": [prefix + words, *suffix], - "n_predict": 1024, - "cache_prompt": True, - "return_tokens": True, - "samplers": ["top_k"], - "top_k": 16, - "seed": 1003, - } -) - -response_json = response.json() - -#print(json.dumps(response_json, indent=4)) -#print(json.dumps(response_json["prompt"], indent=4).replace("\\n", "\n")) -#print(json.dumps(response_json["timings"], indent=4)) -#print(json.dumps(response_json["tokens"], indent=4)) - -codes = response_json["tokens"] - -codes = [t - 151672 for t in codes if t >= 151672 and t <= 155772] - -response = requests.post( - host_dec + "/embeddings", - json={ - "input": [*codes], - } -) - -response_json = response.json() - -#print(json.dumps(response_json, indent=4)) - -# spectrogram -embd = response_json[0]["embedding"] - -n_codes = len(embd) -n_embd = len(embd[0]) - -print('spectrogram generated: n_codes: %d, n_embd: %d' % (n_codes, n_embd)) - -# post-process the spectrogram to convert to audio -print('converting to audio ...') -audio = embd_to_audio(embd, n_codes, n_embd) -print('audio generated: %d samples' % len(audio)) - -filename = "output.wav" -sample_rate = 24000 # sampling rate - -# zero out first 0.25 seconds -audio[:24000 // 4] = 0.0 - -save_wav(filename, audio, sample_rate) -print('audio written to file "%s"' % filename) diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index 2a1bdccc9192..b68edcaf5758 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -1,1096 +1,205 @@ -#define _USE_MATH_DEFINES // For M_PI on MSVC - #include "arg.h" #include "common.h" #include "sampling.h" #include "log.h" #include "llama.h" +#include "mtmd.h" +#include "mtmd-helper.h" -#define JSON_ASSERT GGML_ASSERT -#include - -#include -#include -#include #include -#include -#include -#include +#include #include -#include -#include - -using json = nlohmann::ordered_json; - -enum outetts_version { - OUTETTS_V0_2, - OUTETTS_V0_3, -}; - -// -// Terminal utils -// - -#define SQR(X) ((X) * (X)) -#define UNCUBE(x) x < 48 ? 0 : x < 115 ? 1 : (x - 35) / 40 /** - * Quantizes 24-bit RGB to xterm256 code range [16,256). + * Please note that this is NOT a production-ready binary. + * It is a playground for trying TTS support in llama.cpp. + * For contributors: please keep this code simple and easy to understand. Do not add unnecessary complexity. The goal is to have a simple CLI for testing TTS support. */ -static int rgb2xterm256(int r, int g, int b) { - unsigned char cube[] = {0, 0137, 0207, 0257, 0327, 0377}; - int av, ir, ig, ib, il, qr, qg, qb, ql; - av = r * .299 + g * .587 + b * .114 + .5; - ql = (il = av > 238 ? 23 : (av - 3) / 10) * 10 + 8; - qr = cube[(ir = UNCUBE(r))]; - qg = cube[(ig = UNCUBE(g))]; - qb = cube[(ib = UNCUBE(b))]; - if (SQR(qr - r) + SQR(qg - g) + SQR(qb - b) <= - SQR(ql - r) + SQR(ql - g) + SQR(ql - b)) - return ir * 36 + ig * 6 + ib + 020; - return il + 0350; -} - -static std::string set_xterm256_foreground(int r, int g, int b) { - int x = rgb2xterm256(r, g, b); - std::ostringstream oss; - oss << "\033[38;5;" << x << "m"; - return oss.str(); -} - -const std::vector k_colors = { - set_xterm256_foreground(220, 5, 12), - set_xterm256_foreground(232, 96, 28), - set_xterm256_foreground(241, 147, 45), - set_xterm256_foreground(246, 193, 65), - set_xterm256_foreground(247, 240, 86), - set_xterm256_foreground(144, 201, 135), - set_xterm256_foreground( 78, 178, 101), -}; - -static void print_usage(int, char ** argv) { - LOG("\nexample usage:\n"); - LOG("\n %s -m model.gguf -p \"Hello!\"\n", argv[0]); - LOG("\n"); -} - -struct wav_header { - char riff[4] = {'R', 'I', 'F', 'F'}; - uint32_t chunk_size; - char wave[4] = {'W', 'A', 'V', 'E'}; - char fmt[4] = {'f', 'm', 't', ' '}; - uint32_t fmt_chunk_size = 16; - uint16_t audio_format = 1; // PCM - uint16_t num_channels = 1; // Mono - uint32_t sample_rate; - uint32_t byte_rate; - uint16_t block_align; - uint16_t bits_per_sample = 16; - char data[4] = {'d', 'a', 't', 'a'}; - uint32_t data_size; -}; - -static bool save_wav16(const std::string & fname, const std::vector & data, int sample_rate) { - std::ofstream file(fname, std::ios::binary); - if (!file) { - LOG_ERR("%s: Failed to open file '%s' for writing.\n", __func__, fname.c_str()); - return false; - } - - wav_header header; - header.sample_rate = sample_rate; - header.byte_rate = header.sample_rate * header.num_channels * (header.bits_per_sample / 8); - header.block_align = header.num_channels * (header.bits_per_sample / 8); - header.data_size = data.size() * (header.bits_per_sample / 8); - header.chunk_size = 36 + header.data_size; - - file.write(reinterpret_cast(&header), sizeof(header)); - - for (const auto & sample : data) { - int16_t pcm_sample = static_cast(std::clamp(sample * 32767.0, -32768.0, 32767.0)); - file.write(reinterpret_cast(&pcm_sample), sizeof(pcm_sample)); - } - - return file.good(); -} -static void fill_hann_window(int length, bool periodic, float * output) { - int offset = -1; - if (periodic) { - offset = 0; - } - for (int i = 0; i < length; i++) { - output[i] = 0.5 * (1.0 - cosf((2.0 * M_PI * i) / (length + offset))); - } -} - -// very poor-man fft -static void twiddle(float * real, float * imag, int k, int N) { - float angle = 2 * M_PI * k / N; - *real = cos(angle); - *imag = sin(angle); -} - -static void irfft(int n, const float * inp_cplx, float * out_real) { - int N = n / 2 + 1; - - std::vector real_input(N); - std::vector imag_input(N); - for (int i = 0; i < N; ++i) { - real_input[i] = inp_cplx[2 * i]; - imag_input[i] = inp_cplx[2 * i + 1]; - } +struct tts_timings { + int64_t t_start_us = ggml_time_us(); + int64_t t_last_us = t_start_us; - std::vector real_output(n); - std::vector imag_output(n); - - for (int k = 0; k < n; ++k) { - real_output[k] = 0.0f; - imag_output[k] = 0.0f; - for (int m = 0; m < N; ++m) { - float twiddle_real; - float twiddle_imag; - - twiddle(&twiddle_real, &twiddle_imag, k * m, n); - - real_output[k] += real_input[m] * twiddle_real - imag_input[m] * twiddle_imag; - imag_output[k] += real_input[m] * twiddle_imag + imag_input[m] * twiddle_real; + void report(int n_frames) { + const int64_t t_now_us = ggml_time_us(); + if (t_now_us - t_last_us < 2000000) { + return; } + t_last_us = t_now_us; + const double t_elapsed_s = (t_now_us - t_start_us) / 1e6; + const double fps = t_elapsed_s > 0 ? n_frames / t_elapsed_s : 0.0; + LOG_INF("frames generated: %d, speed: %.2f frames/s\n", n_frames, fps); } - - for (int i = 0; i < n; ++i) { - out_real[i] = real_output[i] / N; - } -} - -// -// y = torch.nn.functional.fold( -// data, output_size=(1, output_size), kernel_size=(1, self.win_length), stride=(1, self.hop_length), -// )[:, 0, 0, pad:-pad] -// -// data.shape = torch.Size([1, 1280, 261]) -// output_size = 84480 -// win_length = 1280 -// hop_length = 320 -// pad = 480 -// -static void fold(const std::vector & data, int64_t n_out, int64_t n_win, int64_t n_hop, int64_t n_pad, std::vector & output) { - int64_t output_height = n_out; - int64_t kernel_w = n_win; - int64_t stride_w = n_hop; - int64_t width = n_out; - - output.resize(width, 0.0f); - - int64_t col_idx = 0; - for (int64_t w_col = 0; w_col < width; ++w_col) { - int64_t start = w_col * stride_w - n_pad; - int64_t end = start + kernel_w; - - for (int64_t w_im = start; w_im < end; ++w_im) { - if (w_im >= 0 && w_im < output_height && col_idx < (int64_t) data.size()) { - output[w_im] += data[col_idx]; - } - col_idx++; - } - } - - output.resize(n_out - 2 * n_pad); -} - -// TODO: not optimized at all -static std::vector embd_to_audio( - const float * embd, - const int n_codes, - const int n_embd, - const int n_thread) { - const int n_fft = 1280; - const int n_hop = 320; - const int n_win = 1280; - const int n_pad = (n_win - n_hop)/2; - const int n_out = (n_codes - 1)*n_hop + n_win; - - std::vector hann(n_fft); - - fill_hann_window(hann.size(), true, hann.data()); - - int n_spec = n_embd*n_codes; - - std::vector E (n_spec); - std::vector S (n_spec); - std::vector ST(n_spec); - - for (int l = 0; l < n_codes; ++l) { - for (int k = 0; k < n_embd; ++k) { - E[k*n_codes + l] = embd[l*n_embd + k]; - } - } - - for (int k = 0; k < n_embd/2; ++k) { - for (int l = 0; l < n_codes; ++l) { - float mag = E[(k )*n_codes + l]; - float phi = E[(k + n_embd/2)*n_codes + l]; - - mag = exp(mag); - - if (mag > 1e2) { - mag = 1e2; - } - S[2*(k*n_codes + l) + 0] = mag*cosf(phi); - S[2*(k*n_codes + l) + 1] = mag*sinf(phi); - } - } - - for (int l = 0; l < n_codes; ++l) { - for (int k = 0; k < n_embd/2; ++k) { - ST[l*n_embd + 2*k + 0] = S[2*(k*n_codes + l) + 0]; - ST[l*n_embd + 2*k + 1] = S[2*(k*n_codes + l) + 1]; - } - } - - std::vector res (n_codes*n_fft); - std::vector hann2(n_codes*n_fft); - - std::vector workers(n_thread); - for (int i = 0; i < n_thread; ++i) { - workers[i] = std::thread([&, i]() { - for (int l = i; l < n_codes; l += n_thread) { - irfft(n_fft, ST.data() + l*n_embd, res.data() + l*n_fft); - for (int j = 0; j < n_fft; ++j) { - res [l*n_fft + j] *= hann[j]; - hann2[l*n_fft + j] = hann[j] * hann[j]; - } - } - }); - } - for (int i = 0; i < n_thread; ++i) { - workers[i].join(); - } - - std::vector audio; - std::vector env; - - fold(res, n_out, n_win, n_hop, n_pad, audio); - fold(hann2, n_out, n_win, n_hop, n_pad, env); // TODO: can be done once - - for (size_t i = 0; i < audio.size(); ++i) { - audio[i] /= env[i]; - } - - return audio; -} - -static const std::map ones = { - {0, "zero"}, {1, "one"}, {2, "two"}, {3, "three"}, {4, "four"}, - {5, "five"}, {6, "six"}, {7, "seven"}, {8, "eight"}, {9, "nine"}, - {10, "ten"}, {11, "eleven"}, {12, "twelve"}, {13, "thirteen"}, {14, "fourteen"}, - {15, "fifteen"}, {16, "sixteen"}, {17, "seventeen"}, {18, "eighteen"}, {19, "nineteen"} }; -static const std::map tens = { - {2, "twenty"}, {3, "thirty"}, {4, "forty"}, {5, "fifty"}, - {6, "sixty"}, {7, "seventy"}, {8, "eighty"}, {9, "ninety"} -}; - -// Convert a number less than 1000 to words -static std::string convert_less_than_thousand(int num) { - std::string result; - - if (num >= 100) { - result += ones.at(num / 100) + " hundred "; - num %= 100; - } - - if (num >= 20) { - result += tens.at(num / 10); - if (num % 10 > 0) { - result += "-" + ones.at(num % 10); - } - } else if (num > 0) { - result += ones.at(num); - } - - return result; -} - -static std::string number_to_words(const std::string & number_str) { - try { - size_t decimal_pos = number_str.find('.'); - std::string integer_part = number_str.substr(0, decimal_pos); - - int int_number = std::stoi(integer_part); - std::string result; - - if (int_number == 0) { - result = "zero"; - } else { - if (int_number >= 1000000000) { - int billions = int_number / 1000000000; - result += convert_less_than_thousand(billions) + " billion "; - int_number %= 1000000000; - } - - if (int_number >= 1000000) { - int millions = int_number / 1000000; - result += convert_less_than_thousand(millions) + " million "; - int_number %= 1000000; - } - - if (int_number >= 1000) { - int thousands = int_number / 1000; - result += convert_less_than_thousand(thousands) + " thousand "; - int_number %= 1000; - } - - if (int_number > 0) { - result += convert_less_than_thousand(int_number); - } - } - - // Handle decimal part - if (decimal_pos != std::string::npos) { - result += " point"; - std::string decimal_part = number_str.substr(decimal_pos + 1); - for (char digit : decimal_part) { - result += " " + ones.at(digit - '0'); - } - } - - return result; - } catch (const std::exception& e) { - // Skip if fails - return " "; - } -} - -static std::string replace_numbers_with_words(const std::string & input_text) { - std::regex number_pattern(R"(\d+(\.\d+)?)"); - std::string result; - auto it = std::sregex_iterator(input_text.begin(), input_text.end(), number_pattern); - auto end = std::sregex_iterator(); - - size_t last_pos = 0; - for (std::sregex_iterator i = it; i != end; ++i) { - const std::smatch& match = *i; - result.append(input_text, last_pos, match.position() - last_pos); - result.append(number_to_words(match.str())); - last_pos = match.position() + match.length(); - } - result.append(input_text, last_pos); - - return result; -} - -// Based on: https://github.com/edwko/OuteTTS/blob/a613e79c489d8256dd657ea9168d78de75895d82/outetts/version/v1/prompt_processor.py#L39 -static std::string process_text(const std::string & text, const outetts_version tts_version = OUTETTS_V0_2) { - - // For now I skipped text romanization as I am unsure how to handle - // uroman and MeCab implementations in C++ - // maybe something like https://github.com/anyascii/anyascii/ could work. - // currently only English would be supported in this function - - std::string processed_text = replace_numbers_with_words(text); - - std::transform(processed_text.begin(), processed_text.end(), - processed_text.begin(), ::tolower); - - std::regex special_chars(R"([-_/,\.\\])"); - processed_text = std::regex_replace(processed_text, special_chars, " "); - - std::regex non_alpha(R"([^a-z\s])"); - processed_text = std::regex_replace(processed_text, non_alpha, ""); - - std::regex multiple_spaces(R"(\s+)"); - processed_text = std::regex_replace(processed_text, multiple_spaces, " "); - - processed_text = std::regex_replace(processed_text, std::regex(R"(^\s+|\s+$)"), ""); - - /* - Replace spaces with the separator token same as in line 365 - - for (auto & c : prompt_user) { - if (c == ' ') { - prompt_clean += "<|text_sep|>"; - */ - std::string separator = (tts_version == OUTETTS_V0_3) ? "<|space|>" : "<|text_sep|>"; - processed_text = std::regex_replace(processed_text, std::regex(R"(\s)"), separator); - - return processed_text; -} - -static void prompt_add(llama_tokens & prompt, llama_token token) { - prompt.push_back(token); -} - -static void prompt_add(llama_tokens & prompt, const llama_tokens & tokens) { - prompt.insert(prompt.end(), tokens.begin(), tokens.end()); -} - -static void prompt_add(llama_tokens & prompt, const llama_vocab * vocab, const std::string & txt, bool add_special, bool parse_special) { - auto tmp = common_tokenize(vocab, txt, add_special, parse_special); - prompt_add(prompt, tmp); -} - -static void prompt_init(llama_tokens & prompt, const llama_vocab * vocab) { - prompt.clear(); - - prompt_add(prompt, vocab, "<|im_start|>\n", true, true); -} - -static std::vector prepare_guide_tokens(const llama_vocab * vocab, const std::string & str, const outetts_version tts_version = OUTETTS_V0_2) { - const std::string& delimiter = (tts_version == OUTETTS_V0_3 ? "<|space|>" : "<|text_sep|>"); - - std::vector result; - size_t start = 0; - size_t end = str.find(delimiter); - - //first token is always a newline, as it was not previously added - result.push_back(common_tokenize(vocab, "\n", false, true)[0]); - - while (end != std::string::npos) { - std::string current_word = str.substr(start, end - start); - auto tmp = common_tokenize(vocab, current_word, false, true); - result.push_back(tmp[0]); - start = end + delimiter.length(); - end = str.find(delimiter, start); - } - - // Add the last part - std::string current_word = str.substr(start); - auto tmp = common_tokenize(vocab, current_word, false, true); - if (tmp.size() > 0) { - result.push_back(tmp[0]); - } - return result; -} - -static json speaker_from_file(const std::string & speaker_file) { - std::ifstream file(speaker_file); - if (!file) { - LOG_ERR("%s: Failed to open file '%s' for reading\n", __func__, speaker_file.c_str()); - return json(); - } - - json speaker = json::parse(file); - return speaker; -} - -static outetts_version get_tts_version(llama_model *model, json speaker = json::object()) { - if (speaker.contains("version")) { - std::string version = speaker["version"].get(); - if (version == "0.2") { - return OUTETTS_V0_2; - } else if (version == "0.3") { - return OUTETTS_V0_3; - } else { - LOG_ERR("%s: Unsupported speaker version '%s'\n", __func__, version.c_str()); - } - } - - // Also could get version from model itself - const char *chat_template = llama_model_chat_template(model, nullptr); - if (chat_template && std::string(chat_template) == "outetts-0.3") { - return OUTETTS_V0_3; - } - - // Use 0.2 as the default version - return OUTETTS_V0_2; -} - -static std::string audio_text_from_speaker(json speaker, const outetts_version tts_version = OUTETTS_V0_2) { - std::string audio_text = "<|text_start|>"; - - if (tts_version == OUTETTS_V0_2 || tts_version == OUTETTS_V0_3) { - std::string separator = (tts_version == OUTETTS_V0_3) ? "<|space|>" : "<|text_sep|>"; - for (const auto &word : speaker["words"]) { - audio_text += word["word"].get() + separator; - } - } - - return audio_text; -} - -static std::string audio_data_from_speaker(json speaker, const outetts_version tts_version = OUTETTS_V0_2) { - std::string audio_data = "<|audio_start|>\n"; - - if (tts_version == OUTETTS_V0_2 || tts_version == OUTETTS_V0_3) { - std::string code_start = (tts_version == OUTETTS_V0_3) ? "" : "<|code_start|>"; - std::string code_end = (tts_version == OUTETTS_V0_3) ? "<|space|>" : "<|code_end|>"; - for (const auto &word : speaker["words"]) { - std::string word_text = word["word"].get(); - double duration = word["duration"].get(); - std::vector codes = word["codes"].get>(); - - // Create the audio output entry - std::ostringstream word_entry; - word_entry << word_text << "<|t_" << std::fixed << std::setprecision(2) - << duration << "|>" + code_start; - for (const auto &Code : codes) { - word_entry << "<|" << Code << "|>"; - } - word_entry << code_end << "\n"; - audio_data += word_entry.str(); - } - } - - return audio_data; +static void print_usage(int, char ** argv) { + LOG("\nexample usage:\n"); + LOG("\n %s -m backbone.gguf -mm mmproj.gguf -p \"text to speak\" -o output.wav", argv[0]); + LOG("\n %s -hf user/model -p \"text to speak\" -o output.wav\n", argv[0]); + LOG("\nnote: --tts-lang and --tts-speaker-file may not be supported in all models"); + LOG("\n use -n to limit the output length"); + LOG("\n see tts/README.md for per-model usage notes"); + LOG("\n\n"); } int main(int argc, char ** argv) { - std::setlocale(LC_NUMERIC, "C"); - common_params params; - params.out_file = "output.wav"; - params.prompt = ""; - - params.n_predict = 4096; - params.n_batch = 8192; - params.n_ctx = 8192; - - params.sampling.top_k = 4; - params.sampling.samplers = { COMMON_SAMPLER_TYPE_TOP_K, }; - common_init(); if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_TTS, print_usage)) { return 1; } - const int n_parallel = params.n_parallel; - const int n_predict = params.n_predict; - - // init LLM - - llama_backend_init(); - llama_numa_init(params.numa); - - llama_model * model_ttc = NULL; // text-to-codes - llama_model * model_cts = NULL; // codes-to-speech - - llama_context * ctx_ttc = NULL; - llama_context * ctx_cts = NULL; - - auto llama_init_ttc = common_init_from_params(params); - - model_ttc = llama_init_ttc->model(); - ctx_ttc = llama_init_ttc->context(); + mtmd_helper_log_set(common_log_default_callback, nullptr); - if (model_ttc == nullptr || ctx_ttc == nullptr) { - return ENOENT; + if (params.prompt.empty()) { + LOG_ERR("no prompt provided, use -p \"text\"\n"); + return 1; + } + if (params.mmproj.path.empty()) { + LOG_ERR("no mmproj provided, use --mmproj\n"); + return 1; } - const llama_vocab * vocab = llama_model_get_vocab(model_ttc); + // important: keep this file as generic as possible + // model-specific logic should be in mtmd-helper-gen or mtmd API - params.model = params.vocoder.model; + // always enable embd, so that we can pass hidden states to the audio generation helper params.embedding = true; - params.n_ubatch = params.n_batch; - auto llama_init_cts = common_init_from_params(params); + llama_backend_init(); + llama_numa_init(params.numa); - model_cts = llama_init_cts->model(); - ctx_cts = llama_init_cts->context(); + // + // load backbone model and mmproj + // - if (model_cts == nullptr || ctx_cts == nullptr) { - return ENOENT; + auto llama_init = common_init_from_params(params); + llama_model * model = llama_init->model(); + llama_context * lctx = llama_init->context(); + common_sampler * smpl = llama_init->sampler(0); + if (!model || !lctx) { + LOG_ERR("failed to init model/context\n"); + return 1; } - std::vector smpl(n_parallel); - for (int i = 0; i < n_parallel; ++i) { - params.sampling.no_perf = (i != 0); - params.sampling.seed = params.sampling.seed + 1; - - smpl[i] = common_sampler_init(model_ttc, params.sampling); + mtmd_context_params mtmd_params = mtmd_context_params_default(); + mtmd_params.use_gpu = params.mmproj_use_gpu; + mtmd::context_ptr mctx(mtmd_init_from_file(params.mmproj.path.c_str(), model, mtmd_params)); + if (!mctx) { + LOG_ERR("failed to load mmproj %s\n", params.mmproj.path.c_str()); + return 1; } - - LOG_INF("sampler seed: %u\n", common_sampler_get_seed(smpl[0])); - LOG_INF("sampler chain: %s\n", common_sampler_print(smpl[0]).c_str()); - LOG_INF("sampler params: \n%s\n", params.sampling.print().c_str()); - - LOG_INF("%s: loading done\n", __func__); - - const auto t_main_start = ggml_time_us(); - - std::vector codes; - std::vector guide_tokens; - - // the default speaker profile is from: https://github.com/edwko/OuteTTS/blob/main/outetts/version/v1/default_speakers/en_male_1.json - std::string audio_text = "<|text_start|>the<|text_sep|>overall<|text_sep|>package<|text_sep|>from<|text_sep|>just<|text_sep|>two<|text_sep|>people<|text_sep|>is<|text_sep|>pretty<|text_sep|>remarkable<|text_sep|>sure<|text_sep|>i<|text_sep|>have<|text_sep|>some<|text_sep|>critiques<|text_sep|>about<|text_sep|>some<|text_sep|>of<|text_sep|>the<|text_sep|>gameplay<|text_sep|>aspects<|text_sep|>but<|text_sep|>its<|text_sep|>still<|text_sep|>really<|text_sep|>enjoyable<|text_sep|>and<|text_sep|>it<|text_sep|>looks<|text_sep|>lovely<|text_sep|>"; - std::string audio_data = R"(<|audio_start|> -the<|t_0.08|><|code_start|><|257|><|740|><|636|><|913|><|788|><|1703|><|code_end|> -overall<|t_0.36|><|code_start|><|127|><|201|><|191|><|774|><|700|><|532|><|1056|><|557|><|798|><|298|><|1741|><|747|><|1662|><|1617|><|1702|><|1527|><|368|><|1588|><|1049|><|1008|><|1625|><|747|><|1576|><|728|><|1019|><|1696|><|1765|><|code_end|> -package<|t_0.56|><|code_start|><|935|><|584|><|1319|><|627|><|1016|><|1491|><|1344|><|1117|><|1526|><|1040|><|239|><|1435|><|951|><|498|><|723|><|1180|><|535|><|789|><|1649|><|1637|><|78|><|465|><|1668|><|901|><|595|><|1675|><|117|><|1009|><|1667|><|320|><|840|><|79|><|507|><|1762|><|1508|><|1228|><|1768|><|802|><|1450|><|1457|><|232|><|639|><|code_end|> -from<|t_0.19|><|code_start|><|604|><|782|><|1682|><|872|><|1532|><|1600|><|1036|><|1761|><|647|><|1554|><|1371|><|653|><|1595|><|950|><|code_end|> -just<|t_0.25|><|code_start|><|1782|><|1670|><|317|><|786|><|1748|><|631|><|599|><|1155|><|1364|><|1524|><|36|><|1591|><|889|><|1535|><|541|><|440|><|1532|><|50|><|870|><|code_end|> -two<|t_0.24|><|code_start|><|1681|><|1510|><|673|><|799|><|805|><|1342|><|330|><|519|><|62|><|640|><|1138|><|565|><|1552|><|1497|><|1552|><|572|><|1715|><|1732|><|code_end|> -people<|t_0.39|><|code_start|><|593|><|274|><|136|><|740|><|691|><|633|><|1484|><|1061|><|1138|><|1485|><|344|><|428|><|397|><|1562|><|645|><|917|><|1035|><|1449|><|1669|><|487|><|442|><|1484|><|1329|><|1832|><|1704|><|600|><|761|><|653|><|269|><|code_end|> -is<|t_0.16|><|code_start|><|566|><|583|><|1755|><|646|><|1337|><|709|><|802|><|1008|><|485|><|1583|><|652|><|10|><|code_end|> -pretty<|t_0.32|><|code_start|><|1818|><|1747|><|692|><|733|><|1010|><|534|><|406|><|1697|><|1053|><|1521|><|1355|><|1274|><|816|><|1398|><|211|><|1218|><|817|><|1472|><|1703|><|686|><|13|><|822|><|445|><|1068|><|code_end|> -remarkable<|t_0.68|><|code_start|><|230|><|1048|><|1705|><|355|><|706|><|1149|><|1535|><|1787|><|1356|><|1396|><|835|><|1583|><|486|><|1249|><|286|><|937|><|1076|><|1150|><|614|><|42|><|1058|><|705|><|681|><|798|><|934|><|490|><|514|><|1399|><|572|><|1446|><|1703|><|1346|><|1040|><|1426|><|1304|><|664|><|171|><|1530|><|625|><|64|><|1708|><|1830|><|1030|><|443|><|1509|><|1063|><|1605|><|1785|><|721|><|1440|><|923|><|code_end|> -sure<|t_0.36|><|code_start|><|792|><|1780|><|923|><|1640|><|265|><|261|><|1525|><|567|><|1491|><|1250|><|1730|><|362|><|919|><|1766|><|543|><|1|><|333|><|113|><|970|><|252|><|1606|><|133|><|302|><|1810|><|1046|><|1190|><|1675|><|code_end|> -i<|t_0.08|><|code_start|><|123|><|439|><|1074|><|705|><|1799|><|637|><|code_end|> -have<|t_0.16|><|code_start|><|1509|><|599|><|518|><|1170|><|552|><|1029|><|1267|><|864|><|419|><|143|><|1061|><|0|><|code_end|> -some<|t_0.16|><|code_start|><|619|><|400|><|1270|><|62|><|1370|><|1832|><|917|><|1661|><|167|><|269|><|1366|><|1508|><|code_end|> -critiques<|t_0.60|><|code_start|><|559|><|584|><|1163|><|1129|><|1313|><|1728|><|721|><|1146|><|1093|><|577|><|928|><|27|><|630|><|1080|><|1346|><|1337|><|320|><|1382|><|1175|><|1682|><|1556|><|990|><|1683|><|860|><|1721|><|110|><|786|><|376|><|1085|><|756|><|1523|><|234|><|1334|><|1506|><|1578|><|659|><|612|><|1108|><|1466|><|1647|><|308|><|1470|><|746|><|556|><|1061|><|code_end|> -about<|t_0.29|><|code_start|><|26|><|1649|><|545|><|1367|><|1263|><|1728|><|450|><|859|><|1434|><|497|><|1220|><|1285|><|179|><|755|><|1154|><|779|><|179|><|1229|><|1213|><|922|><|1774|><|1408|><|code_end|> -some<|t_0.23|><|code_start|><|986|><|28|><|1649|><|778|><|858|><|1519|><|1|><|18|><|26|><|1042|><|1174|><|1309|><|1499|><|1712|><|1692|><|1516|><|1574|><|code_end|> -of<|t_0.07|><|code_start|><|197|><|716|><|1039|><|1662|><|64|><|code_end|> -the<|t_0.08|><|code_start|><|1811|><|1568|><|569|><|886|><|1025|><|1374|><|code_end|> -gameplay<|t_0.48|><|code_start|><|1269|><|1092|><|933|><|1362|><|1762|><|1700|><|1675|><|215|><|781|><|1086|><|461|><|838|><|1022|><|759|><|649|><|1416|><|1004|><|551|><|909|><|787|><|343|><|830|><|1391|><|1040|><|1622|><|1779|><|1360|><|1231|><|1187|><|1317|><|76|><|997|><|989|><|978|><|737|><|189|><|code_end|> -aspects<|t_0.56|><|code_start|><|1423|><|797|><|1316|><|1222|><|147|><|719|><|1347|><|386|><|1390|><|1558|><|154|><|440|><|634|><|592|><|1097|><|1718|><|712|><|763|><|1118|><|1721|><|1311|><|868|><|580|><|362|><|1435|><|868|><|247|><|221|><|886|><|1145|><|1274|><|1284|><|457|><|1043|><|1459|><|1818|><|62|><|599|><|1035|><|62|><|1649|><|778|><|code_end|> -but<|t_0.20|><|code_start|><|780|><|1825|><|1681|><|1007|><|861|><|710|><|702|><|939|><|1669|><|1491|><|613|><|1739|><|823|><|1469|><|648|><|code_end|> -its<|t_0.09|><|code_start|><|92|><|688|><|1623|><|962|><|1670|><|527|><|599|><|code_end|> -still<|t_0.27|><|code_start|><|636|><|10|><|1217|><|344|><|713|><|957|><|823|><|154|><|1649|><|1286|><|508|><|214|><|1760|><|1250|><|456|><|1352|><|1368|><|921|><|615|><|5|><|code_end|> -really<|t_0.36|><|code_start|><|55|><|420|><|1008|><|1659|><|27|><|644|><|1266|><|617|><|761|><|1712|><|109|><|1465|><|1587|><|503|><|1541|><|619|><|197|><|1019|><|817|><|269|><|377|><|362|><|1381|><|507|><|1488|><|4|><|1695|><|code_end|> -enjoyable<|t_0.49|><|code_start|><|678|><|501|><|864|><|319|><|288|><|1472|><|1341|><|686|><|562|><|1463|><|619|><|1563|><|471|><|911|><|730|><|1811|><|1006|><|520|><|861|><|1274|><|125|><|1431|><|638|><|621|><|153|><|876|><|1770|><|437|><|987|><|1653|><|1109|><|898|><|1285|><|80|><|593|><|1709|><|843|><|code_end|> -and<|t_0.15|><|code_start|><|1285|><|987|><|303|><|1037|><|730|><|1164|><|502|><|120|><|1737|><|1655|><|1318|><|code_end|> -it<|t_0.09|><|code_start|><|848|><|1366|><|395|><|1601|><|1513|><|593|><|1302|><|code_end|> -looks<|t_0.27|><|code_start|><|1281|><|1266|><|1755|><|572|><|248|><|1751|><|1257|><|695|><|1380|><|457|><|659|><|585|><|1315|><|1105|><|1776|><|736|><|24|><|736|><|654|><|1027|><|code_end|> -lovely<|t_0.56|><|code_start|><|634|><|596|><|1766|><|1556|><|1306|><|1285|><|1481|><|1721|><|1123|><|438|><|1246|><|1251|><|795|><|659|><|1381|><|1658|><|217|><|1772|><|562|><|952|><|107|><|1129|><|1112|><|467|><|550|><|1079|><|840|><|1615|><|1469|><|1380|><|168|><|917|><|836|><|1827|><|437|><|583|><|67|><|595|><|1087|><|1646|><|1493|><|1677|><|code_end|>)"; - - // audio data for 0.3 version - outetts_version tts_version = get_tts_version(model_ttc); - if (tts_version == OUTETTS_V0_3) { - audio_text = std::regex_replace(audio_text, std::regex(R"(<\|text_sep\|>)"), "<|space|>"); - audio_data = std::regex_replace(audio_data, std::regex(R"(<\|code_start\|>)"), ""); - audio_data = std::regex_replace(audio_data, std::regex(R"(<\|code_end\|>)"), "<|space|>"); + if (mtmd_gen_audio_get_info(mctx.get()).type == MTMD_GEN_AUDIO_TYPE_NONE) { + LOG_ERR("mmproj does not support audio generation\n"); + return 1; } - // load speaker if given - if (!params.vocoder.speaker_file.empty()) { - LOG_INF("%s: loading speaker ..\n", __func__); - json speaker = speaker_from_file(params.vocoder.speaker_file); - if (speaker.empty()) { - LOG_ERR("%s: Failed to load speaker file '%s'\n", __func__, params.vocoder.speaker_file.c_str()); + // + // stage 0: process speaker reference file, if any + // + + mtmd::bitmap_ptr speaker_bitmap; + if (!params.tts_speaker_file.empty()) { + auto wrapper = mtmd_helper_bitmap_init_from_file(mctx.get(), params.tts_speaker_file.c_str(), false); + if (!wrapper.bitmap) { + LOG_ERR("failed to load speaker file %s\n", params.tts_speaker_file.c_str()); return 1; } - audio_text = audio_text_from_speaker(speaker, tts_version); - audio_data = audio_data_from_speaker(speaker, tts_version); + speaker_bitmap.reset(wrapper.bitmap); } - // process prompt and generate voice codes - { - LOG_INF("%s: constructing prompt ..\n", __func__); - - std::vector prompt_inp; - - prompt_init(prompt_inp, vocab); - - prompt_add(prompt_inp, vocab, audio_text, false, true); - - // convert the input text into the necessary format expected by OuteTTS - { - std::string prompt_clean = process_text(params.prompt, tts_version); - if (params.vocoder.use_guide_tokens) { - guide_tokens = prepare_guide_tokens(vocab, prompt_clean, tts_version); - } - - LOG_INF("%s: prompt: '%s'\n", __func__, prompt_clean.c_str()); - - prompt_add(prompt_inp, vocab, prompt_clean, false, true); - } - - prompt_add(prompt_inp, vocab, "<|text_end|>\n", false, true); - - if (!params.vocoder.speaker_file.empty()) { - prompt_add(prompt_inp, vocab, audio_data, false, true); - } else { - // disabled to save time on tokenizing each time -#if 1 - const std::string voice_data = audio_data; - - auto tmp = common_tokenize(vocab, voice_data, false, true); - - std::ostringstream tokens_oss; - for (size_t i = 0; i < tmp.size(); ++i) { - tokens_oss << tmp[i] << ", "; - } - LOG_INF("\n\n%s: llama tokens: %s\n\n", __func__, tokens_oss.str().c_str()); - - prompt_add(prompt_inp, tmp); -#else - prompt_add(prompt_inp, llama_tokens { - 151667, 198, 1782, 155780, 151669, 151929, 152412, 152308, 152585, - 152460, 153375, 151670, 198, 74455, 155808, 151669, 151799, - 151873, 151863, 152446, 152372, 152204, 152728, 152229, 152470, - 151970, 153413, 152419, 153334, 153289, 153374, 153199, 152040, - 153260, 152721, 152680, 153297, 152419, 153248, 152400, 152691, - 153368, 153437, 151670, 198, 1722, 155828, 151669, 152607, - 152256, 152991, 152299, 152688, 153163, 153016, 152789, 153198, - 152712, 151911, 153107, 152623, 152170, 152395, 152852, 152207, - 152461, 153321, 153309, 151750, 152137, 153340, 152573, 152267, - 153347, 151789, 152681, 153339, 151992, 152512, 151751, 152179, - 153434, 153180, 152900, 153440, 152474, 153122, 153129, 151904, - 152311, 151670, 198, 1499, 155791, 151669, 152276, 152454, - 153354, 152544, 153204, 153272, 152708, 153433, 152319, 153226, - 153043, 152325, 153267, 152622, 151670, 198, 4250, 155797, - 151669, 153454, 153342, 151989, 152458, 153420, 152303, 152271, - 152827, 153036, 153196, 151708, 153263, 152561, 153207, 152213, - 152112, 153204, 151722, 152542, 151670, 198, 19789, 155796, - 151669, 153353, 153182, 152345, 152471, 152477, 153014, 152002, - 152191, 151734, 152312, 152810, 152237, 153224, 153169, 153224, - 152244, 153387, 153404, 151670, 198, 16069, 155811, 151669, - 152265, 151946, 151808, 152412, 152363, 152305, 153156, 152733, - 152810, 153157, 152016, 152100, 152069, 153234, 152317, 152589, - 152707, 153121, 153341, 152159, 152114, 153156, 153001, 153504, - 153376, 152272, 152433, 152325, 151941, 151670, 198, 285, - 155788, 151669, 152238, 152255, 153427, 152318, 153009, 152381, - 152474, 152680, 152157, 153255, 152324, 151682, 151670, 198, - 32955, 155804, 151669, 153490, 153419, 152364, 152405, 152682, - 152206, 152078, 153369, 152725, 153193, 153027, 152946, 152488, - 153070, 151883, 152890, 152489, 153144, 153375, 152358, 151685, - 152494, 152117, 152740, 151670, 198, 37448, 480, 155840, 151669, - 151902, 152720, 153377, 152027, 152378, 152821, 153207, 153459, - 153028, 153068, 152507, 153255, 152158, 152921, 151958, 152609, - 152748, 152822, 152286, 151714, 152730, 152377, 152353, 152470, - 152606, 152162, 152186, 153071, 152244, 153118, 153375, 153018, - 152712, 153098, 152976, 152336, 151843, 153202, 152297, 151736, - 153380, 153502, 152702, 152115, 153181, 152735, 153277, 153457, - 152393, 153112, 152595, 151670, 198, 19098, 155808, 151669, - 152464, 153452, 152595, 153312, 151937, 151933, 153197, 152239, - 153163, 152922, 153402, 152034, 152591, 153438, 152215, 151673, - 152005, 151785, 152642, 151924, 153278, 151805, 151974, 153482, - 152718, 152862, 153347, 151670, 198, 72, 155780, 151669, 151795, - 152111, 152746, 152377, 153471, 152309, 151670, 198, 19016, - 155788, 151669, 153181, 152271, 152190, 152842, 152224, 152701, - 152939, 152536, 152091, 151815, 152733, 151672, 151670, 198, - 14689, 155788, 151669, 152291, 152072, 152942, 151734, 153042, - 153504, 152589, 153333, 151839, 151941, 153038, 153180, 151670, - 198, 36996, 8303, 155832, 151669, 152231, 152256, 152835, - 152801, 152985, 153400, 152393, 152818, 152765, 152249, 152600, - 151699, 152302, 152752, 153018, 153009, 151992, 153054, 152847, - 153354, 153228, 152662, 153355, 152532, 153393, 151782, 152458, - 152048, 152757, 152428, 153195, 151906, 153006, 153178, 153250, - 152331, 152284, 152780, 153138, 153319, 151980, 153142, 152418, - 152228, 152733, 151670, 198, 9096, 155801, 151669, 151698, - 153321, 152217, 153039, 152935, 153400, 152122, 152531, 153106, - 152169, 152892, 152957, 151851, 152427, 152826, 152451, 151851, - 152901, 152885, 152594, 153446, 153080, 151670, 198, 14689, - 155795, 151669, 152658, 151700, 153321, 152450, 152530, 153191, - 151673, 151690, 151698, 152714, 152846, 152981, 153171, 153384, - 153364, 153188, 153246, 151670, 198, 1055, 155779, 151669, - 151869, 152388, 152711, 153334, 151736, 151670, 198, 1782, - 155780, 151669, 153483, 153240, 152241, 152558, 152697, 153046, - 151670, 198, 5804, 1363, 155820, 151669, 152941, 152764, 152605, - 153034, 153434, 153372, 153347, 151887, 152453, 152758, 152133, - 152510, 152694, 152431, 152321, 153088, 152676, 152223, 152581, - 152459, 152015, 152502, 153063, 152712, 153294, 153451, 153032, - 152903, 152859, 152989, 151748, 152669, 152661, 152650, 152409, - 151861, 151670, 198, 300, 7973, 155828, 151669, 153095, 152469, - 152988, 152894, 151819, 152391, 153019, 152058, 153062, 153230, - 151826, 152112, 152306, 152264, 152769, 153390, 152384, 152435, - 152790, 153393, 152983, 152540, 152252, 152034, 153107, 152540, - 151919, 151893, 152558, 152817, 152946, 152956, 152129, 152715, - 153131, 153490, 151734, 152271, 152707, 151734, 153321, 152450, - 151670, 198, 8088, 155792, 151669, 152452, 153497, 153353, - 152679, 152533, 152382, 152374, 152611, 153341, 153163, 152285, - 153411, 152495, 153141, 152320, 151670, 198, 1199, 155781, - 151669, 151764, 152360, 153295, 152634, 153342, 152199, 152271, - 151670, 198, 43366, 155799, 151669, 152308, 151682, 152889, - 152016, 152385, 152629, 152495, 151826, 153321, 152958, 152180, - 151886, 153432, 152922, 152128, 153024, 153040, 152593, 152287, - 151677, 151670, 198, 53660, 155808, 151669, 151727, 152092, - 152680, 153331, 151699, 152316, 152938, 152289, 152433, 153384, - 151781, 153137, 153259, 152175, 153213, 152291, 151869, 152691, - 152489, 151941, 152049, 152034, 153053, 152179, 153160, 151676, - 153367, 151670, 198, 268, 4123, 480, 155821, 151669, 152350, - 152173, 152536, 151991, 151960, 153144, 153013, 152358, 152234, - 153135, 152291, 153235, 152143, 152583, 152402, 153483, 152678, - 152192, 152533, 152946, 151797, 153103, 152310, 152293, 151825, - 152548, 153442, 152109, 152659, 153325, 152781, 152570, 152957, - 151752, 152265, 153381, 152515, 151670, 198, 437, 155787, - 151669, 152957, 152659, 151975, 152709, 152402, 152836, 152174, - 151792, 153409, 153327, 152990, 151670, 198, 275, 155781, - 151669, 152520, 153038, 152067, 153273, 153185, 152265, 152974, - 151670, 198, 94273, 155799, 151669, 152953, 152938, 153427, - 152244, 151920, 153423, 152929, 152367, 153052, 152129, 152331, - 152257, 152987, 152777, 153448, 152408, 151696, 152408, 152326, - 152699, 151670, 198, 385, 16239, 155828, 151669, 152306, 152268, - 153438, 153228, 152978, 152957, 153153, 153393, 152795, 152110, - 152918, 152923, 152467, 152331, 153053, 153330, 151889, 153444, - 152234, 152624, 151779, 152801, 152784, 152139, 152222, 152751, - 152512, 153287, 153141, 153052, 151840, 152589, 152508, 153499, - 152109, 152255, 151739, 152267, 152759, 153318, 153165, 153349, - 151670,}); -#endif - } - - // print the prompt token-by-token - - LOG("\n"); - - for (auto id : prompt_inp) { - LOG("%s", common_token_to_piece(ctx_ttc, id).c_str()); - } + mtmd_helper::gen_audio gen(lctx, mctx.get()); + mtmd_helper_gen_audio_inp inp{}; + inp.seq_id = 0; + inp.prompt = params.prompt.c_str(); + inp.prompt_len = params.prompt.size(); + inp.speaker_ref = speaker_bitmap.get(); + inp.lang = params.tts_lang.c_str(); + inp.top_k = params.sampling.top_k; + inp.top_p = params.sampling.top_p; + inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; - LOG_INF("%s: prompt size: %d\n", __func__, (int) prompt_inp.size()); + // + // stage 1: process prompt via backbone model, generate semantic representation + // - LOG("\n"); - - // create a llama_batch - // we use this object to submit token data for decoding - llama_batch batch = llama_batch_init(std::max(prompt_inp.size(), (size_t) n_parallel), 0, n_parallel); - - std::vector seq_ids(n_parallel, 0); - for (int32_t i = 0; i < n_parallel; ++i) { - seq_ids[i] = i; - } - - // evaluate the initial prompt - for (size_t i = 0; i < prompt_inp.size(); ++i) { - common_batch_add(batch, prompt_inp[i], i, seq_ids, false); - } - GGML_ASSERT(batch.n_tokens == (int) prompt_inp.size()); + if (gen.set_input(&inp) != 0) { + LOG_ERR("set_input failed\n"); + return 1; + } - // llama_decode will output logits only for the last token of the prompt - batch.logits[batch.n_tokens - 1] = true; + const int64_t t_prompt_start_us = ggml_time_us(); - if (llama_decode(ctx_ttc, batch) != 0) { - LOG_ERR("%s: llama_decode() failed\n", __func__); + for (;;) { + int32_t ret = gen.step_prompt(params.n_batch); + if (ret < 0) { + LOG_ERR("prompt processing failed\n"); return 1; } - - if (n_parallel > 1) { - LOG_INF("\n\n%s: generating %d sequences ...\n", __func__, n_parallel); - } - - llama_synchronize(ctx_ttc); - - LOG_INF("%s: time for prompt: %.3f ms\n\n", __func__, (ggml_time_us() - t_main_start) / 1000.0f); - - const auto t_dec_start = ggml_time_us(); - - // main loop - - // remember the batch index of the last token for each parallel sequence - // we need this to determine which logits to sample from - std::vector i_batch(n_parallel, batch.n_tokens - 1); - - int n_past = batch.n_tokens; - int n_decode = 0; - - bool next_token_uses_guide_token = true; - - while (n_decode <= n_predict) { - // prepare the next batch - common_batch_clear(batch); - - // sample the next token for each parallel sequence / stream - for (int32_t i = 0; i < n_parallel; ++i) { - if (i_batch[i] < 0) { - // the stream has already finished - continue; - } - - llama_token new_token_id = common_sampler_sample(smpl[i], ctx_ttc, i_batch[i]); - - //guide tokens help prevent hallucinations by forcing the TTS to use the correct word - if (!guide_tokens.empty() && next_token_uses_guide_token && !llama_vocab_is_control(vocab, new_token_id) && !llama_vocab_is_eog(vocab, new_token_id)) { - llama_token guide_token = guide_tokens[0]; - guide_tokens.erase(guide_tokens.begin()); - new_token_id = guide_token; //ensure correct word fragment is used - } - - //this is the token id that always precedes a new word - next_token_uses_guide_token = (new_token_id == 198); - - common_sampler_accept(smpl[i], new_token_id, true); - - codes.push_back(new_token_id); - - const auto * cands = common_sampler_get_candidates(smpl[i], false); - - // is it an end of generation? -> mark the stream as finished - if (llama_vocab_is_eog(vocab, new_token_id) || n_decode == n_predict) { - std::string reason; - if (llama_vocab_is_eog(vocab, new_token_id)) { - reason = "eos"; - } else { - reason = "n_predict"; - } - - i_batch[i] = -1; - - LOG("\n"); - if (n_parallel > 1) { - LOG_CNT("\n"); - LOG_INF("%s: stream %d finished at n_past = %d, reason = '%s'\n", __func__, i, n_past, reason.c_str()); - } - - continue; - } - - { - const float p = cands->data[cands->selected].p; - - const int col = std::max(0, std::min((int) k_colors.size() - 1, (int) ((3*p)*float(k_colors.size())))); - - LOG_CNT("%s%d%s", k_colors[col].c_str(), i, "\033[0m"); - //LOG_CNT("%d", i); - } - - i_batch[i] = batch.n_tokens; - - // push this new token for next evaluation - common_batch_add(batch, new_token_id, n_past, { i }, true); - } - - // all streams are finished - if (batch.n_tokens == 0) { - break; - } - - n_decode += 1; - n_past += 1; - - // evaluate the current batch with the transformer model - if (llama_decode(ctx_ttc, batch)) { - LOG_ERR("%s : failed to eval, return code %d\n", __func__, 1); - return 1; - } + if (ret == 0) { + break; } - - llama_batch_free(batch); - - LOG("\n"); - LOG_INF("%s: time for decoder: %.3f ms\n", __func__, (ggml_time_us() - t_dec_start) / 1000.0f); } - common_perf_print(ctx_ttc, smpl[0]); + const llama_vocab * vocab = llama_model_get_vocab(model); - //std::vector codes = {198, 88225, 155856, 151669, 152205, - // 153064, 152537, 153421, 153209, 152524, 151689, 152993, 152438, 152695, - // 153091, 152945, 152829, 152534, 152934, 153020, 151997, 152263, 153010, - // 153146, 152399, 153208, 152496, 151793, 152848, 152263, 152571, 153286, - // 152227, 153300, 152934, 152263, 153208, 152263, 152965, 152430, 152296, - // 153146, 152920, 152376, 152556, 153363, 151775, 152044, 152972, 152690, - // 153379, 152368, 152233, 153422, 152490, 151996, 152022, 151694, 152061, - // 153238, 152539, 153356, 152640, 153021, 153123, 151962, 153094, 151670, - // 198, 20339, 13189, 155824, 151669, 152070, 152007, 152910, 151683, - // 152000, 152373, 152760, 152046, 151735, 152334, 152394, 153073, 152908, - // 151856, 151953, 153247, 153293, 151903, 153480, 153168, 152478, 153359, - // 153429, 151905, 151678, 152567, 152411, 152165, 152556, 153075, 153424, - // 151993, 152999, 153078, 152151, 152088, 153389, 152484, 151874, 151670, - // 198, 285, 155784, 151669, 152226, 152126, 152638, 153215, 151729, - // 152959, 153479, 153059, 151838, 151670, 198, 1782, 155783, 151669, - // 153288, 153055, 153314, 152497, 152962, 152741, 152076, 153253, 151670, - // 198, 471, 16488, 155825, 151669, 152060, 152916, 151893, 153469, 152501, - // 152080, 152743, 151932, 153161, 152096, 152761, 152698, 153401, 153242, - // 153336, 152441, 152838, 153467, 152706, 153496, 153310, 152422, 153360, - // 153115, 152763, 151998, 152373, 153450, 152554, 151968, 153323, 152055, - // 152468, 153111, 153358, 152813, 152010, 151770, 152823, 152960, 151670, - // 198, 22627, 155823, 151669, 152814, 152366, 153484, 152931, 153441, - // 152164, 152877, 152915, 153463, 151692, 152911, 152747, 152776, 151831, - // 153449, 151882, 152975, 152031, 152513, 153150, 152448, 152667, 153133, - // 153189, 152619, 153466, 152054, 152106, 153119, 152277, 152439, 153109, - // 152997, 152141, 153154, 153256, 153311, 151922, 151670, 198, 1055, - // 155781, 151669, 152633, 151850, 153060, 153270, 152560, 153348, 152729, - // 151670, 198, 25312, 155803, 151669, 152521, 153403, 152561, 153337, - // 153383, 152199, 153493, 153326, 151830, 152254, 152248, 152349, 152153, - // 153007, 151823, 153037, 152575, 152457, 152406, 152592, 153116, 153365, - // 153456, 151670, 198, 88225, 155817, 151669, 153271, 151925, 152218, - // 152418, 152253, 153140, 151903, 153151, 152626, 152338, 152647, 153464, - // 152785, 152768, 151711, 152037, 152033, 151804, 152216, 151701, 151855, - // 152348, 152995, 152955, 152905, 152342, 152340, 153391, 153453, 152418, - // 153415, 151990, 153083, 152884, 151670, 198, 151668, 198, 151645}; + auto sample_semantic_code = [&]() -> llama_token { + llama_token t = common_sampler_sample(smpl, lctx, -1); + common_sampler_accept(smpl, t, true); + return t; + }; - { - const std::string inp_txt = common_detokenize(ctx_ttc, codes, true); + const int max_new = params.n_predict > 0 ? params.n_predict : 512; + int n_frames = 0; + llama_token sampled = sample_semantic_code(); + const float * h_state = llama_get_embeddings_ith(lctx, -1); - LOG("\n"); - LOG_INF("codes: '%s'\n", inp_txt.c_str()); - LOG_INF("%s: codes size: %d\n", __func__, (int) codes.size()); - } - - // remove all non-audio tokens (i.e. < 151672 || > 155772) - codes.erase(std::remove_if(codes.begin(), codes.end(), [](llama_token t) { return t < 151672 || t > 155772; }), codes.end()); + tts_timings timings; + const int64_t t_gen_start_us = ggml_time_us(); - { - const std::string inp_txt = common_detokenize(ctx_ttc, codes, true); - LOG_INF("codes audio: '%s'\n", inp_txt.c_str()); - LOG_INF("%s: codes audio size: %d\n", __func__, (int) codes.size()); - } + for (; n_frames < max_new && !llama_vocab_is_eog(vocab, sampled); n_frames++) { + const float * h_next = nullptr; - for (auto & token : codes) { - token -= 151672; - } - - const auto t_voc_start = ggml_time_us(); - - const int n_codes = codes.size(); - - llama_batch batch = llama_batch_init(n_codes, 0, 1); - - for (size_t i = 0; i < codes.size(); ++i) { - common_batch_add(batch, codes[i], i, { 0 }, true); // TODO: all logits? - } - GGML_ASSERT(batch.n_tokens == n_codes); - - if (llama_encode(ctx_cts, batch) != 0) { - LOG_ERR("%s: llama_encode() failed\n", __func__); - return 1; - } - - llama_synchronize(ctx_cts); - - LOG_INF("%s: time for vocoder: %.3f ms\n", __func__, (ggml_time_us() - t_voc_start) / 1000.0f); - - const auto t_spec_start = ggml_time_us(); - -#if 1 - // spectral operations - const int n_embd = llama_model_n_embd_out(model_cts); - const float * embd = llama_get_embeddings(ctx_cts); - - auto audio = embd_to_audio(embd, n_codes, n_embd, params.cpuparams.n_threads); - -#else - // read the spectrogram from a file for debugging purposes - std::vector audio; - { - std::ifstream fin("out.bin", std::ios::binary); - if (!fin) { - LOG_ERR("%s: failed to open file '%s'\n", __func__, "out.bin"); + // stage 2+3: semantic --> acoustic details --> audio waveform + // step_gen() runs both stages and returns new h_state for next step + if (gen.step_gen(sampled, h_state, &h_next) != 0) { + LOG_ERR("step_gen failed at frame %d\n", n_frames); return 1; } - std::vector embd; - - int n_codes; - int n_embd; - - fin.read(reinterpret_cast(&n_codes), sizeof(int)); - fin.read(reinterpret_cast(&n_embd), sizeof(int)); - - embd.resize(n_codes * n_embd); - fin.read(reinterpret_cast(embd.data()), n_codes * n_embd * sizeof(float)); - fin.close(); - - LOG_INF("%s: n_codes: %d, n_embd: %d\n", __func__, n_codes, n_embd); - - audio = embd_to_audio(embd.data(), n_codes, n_embd, params.cpuparams.n_threads); + h_state = h_next; + sampled = sample_semantic_code(); + timings.report(n_frames + 1); } -#endif - - const int n_sr = 24000; // sampling rate + const double t_gen_s = (ggml_time_us() - t_gen_start_us) / 1e6; - // zero out first 0.25 seconds - for (int i = 0; i < 24000/4; ++i) { - audio[i] = 0.0f; + int32_t sample_rate = 0; + const char * data = nullptr; + size_t data_len = 0; + int64_t n_samples = 0; + if (gen.get_output(&sample_rate, &data, &data_len, &n_samples) != 0) { + LOG_ERR("get_output failed\n"); + return 1; } - LOG_INF("%s: time for spectral ops: %.3f ms\n", __func__, (ggml_time_us() - t_spec_start) / 1000.0f); - LOG_INF("%s: total time: %.3f ms\n", __func__, (ggml_time_us() - t_main_start) / 1000.0f); - - int retval = 0; + LOG_INF("generated %d frames, %zu bytes of WAV audio (%d Hz)\n", n_frames, data_len, sample_rate); - if (save_wav16(params.out_file, audio, n_sr)) { - LOG_INF("%s: audio written to file '%s'\n", __func__, params.out_file.c_str()); - } else { - retval = ENOENT; + const double t_prompt_s = (t_gen_start_us - t_prompt_start_us) / 1e6; + const double t_total_s = t_prompt_s + t_gen_s; + const double audio_s = sample_rate > 0 ? (double) n_samples / sample_rate : 0.0; + LOG_INF("timings: prompt eval %.2fs + generation %.2fs = total %.2fs\n", t_prompt_s, t_gen_s, t_total_s); + LOG_INF(" output audio = %.2fs (audio time = %.2fx process time)\n", audio_s, t_total_s > 0 ? audio_s / t_total_s : 0.0); + FILE * f = fopen(params.out_file.c_str(), "wb"); + if (!f) { + LOG_ERR("failed to open %s\n", params.out_file.c_str()); + return 1; } + fwrite(data, 1, data_len, f); + fclose(f); + LOG_INF("wrote %s\n", params.out_file.c_str()); llama_backend_free(); - - return retval; + return 0; } From 2f56fc3431f47fe042bf3825e4d5523bdddda993 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Tue, 4 Aug 2026 19:05:48 +0200 Subject: [PATCH 67/98] ui: CWD for agent (#26518) * server : extend file_glob_search for UI pickers * ui : add per-conversation working directory with picker * ui : add path navigation and search scope to cwd picker Treat path-like queries (starting with / or ~) as directory navigation instead of glob-matching the whole query: search the parent for the last segment, and descend into an exactly-typed directory by listing its children. Show the effective search scope in the footer and auto-search on open so the current directory and its siblings appear immediately. Assisted-by: Claude * db : persist per-call tool cwd on tool result messages * ui : abbreviate tool paths under home with a tilde * ui : show the per-call cwd on exec shell rows * ui : clarify the synthetic cwd message for the model * ui : reuse the trailing cwd row on a repeated pick * ui : don't jump when a cwd row is injected mid-chat * chore: Formatting * refactor: Cleanup comments * ui : unify working directory naming and add a synthetic-message flag * ui : render synthetic cwd rows without a scroll jump * ui : decouple the working directory picker into utils and sub-components * ui : add get_info tool call block * chore: Formatting * refactor: Cleanup * refactor: Cleanup * refactor: Cleanup * fix: UI * server : harden file_glob_search listing (kind enum, timeout, symlink guard, absolute base) * ui : use persisted isSynthetic flag for cwd rows, drop legacy formats * ui : cache picker search, fail visibly on native resolve * ui : escape glob metacharacters in picker search glob * ui : simplify auto-scroll pin * chore: Format * fix: Use `SvelteMap` * refactor: Post-review fixes * ui: accept Windows roots in the working directory picker recognize a drive root (C:) and a UNC share (//host/share) as path navigation, alongside the POSIX root and ~, so a query like D:\repos lists that directory instead of glob-matching it under the home dir split below the root, so a bare drive resolves to its root rather than to a drive-relative prefix rewrite backslashes into forward slashes only when the query carries a Windows root, since a backslash is a legal POSIX filename character paths keep travelling with forward slashes, which is what the server returns and what Windows accepts --------- Co-authored-by: Pascal --- tools/server/server-tools.cpp | 260 +++++++--- tools/server/tests/unit/test_tools_builtin.py | 98 ++++ tools/ui/src/app.d.ts | 9 + .../app/chat/ChatForm/ChatForm.svelte | 35 +- .../ChatForm/ChatFormWorkingDirectory.svelte | 479 ++++++++++++++++++ .../ChatFormWorkingDirectoryChip.svelte | 69 +++ ...ChatFormWorkingDirectoryResultsList.svelte | 72 +++ .../ChatMessage/ChatMessage.svelte | 21 +- .../ChatMessage/ChatMessageCwdChange.svelte | 31 ++ .../ChatMessage/ChatMessageSynthetic.svelte | 23 + .../ChatMessageToolCallBlock.svelte | 3 + .../ChatMessageToolCallBlockEditFile.svelte | 8 +- ...essageToolCallBlockExecShellCommand.svelte | 32 ++ ...tMessageToolCallBlockFileGlobSearch.svelte | 8 +- .../ChatMessageToolCallBlockGetInfo.svelte | 69 +++ .../ChatMessageToolCallBlockGrepSearch.svelte | 6 +- .../ChatMessageToolCallBlockWriteFile.svelte | 8 +- tools/ui/src/lib/components/app/chat/index.ts | 26 + tools/ui/src/lib/constants/built-in-tools.ts | 2 + tools/ui/src/lib/constants/index.ts | 2 + tools/ui/src/lib/constants/path-display.ts | 22 + tools/ui/src/lib/constants/tools.ts | 3 + .../ui/src/lib/constants/working-directory.ts | 40 ++ tools/ui/src/lib/enums/index.ts | 8 +- tools/ui/src/lib/enums/tools.enums.ts | 11 + tools/ui/src/lib/enums/ui.enums.ts | 1 + tools/ui/src/lib/services/database.service.ts | 3 +- tools/ui/src/lib/services/tools.service.ts | 33 +- tools/ui/src/lib/stores/agentic.svelte.ts | 9 +- tools/ui/src/lib/stores/chat.svelte.ts | 64 ++- .../ui/src/lib/stores/conversations.svelte.ts | 57 ++- tools/ui/src/lib/stores/tools.svelte.ts | 38 +- tools/ui/src/lib/types/agentic.d.ts | 3 +- tools/ui/src/lib/types/chat.d.ts | 3 +- tools/ui/src/lib/types/database.d.ts | 5 + tools/ui/src/lib/utils/agentic.ts | 4 + tools/ui/src/lib/utils/index.ts | 23 + tools/ui/src/lib/utils/path-display.ts | 93 ++++ tools/ui/src/lib/utils/working-directory.ts | 151 ++++++ tools/ui/tests/unit/tool-calls.test.ts | 85 ++++ tools/ui/tests/unit/working-directory.test.ts | 126 +++++ 41 files changed, 1947 insertions(+), 96 deletions(-) create mode 100644 tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectory.svelte create mode 100644 tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectoryChip.svelte create mode 100644 tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectoryResultsList.svelte create mode 100644 tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageCwdChange.svelte create mode 100644 tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSynthetic.svelte create mode 100644 tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockGetInfo.svelte create mode 100644 tools/ui/src/lib/constants/path-display.ts create mode 100644 tools/ui/src/lib/constants/working-directory.ts create mode 100644 tools/ui/src/lib/utils/path-display.ts create mode 100644 tools/ui/src/lib/utils/working-directory.ts create mode 100644 tools/ui/tests/unit/working-directory.test.ts diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index 984bb478ea23..e050d03b5288 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -10,8 +10,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -34,7 +36,40 @@ json server_tool::to_json() const { } static constexpr size_t SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT = 8 * 1024 * 1024; // 8 MB -static constexpr int SERVER_TOOL_GIT_LS_FILES_TIMEOUT = 15; // seconds +// budget for one listing call, shared by the git and walker paths +static constexpr int SERVER_TOOL_LIST_ENTRIES_TIMEOUT = 15; // seconds + +// entry kinds a directory listing may return +enum class list_kind { + files, // regular files only + dirs, // directories only + all, // both +}; + +// home directory, read once at first use (getenv is not thread safe against setenv) +static const std::string & home_dir() { + static const std::string home = [] { + const char * h = getenv("HOME"); +#ifdef _WIN32 + if (h == nullptr) h = getenv("USERPROFILE"); +#endif + return h ? std::string(h) : std::string(); + }(); + return home; +} + +static std::string expand_home(const std::string & path) { + if (path.empty() || path[0] != '~') return path; + if (path.size() > 1 && path[1] != '/' && path[1] != '\\') return path; + const std::string & home = home_dir(); + if (home.empty()) return path; + return home + path.substr(1); +} + +// depth of a '/'-separated relative path: "a/b/c" is 3 +static int entry_depth(const std::string & rel) { + return 1 + (int) std::count(rel.begin(), rel.end(), '/'); +} class tools_io { public: @@ -51,8 +86,17 @@ class tools_io { virtual bool file_size(const std::string & path, uintmax_t & out_size) const = 0; virtual bool read_file(const std::string & path, std::string & out) const = 0; virtual bool write_file(const std::string & path, const std::string & content) const = 0; - // paths relative to `base`, '/'-separated; sets `err` if `base` isn't a directory - virtual std::vector list_files(const std::string & base, std::string & err) const = 0; + // resolve `path` against the IO's working directory; absolute paths are returned unchanged + virtual std::string resolve(const std::string & path) const = 0; + struct list_entry { + std::string rel; // '/'-separated, relative to `base` + bool is_dir = false; + }; + // entries relative to `base`; sets `err` if `base` isn't a directory + // max_depth == 0 means unlimited, 1 means direct children of `base` only + // `base` must already be resolved (absolute); `caller_path` is the path the + // caller passed, used only for error messages + virtual std::vector list_entries(const std::string & base, const std::string & caller_path, int max_depth, list_kind kind, std::string & err, bool & truncated) const = 0; // on_chunk, if set, is called with each chunk of output as it is read (before truncation cuts in); // returning false terminates the process early (e.g. the client disconnected) virtual exec_result run( @@ -67,6 +111,22 @@ class tools_io_basic : public tools_io { // cwd, if non-empty, is used to resolve relative paths and as the working directory for run() explicit tools_io_basic(std::string cwd = "") : cwd(std::move(cwd)) {} + // expands a leading `~`, then resolves `path` against `cwd` (or the server + // working directory when `cwd` is unset); the result is always absolute + std::string resolve(const std::string & path) const override { + std::string p = expand_home(path); + if (fs::path(p).is_absolute()) { + return p; + } + if (cwd.empty()) { + std::error_code ec; + fs::path cur = fs::current_path(ec); + if (ec) return p; + return (cur / p).string(); + } + return (fs::path(cwd) / p).string(); + } + bool is_directory(const std::string & path) const override { std::error_code ec; return fs::is_directory(resolve(path), ec) && !ec; @@ -105,34 +165,41 @@ class tools_io_basic : public tools_io { return (bool) f; } - std::vector list_files(const std::string & base, std::string & err) const override { + std::vector list_entries(const std::string & base, const std::string & caller_path, int max_depth, list_kind kind, std::string & err, bool & truncated) const override { err.clear(); - std::string abs_base = resolve(base); - if (!is_directory(base)) { - err = "path does not exist or is not a directory: " + base; + truncated = false; + std::error_code ec; + if (!fs::is_directory(base, ec) || ec) { + err = "path does not exist or is not a directory: " + caller_path; return {}; } - auto res = run( - {"git", "-C", abs_base, "ls-files", "--cached", "--others", "--exclude-standard"}, - SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, SERVER_TOOL_GIT_LS_FILES_TIMEOUT); - - if (res.exit_code == 0 && !res.timed_out) { - std::vector result; - std::istringstream iss(res.output); - std::string line; - while (std::getline(iss, line)) { - if (!line.empty() && line.back() == '\r') line.pop_back(); - if (line.empty()) continue; - std::replace(line.begin(), line.end(), '\\', '/'); - if (is_regular_file((fs::path(base) / line).string())) { - result.push_back(line); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(SERVER_TOOL_LIST_ENTRIES_TIMEOUT); + + // git ls-files cannot list directories; use the walker when they are requested + if (kind == list_kind::files) { + auto res = run( + {"git", "-C", base, "ls-files", "--cached", "--others", "--exclude-standard"}, + SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, SERVER_TOOL_LIST_ENTRIES_TIMEOUT); + + if (res.exit_code == 0 && !res.timed_out) { + std::vector result; + std::istringstream iss(res.output); + std::string line; + while (std::getline(iss, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.empty()) continue; + std::replace(line.begin(), line.end(), '\\', '/'); + if (max_depth > 0 && entry_depth(line) > max_depth) continue; + if (is_regular_file((fs::path(base) / line).string())) { + result.push_back({line, false}); + } } + return result; } - return result; } - return list_files_fallback(abs_base); + return list_entries_fallback(base, max_depth, kind, deadline, truncated); } exec_result run( @@ -211,14 +278,6 @@ class tools_io_basic : public tools_io { private: std::string cwd; - // resolves `path` against `cwd` if `path` is relative and `cwd` is set; otherwise returns `path` unchanged - std::string resolve(const std::string & path) const { - if (cwd.empty() || fs::path(path).is_absolute()) { - return path; - } - return (fs::path(cwd) / path).string(); - } - static const std::unordered_set & junk_dir_names() { static const std::unordered_set names = { ".git", ".svn", ".hg", "node_modules", "__pycache__", @@ -227,28 +286,50 @@ class tools_io_basic : public tools_io { return names; } - std::vector list_files_fallback(const std::string & base) const { - std::vector result; + std::vector list_entries_fallback(const std::string & base, int max_depth, list_kind kind, + std::chrono::steady_clock::time_point deadline, bool & truncated) const { + std::vector result; std::error_code ec; - std::vector> stack; - stack.emplace_back(fs::path(base), fs::path()); + std::vector> stack; + stack.emplace_back(fs::path(base), fs::path(), 0); while (!stack.empty()) { - auto [dir, rel_dir] = stack.back(); + auto [dir, rel_dir, depth] = stack.back(); stack.pop_back(); - for (const auto & entry : fs::directory_iterator(dir, fs::directory_options::skip_permission_denied, ec)) { + // the throwing increment would escape the tool on a directory that + // goes away mid walk, so step the iterator explicitly + fs::directory_iterator it(dir, fs::directory_options::skip_permission_denied, ec); + for (const fs::directory_iterator end; it != end; it.increment(ec)) { if (ec) break; + if (std::chrono::steady_clock::now() >= deadline) { + truncated = true; + return result; + } + const fs::directory_entry & entry = *it; std::string fname = entry.path().filename().string(); std::error_code tec; if (entry.is_directory(tec)) { + std::string rel = (rel_dir / fname).string(); + std::replace(rel.begin(), rel.end(), '\\', '/'); + if (kind == list_kind::dirs || kind == list_kind::all) { + result.push_back({rel, true}); + } + // junk directories stay selectable but are never walked: they + // hold nothing worth searching and can be enormous if (junk_dir_names().count(fname) > 0) continue; - stack.emplace_back(entry.path(), rel_dir / fname); + // do not descend into symlinks: a link can point back to an + // ancestor and loop forever + if (!entry.is_symlink(tec) && (max_depth == 0 || depth + 1 < max_depth)) { + stack.emplace_back(entry.path(), rel_dir / fname, depth + 1); + } } else if (entry.is_regular_file(tec)) { std::string rel = (rel_dir / fname).string(); std::replace(rel.begin(), rel.end(), '\\', '/'); - result.push_back(rel); + if (kind == list_kind::files || kind == list_kind::all) { + result.push_back({rel, false}); + } } } } @@ -363,6 +444,9 @@ struct server_tool_read_file : server_tool { // static constexpr size_t SERVER_TOOL_FILE_SEARCH_MAX_RESULTS = 100; +static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_FILE = "file"; +static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_DIR = "dir"; +static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_ALL = "all"; struct server_tool_file_glob_search : server_tool { server_tool_file_glob_search() { @@ -382,13 +466,18 @@ struct server_tool_file_glob_search : server_tool { "and common junk directories (.git, node_modules, build, dist, etc.) otherwise. " "A pattern with no '/' (e.g. \"*.cpp\") matches the file's basename at any depth. " "A pattern containing '/' matches the full relative path; unless already anchored with " - "\"**/\" or a leading '/', it is automatically prefixed with \"**/\"."}, + "\"**/\" or a leading '/', it is automatically prefixed with \"**/\". " + "Use type=\"dir\" or \"all\" to also list directories; directory entries are suffixed with '/' in the output. " + "Note: directory listings do not apply .gitignore filtering."}, {"parameters", { {"type", "object"}, {"properties", { - {"path", {{"type", "string"}, {"description", "Base directory to search in"}}}, - {"include", {{"type", "string"}, {"description", "Glob pattern for files to include (e.g. \"*.cpp\" or \"src/**/*.cpp\"). Default: **"}}}, - {"exclude", {{"type", "string"}, {"description", "Glob pattern for files to exclude"}}}, + {"path", {{"type", "string"}, {"description", "Base directory to search in"}}}, + {"include", {{"type", "string"}, {"description", "Glob pattern for files to include (e.g. \"*.cpp\" or \"src/**/*.cpp\"). Default: **"}}}, + {"exclude", {{"type", "string"}, {"description", "Glob pattern for files to exclude"}}}, + {"type", {{"type", "string"}, {"description", "Entry type to return: \"file\" (default), \"dir\" or \"all\""}}}, + {"max_depth", {{"type", "integer"}, {"description", "Maximum depth to descend into subdirectories (default: 0 = unlimited; 1 = direct children only)"}}}, + {"limit", {{"type", "integer"}, {"description", string_format("Maximum number of results to return (default %zu; values below 1 fall back to the default)", SERVER_TOOL_FILE_SEARCH_MAX_RESULTS)}}}, }}, {"required", json::array({"path"})}, }}, @@ -397,30 +486,56 @@ struct server_tool_file_glob_search : server_tool { } json invoke(json params, server_tool::stream *) const override { - std::string base = params.at("path").get(); - std::string include = json_value(params, "include", std::string("**")); - std::string exclude = json_value(params, "exclude", std::string("")); - auto io = make_tools_io(params); + + std::string base = io->resolve(params.at("path").get()); + // normalize to forward slashes so the web UI (which assumes '/') can + // join the relative entries into absolute paths on Windows too + std::replace(base.begin(), base.end(), '\\', '/'); + std::string include = json_value(params, "include", std::string("**")); + std::string exclude = json_value(params, "exclude", std::string("")); + std::string type = json_value(params, "type", std::string("file")); + int max_depth = std::max(0, json_value(params, "max_depth", 0)); + int limit = json_value(params, "limit", (int) SERVER_TOOL_FILE_SEARCH_MAX_RESULTS); + if (limit < 1) limit = SERVER_TOOL_FILE_SEARCH_MAX_RESULTS; + limit = std::min(limit, (int) SERVER_TOOL_FILE_SEARCH_MAX_RESULTS); + + list_kind kind; + if (type == SERVER_TOOL_FILE_SEARCH_TYPE_FILE) { + kind = list_kind::files; + } else if (type == SERVER_TOOL_FILE_SEARCH_TYPE_DIR) { + kind = list_kind::dirs; + } else if (type == SERVER_TOOL_FILE_SEARCH_TYPE_ALL) { + kind = list_kind::all; + } else { + return {{"error", "invalid type: " + type + " (expected \"file\", \"dir\" or \"all\")"}}; + } + std::string err; - auto files = io->list_files(base, err); + bool truncated = false; + auto entries = io->list_entries(base, params.at("path").get(), max_depth, kind, err, truncated); if (!err.empty()) { return {{"error", err}}; } - std::vector matches; - for (const auto & rel : files) { - if (!path_glob_match(include, rel)) continue; - if (!exclude.empty() && path_glob_match(exclude, rel)) continue; - matches.push_back(rel); + std::vector matches; + for (const auto & entry : entries) { + if (!path_glob_match(include, entry.rel)) continue; + if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue; + matches.push_back(entry); } size_t total = matches.size(); - size_t shown = std::min(total, SERVER_TOOL_FILE_SEARCH_MAX_RESULTS); + size_t shown = std::min(total, (size_t) limit); std::ostringstream output_text; + json entries_json = json::array(); for (size_t i = 0; i < shown; i++) { - output_text << matches[i] << "\n"; + output_text << matches[i].rel << (matches[i].is_dir ? "/" : "") << "\n"; + entries_json.push_back({ + {"path", matches[i].rel}, + {"type", matches[i].is_dir ? "dir" : "file"}, + }); } output_text << "\n---\nTotal matches: " << total << "\n"; @@ -429,8 +544,16 @@ struct server_tool_file_glob_search : server_tool { "[%zu results limit reached (%zu total matches). Refine the glob pattern to narrow the search.]\n", shown, total); } + if (truncated) { + output_text << "[search timed out, results truncated]\n"; + } - return {{"plain_text_response", output_text.str()}}; + // `base` is always absolute (resolve falls back to the server cwd), so + // API clients (e.g. the web UI picker) can join the relative entries + // into absolute paths. `plain_text_response` is what the model sees; + // `entries` is the same data as structured JSON for the UI picker, + // which reads `entries`/`base` instead of re-parsing the text. + return {{"plain_text_response", output_text.str()}, {"entries", entries_json}, {"base", base}}; } }; @@ -513,18 +636,20 @@ struct server_tool_grep_search : server_tool { // collect (absolute_path, display_path) pairs to search std::vector> files; - if (io->is_regular_file(path)) { - files.emplace_back(path, path); - } else if (io->is_directory(path)) { + const std::string abs_path = io->resolve(path); + if (io->is_regular_file(abs_path)) { + files.emplace_back(abs_path, path); + } else if (io->is_directory(abs_path)) { std::string err; - auto candidates = io->list_files(path, err); + bool truncated = false; + auto candidates = io->list_entries(abs_path, path, 0, list_kind::files, err, truncated); if (!err.empty()) { return {{"error", err}}; } - for (const auto & rel : candidates) { - if (!path_glob_match(include, rel)) continue; - if (!exclude.empty() && path_glob_match(exclude, rel)) continue; - files.emplace_back((fs::path(path) / rel).string(), rel); + for (const auto & entry : candidates) { + if (!path_glob_match(include, entry.rel)) continue; + if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue; + files.emplace_back((fs::path(abs_path) / entry.rel).string(), entry.rel); } } else { return {{"error", "path does not exist: " + path}}; @@ -1094,6 +1219,9 @@ struct server_tool_get_datetime : server_tool { // get_info: returns runtime info (OS name/version and cwd) // +static constexpr size_t SERVER_TOOL_GET_INFO_MAX_OUTPUT = 4096; +static constexpr int SERVER_TOOL_GET_INFO_TIMEOUT = 5; // seconds + struct server_tool_get_info : server_tool { server_tool_get_info() { name = "get_info"; @@ -1119,9 +1247,9 @@ struct server_tool_get_info : server_tool { auto io = make_tools_io(params); #ifdef _WIN32 - auto res = io->run({"cmd", "/c", "ver"}, 4096, 5); + auto res = io->run({"cmd", "/c", "ver"}, SERVER_TOOL_GET_INFO_MAX_OUTPUT, SERVER_TOOL_GET_INFO_TIMEOUT); #else - auto res = io->run({"uname", "-a"}, 4096, 5); + auto res = io->run({"uname", "-a"}, SERVER_TOOL_GET_INFO_MAX_OUTPUT, SERVER_TOOL_GET_INFO_TIMEOUT); #endif // "ver" prints a blank line before the version, so the output is stripped on both ends; // a failed spawn or a timeout leaves a diagnostic in res.output, which is not an OS name diff --git a/tools/server/tests/unit/test_tools_builtin.py b/tools/server/tests/unit/test_tools_builtin.py index fb194cac669a..e713758d91df 100755 --- a/tools/server/tests/unit/test_tools_builtin.py +++ b/tools/server/tests/unit/test_tools_builtin.py @@ -164,3 +164,101 @@ def test_tools_builtin_edit_file_rejects_overlapping_edits(): finally: if os.path.exists(log_path): os.remove(log_path) + + +def test_tools_builtin_file_glob_search_type_dir(tmp_path): + global server + server.start() + + (tmp_path / "project-alpha" / "src").mkdir(parents=True) + (tmp_path / "project-alpha" / "README.md").write_text("alpha") + (tmp_path / "project-alpha" / "src" / "main.cpp").write_text("int main() {}") + (tmp_path / "project-beta").mkdir() + (tmp_path / "project-beta" / "notes.txt").write_text("beta") + + res = call_tool("file_glob_search", {"path": str(tmp_path), "type": "dir"}) + text = res["plain_text_response"] + assert "project-alpha/" in text + assert "project-beta/" in text + assert "project-alpha/src/" in text + assert "README.md" not in text + types = {e["path"]: e["type"] for e in res["entries"]} + assert types["project-alpha"] == "dir" + assert types["project-alpha/src"] == "dir" + + res_all = call_tool("file_glob_search", {"path": str(tmp_path), "type": "all", "include": "*proj*"}) + paths = [e["path"] for e in res_all["entries"]] + assert "project-alpha" in paths + assert "project-beta" in paths + + +def test_tools_builtin_file_glob_search_max_depth_and_limit(tmp_path): + global server + server.start() + + (tmp_path / "a" / "b" / "c").mkdir(parents=True) + (tmp_path / "top.txt").write_text("top") + (tmp_path / "a" / "mid.txt").write_text("mid") + (tmp_path / "a" / "b" / "deep.txt").write_text("deep") + + res = call_tool("file_glob_search", {"path": str(tmp_path), "max_depth": 1}) + assert "top.txt" in res["plain_text_response"] + assert "mid.txt" not in res["plain_text_response"] + + res = call_tool("file_glob_search", {"path": str(tmp_path), "max_depth": 2}) + assert "mid.txt" in res["plain_text_response"] + assert "deep.txt" not in res["plain_text_response"] + + res = call_tool("file_glob_search", {"path": str(tmp_path), "limit": 1}) + assert len(res["entries"]) == 1 + assert "Total matches: 3" in res["plain_text_response"] + + +def test_tools_builtin_file_glob_search_rejects_invalid_type(tmp_path): + global server + server.start() + + err = call_tool_expect_error("file_glob_search", {"path": str(tmp_path), "type": "bogus"}) + assert "invalid type" in err + + +def test_tools_builtin_cwd_header_overrides_model_param(tmp_path): + global server + server.start() + + workdir = tmp_path / "workdir" + workdir.mkdir() + (workdir / "marker.txt").write_text("marker") + + # a model-provided "cwd" in the params is overridden by the x-tool-cwd header + res = call_tool("read_file", {"path": "marker.txt", "cwd": "/definitely/not/a/real/path"}, + headers={"x-tool-cwd": str(workdir)}) + assert "marker" in res["plain_text_response"] + + +def test_tools_builtin_cwd_relative_paths(tmp_path): + global server + server.start() + + workdir = tmp_path / "workdir" + workdir.mkdir() + (workdir / "rel.txt").write_text("relative-content") + + headers = {"x-tool-cwd": str(workdir)} + + # relative paths in file tools resolve against the header cwd + res = call_tool("read_file", {"path": "rel.txt"}, headers=headers) + assert "relative-content" in res["plain_text_response"] + + res = call_tool("write_file", {"path": "sub/out.txt", "content": "written"}, headers=headers) + assert (workdir / "sub" / "out.txt").read_text() == "written" + + res = call_tool("file_glob_search", {"path": ".", "include": "*.txt"}, headers=headers) + assert "rel.txt" in res["plain_text_response"] + + # absolute paths are unaffected by the cwd + other = tmp_path / "other" + other.mkdir() + (other / "abs.txt").write_text("absolute-content") + res = call_tool("read_file", {"path": str(other / "abs.txt")}, headers=headers) + assert "absolute-content" in res["plain_text_response"] diff --git a/tools/ui/src/app.d.ts b/tools/ui/src/app.d.ts index 5264e5cc4d72..b9484d95031c 100644 --- a/tools/ui/src/app.d.ts +++ b/tools/ui/src/app.d.ts @@ -142,5 +142,14 @@ declare global { interface Window { idxThemeStyle?: number; idxCodeBlock?: number; + + // File System Access API - missing from older DOM lib versions. + // Used by ChatFormWorkingDirectory's native folder picker. Feature availability + // is gated at runtime via `typeof window.showDirectoryPicker === 'function'`. + showDirectoryPicker: (options?: { + id?: string; + mode?: 'read' | 'readwrite'; + startIn?: FileSystemHandle | string; + }) => Promise; } } diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index 85683908cc71..105e414fe78f 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -6,6 +6,7 @@ ChatFormMcpResourcesList, ChatFormPickers, ChatFormTextarea, + ChatFormWorkingDirectory, DialogMcpResourcesBrowser } from '$lib/components/app'; import { @@ -31,7 +32,13 @@ import { chatStore } from '$lib/stores/chat.svelte'; import { mcpStore } from '$lib/stores/mcp.svelte'; import { mcpHasResourceAttachments } from '$lib/stores/mcp-resources.svelte'; - import { conversationsStore, activeMessages } from '$lib/stores/conversations.svelte'; + import { toolsStore } from '$lib/stores/tools.svelte'; + import { + conversationsStore, + activeMessages, + activeConversation, + pendingCwd + } from '$lib/stores/conversations.svelte'; import type { GetPromptResult, MCPPromptInfo, MCPResourceInfo, PromptMessage } from '$lib/types'; import { isIMEComposing, parseClipboardContent, uuid } from '$lib/utils'; import { @@ -107,6 +114,15 @@ let isInlineResourcePickerOpen = $state(false); let resourceSearchQuery = $state(''); + let cwd = $derived(activeConversation()?.cwd ?? pendingCwd()); + + async function handleWorkingDirectoryChange(value: string | null) { + await conversationsStore.setCwd(value); + if (conversationsStore.activeConversation) { + await chatStore.recordCwdChange(value?.trim() || null); + } + } + // Resource Dialog State let isResourceDialogOpen = $state(false); let preSelectedResourceUri = $state(undefined); @@ -155,6 +171,12 @@ audioRecorder = new AudioRecorder(); }); + // Defer so the closing popover's focus scope tears down first - bits-ui + // yanks a synchronous focus() back into the still-mounted popover. + function refocusInput() { + queueMicrotask(() => textareaRef?.focus()); + } + export function focus() { textareaRef?.focus(); } @@ -470,7 +492,7 @@
{ event.preventDefault(); @@ -559,6 +581,15 @@ + + {#if toolsStore.builtinTools.length > 0} + + {/if} + import { FolderOpen } from '@lucide/svelte'; + import { untrack } from 'svelte'; + import { SvelteMap } from 'svelte/reactivity'; + import { ToolsService } from '$lib/services/tools.service'; + import { toolsStore } from '$lib/stores/tools.svelte'; + import { BuiltInTool, GlobSearchType, KeyboardKey } from '$lib/enums'; + import { + abbreviateHome, + buildCaseInsensitiveGlob, + joinPath, + lastPathSegment, + rankEntries, + splitPathQuery, + type GlobEntry + } from '$lib/utils'; + import { debounce } from '$lib/utils/debounce'; + import * as Popover from '$lib/components/ui/popover'; + import SearchInput from '$lib/components/app/forms/SearchInput.svelte'; + import ChatFormWorkingDirectoryChip from './ChatFormWorkingDirectoryChip.svelte'; + import ChatFormWorkingDirectoryResultsList from './ChatFormWorkingDirectoryResultsList.svelte'; + import { + DEFAULT_MOBILE_BREAKPOINT, + GLOB_WILDCARD, + HOME_TILDE, + MAX_RESULTS_SHOWN, + NATIVE_LIMIT, + NATIVE_MAX_DEPTH, + PATH_NAV_MAX_DEPTH, + SEARCH_DEBOUNCE_MS, + SEARCH_LIMIT, + SEARCH_MAX_DEPTH + } from '$lib/constants'; + + // Microtask delay so the popover's focus scope tears down first. + const FOCUS_DELAY_MS = 0; + + interface Props { + class?: string; + disabled?: boolean; + directory?: string | null; + onChange?: (directory: string | null) => void; + /** + * Lets the host refocus the chat input so typing can resume without + * an extra click after the popover closes. + */ + onClose?: () => void; + } + + let { + class: className = '', + disabled = false, + directory = $bindable(null), + onChange, + onClose + }: Props = $props(); + + // File System Access API is opt-in: when available (Chrome / Edge / Opera) the popover + // exposes a "Browse" button that opens the native folder picker. When unavailable the + // popover still works via the text input - no alerts, no upload semantics. + const pickerSupported = + typeof window !== 'undefined' && typeof window.showDirectoryPicker === 'function'; + + // Popover open state; the element handles outside-click and Escape. + let isOpen = $state(false); + let inputValue = $state(''); + let searchInputRef: HTMLInputElement | null = $state(null); + + let queryResults = $state([]); + let isSearching = $state(false); + let searchError = $state(null); + let hoveredIndex = $state(-1); + // Bumped only by ArrowUp/ArrowDown handlers; the list scrolls the + // highlighted row into view only via this trigger, never on hover. + let scrollTrigger = $state(0); + let listContainer = $state(null); + + // Absolute home directory on the server, resolved once per session by + // the tools store. Anchors both the search scope and the chip's `~` + // abbreviation. + let homeBase = $derived(toolsStore.serverHome); + + // AbortController + sequence counter to discard stale responses when the user + // keeps typing; a newer call aborts the previous one. The sequence counter + // also covers the gap between abort and the catch handler. + let searchController: AbortController | null = null; + let searchSeq = 0; + + // Cache of the last file_glob_search result per (parent, include, max_depth), + // so repeated queries in the same directory don't re-walk the tree. Entries + // expire after a short TTL. + const SEARCH_CACHE_TTL_MS = 2000; + const searchCache = new SvelteMap(); + + const runSearch = debounce((query: string) => { + void doSearch(query); + }, SEARCH_DEBOUNCE_MS); + + // Resolve home eagerly on mount so the chip can abbreviate before the + // user opens the picker. resolveServerHome() is cached, so repeat calls + // (e.g. from handleOpenChange) are no-ops. + $effect(() => { + if (typeof window === 'undefined') return; + void toolsStore.resolveServerHome(); + }); + + // Auto-focus the search input when the popover opens. + // HTML `autofocus` is unreliable on dynamically shown elements, so we + // use a microtask (0ms setTimeout) after the effect flushes. + $effect(() => { + if (!isOpen) return; + setTimeout(() => searchInputRef?.focus(), FOCUS_DELAY_MS); + }); + + let lastScrollTrigger: number | null = null; + + // hoveredIndex/queryResults are untracked so hover and result replacement + // never re-fire the scroll; keyboard nav is the only path that bumps the trigger + $effect(() => { + if (scrollTrigger === lastScrollTrigger) return; + lastScrollTrigger = scrollTrigger; + untrack(() => { + if (!listContainer) return; + if (hoveredIndex < 0 || hoveredIndex >= queryResults.length) return; + const selectedElement = listContainer.querySelector( + `[data-result-index="${hoveredIndex}"]` + ) as HTMLElement | null; + selectedElement?.scrollIntoView({ block: 'nearest', inline: 'nearest' }); + }); + }); + + function cancelSearch() { + searchController?.abort(); + searchSeq++; + isSearching = false; + } + + // Effective directory the current search runs against (shown in the + // footer); updated by doSearch, including when an exactly-typed + // directory is "entered". + let searchScope = $state(HOME_TILDE); + + // Runs a directory listing through the cache, so a repeated query in the + // same directory does not re-walk the tree on the server. + async function searchDirs( + path: string, + include: string, + maxDepth: number, + signal: AbortSignal + ): Promise<{ base: string; entries: GlobEntry[]; error?: string }> { + const key = `${path}\u0000${include}\u0000${maxDepth}`; + const cached = searchCache.get(key); + if (cached && Date.now() - cached.at < SEARCH_CACHE_TTL_MS) { + return { base: cached.base, entries: cached.results }; + } + const res = await ToolsService.executeToolRaw( + BuiltInTool.FILE_GLOB_SEARCH, + { path, type: GlobSearchType.DIR, include, max_depth: maxDepth, limit: SEARCH_LIMIT }, + signal + ); + if (typeof res.error === 'string') return { base: '', entries: [], error: res.error }; + const base = typeof res.base === 'string' ? res.base : ''; + const entries = Array.isArray(res.entries) ? (res.entries as GlobEntry[]) : []; + searchCache.set(key, { results: entries, base, at: Date.now() }); + return { base, entries }; + } + + async function doSearch(query: string) { + const trimmed = query.trim(); + if (!trimmed) { + queryResults = []; + searchError = null; + isSearching = false; + hoveredIndex = -1; + searchScope = homeBase ?? HOME_TILDE; + return; + } + + cancelSearch(); + const controller = new AbortController(); + searchController = controller; + const mySeq = ++searchSeq; + + const pathQuery = splitPathQuery(trimmed); + + isSearching = true; + try { + // A generous limit is requested because ranking happens + // client-side; only the top 20 are shown. + const searchPath = pathQuery ? pathQuery.parent : (homeBase ?? HOME_TILDE); + const include = pathQuery + ? pathQuery.last + ? buildCaseInsensitiveGlob(pathQuery.last) + : GLOB_WILDCARD + : buildCaseInsensitiveGlob(trimmed); + const maxDepth = pathQuery ? PATH_NAV_MAX_DEPTH : SEARCH_MAX_DEPTH; + const res = await searchDirs(searchPath, include, maxDepth, controller.signal); + if (mySeq !== searchSeq) return; + if (res.error) { + queryResults = []; + hoveredIndex = -1; + searchError = res.error; + return; + } + const { base, entries } = res; + const ranked = rankEntries(entries, pathQuery?.last ?? trimmed); + let results = ranked.map((e) => joinPath(base, e.path)); + searchScope = pathQuery ? pathQuery.parent : (homeBase ?? HOME_TILDE); + + // An exactly-typed directory is "entered": list its children too, + // so path navigation doesn't require a trailing slash. + const last = pathQuery?.last; + const exact = last + ? ranked.find((e) => lastPathSegment(e.path).toLowerCase() === last.toLowerCase()) + : undefined; + if (exact) { + const exactDir = joinPath(base, exact.path); + const childRes = await searchDirs( + exactDir, + GLOB_WILDCARD, + PATH_NAV_MAX_DEPTH, + controller.signal + ); + if (mySeq !== searchSeq) return; + if (!childRes.error) { + const children = childRes.entries + .map((e) => joinPath(childRes.base, e.path)) + .sort((a, b) => a.localeCompare(b)); + results = [...results, ...children]; + searchScope = exactDir; + } + } + + queryResults = results.slice(0, MAX_RESULTS_SHOWN); + hoveredIndex = queryResults.length > 0 ? 0 : -1; + // new results: scroll the list back to the top (first item is hovered) + if (hoveredIndex === 0) scrollTrigger++; + searchError = null; + } catch (err) { + if (mySeq !== searchSeq) return; + queryResults = []; + hoveredIndex = -1; + if (controller.signal.aborted) return; + searchError = err instanceof Error ? err.message : String(err); + } finally { + if (mySeq === searchSeq) isSearching = false; + } + } + + // Single funnel for every local close so the host refocus fires + // regardless of which commit/dismiss path ended the interaction. + function closePicker() { + isOpen = false; + onClose?.(); + } + + function commit(path: string) { + directory = path; + onChange?.(path); + closePicker(); + } + + function setDirectory(value: string) { + const trimmed = value.trim(); + if (!trimmed) return; + directory = trimmed; + onChange?.(trimmed); + } + + // Resolve a folder name picked via the browser-native picker (which exposes + // only the leaf name) to a server-side absolute path. Returns null when the + // server cannot locate a matching directory, so the caller can fail visibly + // instead of committing a bare leaf name that would resolve against the + // server process working directory. + async function resolveNativeName(name: string): Promise { + try { + const res = await ToolsService.executeToolRaw(BuiltInTool.FILE_GLOB_SEARCH, { + path: homeBase ?? HOME_TILDE, + type: GlobSearchType.DIR, + include: buildCaseInsensitiveGlob(name), + max_depth: NATIVE_MAX_DEPTH, + limit: NATIVE_LIMIT + }); + const base = typeof res.base === 'string' ? res.base : ''; + const entries = Array.isArray(res.entries) ? (res.entries as GlobEntry[]) : []; + const match = entries.find( + (e) => lastPathSegment(e.path).toLowerCase() === name.toLowerCase() + ); + return match ? joinPath(base, match.path) : null; + } catch { + return null; + } + } + + async function browseNative() { + if (disabled || !window.showDirectoryPicker) return; + try { + const handle = await window.showDirectoryPicker(); + const path = await resolveNativeName(handle.name); + if (path) { + setDirectory(path); + closePicker(); + } else { + // keep the previous cwd and fail visibly instead of committing a + // bare leaf name that would resolve against the server cwd + searchError = `Could not resolve "${handle.name}" to a server path`; + } + } catch (err) { + // user cancelled - silently ignore; other errors are logged + if (err instanceof DOMException && err.name === 'AbortError') return; + console.error('[ChatFormWorkingDirectory] showDirectoryPicker failed:', err); + } + } + + function handleSubmit() { + const value = inputValue.trim(); + if (!value) { + closePicker(); + return; + } + setDirectory(value); + closePicker(); + } + + function handleKeydown(event: KeyboardEvent) { + if (event.key === KeyboardKey.ENTER) { + event.preventDefault(); + // Commit the highlighted result, falling back to the raw input + // only when the query returned no matches. + if (hoveredIndex >= 0 && queryResults[hoveredIndex]) { + commit(queryResults[hoveredIndex]); + } else if (queryResults.length === 0) { + handleSubmit(); + } + } else if (event.key === KeyboardKey.ARROW_DOWN) { + if (queryResults.length > 0) { + event.preventDefault(); + hoveredIndex = (hoveredIndex + 1) % queryResults.length; + scrollTrigger++; + } + } else if (event.key === KeyboardKey.ARROW_UP) { + if (queryResults.length > 0) { + event.preventDefault(); + hoveredIndex = hoveredIndex <= 0 ? queryResults.length - 1 : hoveredIndex - 1; + scrollTrigger++; + } + } + } + + function handleInputInput(value: string) { + hoveredIndex = -1; + if (value.trim().length > 0) { + runSearch(value); + } + } + + function clearDirectory(event?: MouseEvent) { + // Stop the click from bubbling into the popover trigger and re-opening + // the picker on top of the now-cleared state. + event?.stopPropagation(); + event?.preventDefault(); + directory = null; + onChange?.(null); + closePicker(); + } + + // The chip is always visible; the X clears the directory (no-op when + // already empty). + function handleDismiss(event?: MouseEvent) { + event?.stopPropagation(); + event?.preventDefault(); + if (directory) { + clearDirectory(event); + } + } + + function handleOpenChange(open: boolean) { + isOpen = open; + if (open) { + // Seed the search field with the current path so the user can refine it + // (or hit Enter to confirm / clear via the X icon). + inputValue = directory ?? ''; + hoveredIndex = -1; + queryResults = []; + searchError = null; + void toolsStore.resolveServerHome(); + searchScope = homeBase ?? HOME_TILDE; + if (inputValue.trim()) runSearch(inputValue); + } else { + cancelSearch(); + // bits-ui-initiated close (Escape on the content, outside-click, + // trigger toggle) - the only path that bypasses closePicker(). + onClose?.(); + } + } + + // Tooltips only on wider viewports - hover surfaces get in the way on + // touch / narrow layouts. Mirrors the gate used in ActionIcon. + let innerWidth = $state(0); + const showTooltip = $derived(innerWidth > DEFAULT_MOBILE_BREAKPOINT); + + +
+ + + + + + event.preventDefault()} + > +
+ + + {#if inputValue.trim() && (isSearching || queryResults.length > 0 || searchError)} + (hoveredIndex = index)} + /> + {/if} + + {#if pickerSupported} + + {/if} + + {#if homeBase} + + + + Searching in: + + {abbreviateHome(searchScope, homeBase)} + + {/if} +
+ + +
+ + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectoryChip.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectoryChip.svelte new file mode 100644 index 000000000000..4f8d0f7f7d77 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectoryChip.svelte @@ -0,0 +1,69 @@ + + + +
+ + + {#if showTooltip && displayLabelTitle} + + + {#snippet child({ props })} + {displayLabel} + {/snippet} + + +

{displayLabelTitle}

+ + + {:else} + {displayLabel} + {/if} +
+ + {#if directory} +
+ +
+ {/if} + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectoryResultsList.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectoryResultsList.svelte new file mode 100644 index 000000000000..d62eb88242db --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormWorkingDirectoryResultsList.svelte @@ -0,0 +1,72 @@ + + +
+ {#if isSearching && results.length === 0} +
Searching...
+ {:else if error} +
{error}
+ {:else if results.length === 0} +
No matching folders
+ {:else} + {#each results as path, index (path)} + + {/each} + {/if} +
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte index b8068f790734..afe90f66fe25 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte @@ -12,6 +12,7 @@ ChatMessageAssistant, ChatMessageUser, ChatMessageSystem, + ChatMessageSynthetic, ChatMessageMcpPrompt } from '$lib/components/app/chat'; import { parseFilesToMessageExtras } from '$lib/utils/browser-only'; @@ -56,6 +57,10 @@ : message.content ); + // Synthetic cwd-change messages render with the folder-row UI instead + // of a user bubble. The persisted flag is the single source of truth. + let isSynthetic = $derived(Boolean(message.isSynthetic)); + let rawEditContent = $derived.by(() => { if (message.role !== MessageRole.ASSISTANT) return undefined; @@ -344,7 +349,7 @@ } -
+
{#if message.role === MessageRole.SYSTEM} + {:else if isSynthetic} + {:else if message.role === MessageRole.USER} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageCwdChange.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageCwdChange.svelte new file mode 100644 index 000000000000..0b0133060fda --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageCwdChange.svelte @@ -0,0 +1,31 @@ + + +{#if info} +
+ {#if info.path === null} + + Working directory cleared + {:else} + + Set working directory to  + + {info.display} + + {/if} +
+{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSynthetic.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSynthetic.svelte new file mode 100644 index 000000000000..1597df2ab147 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSynthetic.svelte @@ -0,0 +1,23 @@ + + +{#if isCwdChange} + +{:else} + {message.content} +{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte index b1daedfc818c..1d6cccc9f3ca 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte @@ -12,6 +12,7 @@ import ChatMessageToolCallBlockExecShellCommand from './ChatMessageToolCallBlockExecShellCommand.svelte'; import ChatMessageToolCallBlockFileGlobSearch from './ChatMessageToolCallBlockFileGlobSearch.svelte'; import ChatMessageToolCallBlockGetDatetime from './ChatMessageToolCallBlockGetDatetime.svelte'; + import ChatMessageToolCallBlockGetInfo from './ChatMessageToolCallBlockGetInfo.svelte'; import ChatMessageToolCallBlockGrepSearch from './ChatMessageToolCallBlockGrepSearch.svelte'; import ChatMessageToolCallBlockReadFile from './ChatMessageToolCallBlockReadFile.svelte'; import ChatMessageToolCallBlockRunJavascript from './ChatMessageToolCallBlockRunJavascript.svelte'; @@ -40,6 +41,8 @@ {:else if section.toolName === BuiltInTool.GET_DATETIME} +{:else if section.toolName === BuiltInTool.GET_INFO} + {:else if section.toolName === BuiltInTool.READ_FILE} {:else if section.toolName === BuiltInTool.EDIT_FILE} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte index b990c3898b23..f8618864c953 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte @@ -1,7 +1,8 @@ {#snippet execShellTitle()} + {#if cwd} + {wdDisplay} + $ + {/if} + {#if highlightedCommandHtml} {@html highlightedCommandHtml} {:else} @@ -232,6 +247,23 @@