diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..ee5adfb6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "core/src/main/c/share/zstd"] + path = core/src/main/c/share/zstd + url = https://github.com/facebook/zstd.git diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 87ebb009..b3176673 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -1,6 +1,12 @@ cmake_minimum_required(VERSION 3.5) project(questdb) +# Required for zstd's huf_decompress_amd64.S to be assembled. Without this, +# CMake silently drops the .S file from the build and the link fails at +# _HUF_decompress4X1_usingDTable_internal_fast_asm_loop etc. (The asmlib +# subdirectory enables ASM_NASM separately for Agner Fog's .asm files.) +enable_language(ASM) + include(ExternalProject) set(CMAKE_CXX_STANDARD 17) @@ -49,6 +55,42 @@ set( src/main/c/share/byte_sink.h ) +# libzstd is included via a git submodule at src/main/c/share/zstd (pinned to +# upstream tag v1.5.7). Covers the client side of the QWP egress compression +# feature; the server-side compressor lives in the Rust qdbr crate and isn't +# linked into this library. Only the decompress-only subset of upstream is +# compiled -- the compress/ directory is left out entirely. zstd_jni.c is our +# JNI glue and lives alongside the submodule (not inside) so upstream resets +# don't disturb it. +if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/main/c/share/zstd/lib/zstd.h) + message(FATAL_ERROR + "libzstd submodule not initialised. Run:\n" + " git submodule update --init --recursive\n" + "from java-questdb-client/.") +endif () +set( + ZSTD_FILES + src/main/c/share/zstd_jni.c + src/main/c/share/zstd/lib/common/debug.c + src/main/c/share/zstd/lib/common/entropy_common.c + src/main/c/share/zstd/lib/common/error_private.c + src/main/c/share/zstd/lib/common/fse_decompress.c + src/main/c/share/zstd/lib/common/pool.c + src/main/c/share/zstd/lib/common/threading.c + src/main/c/share/zstd/lib/common/xxhash.c + src/main/c/share/zstd/lib/common/zstd_common.c + src/main/c/share/zstd/lib/decompress/huf_decompress.c + src/main/c/share/zstd/lib/decompress/zstd_ddict.c + src/main/c/share/zstd/lib/decompress/zstd_decompress_block.c + src/main/c/share/zstd/lib/decompress/zstd_decompress.c +) +# x86_64-only hand-tuned Huffman decoder; C fallback kicks in when +# ZSTD_DISABLE_ASM is set. +if (ARCH_AMD64 AND NOT WIN32) + list(APPEND ZSTD_FILES src/main/c/share/zstd/lib/decompress/huf_decompress_amd64.S) +endif () +list(APPEND SOURCE_FILES ${ZSTD_FILES}) + # JNI includes include_directories($ENV{JAVA_HOME}/include/) @@ -111,6 +153,20 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${OUTPUT}) add_library(questdb SHARED ${SOURCE_FILES}) +# libzstd public header is at zstd/lib/zstd.h; internal headers live under +# zstd/lib/common/. Both directories go on the include path so zstd_jni.c can +# use the short include "zstd.h" and the upstream .c files can find their own +# siblings without patching. +target_include_directories(questdb PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/main/c/share/zstd/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/main/c/share/zstd/lib/common) + +# Drop the zstd-internal hand-written amd64 assembly on platforms that can't +# assemble it; libzstd falls back to a C implementation when this is set. +if (NOT ARCH_AMD64 OR WIN32) + target_compile_definitions(questdb PRIVATE ZSTD_DISABLE_ASM=1) +endif () + set(COMMON_OPTIONS "-Wno-gnu-anonymous-struct;-Wno-nested-anon-types;-Wno-unused-parameter;-fPIC;-fno-rtti;-fno-exceptions") set(DEBUG_OPTIONS "-Wall;-pedantic;-Wextra;-g;-O0") diff --git a/core/src/main/c/share/zstd b/core/src/main/c/share/zstd new file mode 160000 index 00000000..f8745da6 --- /dev/null +++ b/core/src/main/c/share/zstd @@ -0,0 +1 @@ +Subproject commit f8745da6ff1ad1e7bab384bd1f9d742439278e99 diff --git a/core/src/main/c/share/zstd_jni.c b/core/src/main/c/share/zstd_jni.c new file mode 100644 index 00000000..d677fba4 --- /dev/null +++ b/core/src/main/c/share/zstd_jni.c @@ -0,0 +1,106 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +/* + * JNI wrapper over the bundled libzstd (decompression only). The server ships + * compression support in the Rust qdbr crate; this file covers the client + * decompression path so RESULT_BATCH frames with FLAG_ZSTD can be decoded + * without any external native dependency. + * + * libzstd is vendored as a git submodule at share/zstd/ pinned to v1.5.7; + * this file lives alongside (not inside) the submodule so upstream resets + * don't nuke our JNI glue. + */ + +#include +#include +#include +#include "zstd.h" + +JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Zstd_createDCtx( + JNIEnv *env, jclass cls) { + return (jlong) (uintptr_t) ZSTD_createDCtx(); +} + +JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Zstd_getFrameContentSize( + JNIEnv *env, jclass cls, + jlong src_addr, jlong src_len) { + /* + * Peeks the zstd frame header at src_addr to recover the declared + * uncompressed size. Returns: + * positive -- declared content size in bytes + * -1 -- frame valid, content size not stored (ZSTD_CONTENTSIZE_UNKNOWN) + * -2 -- invalid frame, truncated header, or size > INT64_MAX + * + * Lets the Java caller size the destination buffer in a single allocation + * instead of retrying decompress on dst-too-small. Crucially, it also lets + * a corrupt frame fail BEFORE any output buffer growth, eliminating a + * memory-amplification vector where one bad frame would have driven + * scratch growth all the way to the 64 MiB cap. + */ + if (src_len < 0 || (src_len > 0 && src_addr == 0)) { + return -2; + } + unsigned long long size = ZSTD_getFrameContentSize( + (const void *) (uintptr_t) src_addr, (size_t) src_len); + if (size == ZSTD_CONTENTSIZE_UNKNOWN) { + return -1; + } + if (size == ZSTD_CONTENTSIZE_ERROR) { + return -2; + } + if (size > (unsigned long long) INT64_MAX) { + /* Cast to jlong would wrap to negative and look like an error code; + * reject upfront so the caller doesn't double-interpret. */ + return -2; + } + return (jlong) size; +} + +JNIEXPORT void JNICALL Java_io_questdb_client_std_Zstd_freeDCtx( + JNIEnv *env, jclass cls, jlong ptr) { + if (ptr != 0) { + ZSTD_freeDCtx((ZSTD_DCtx *) (uintptr_t) ptr); + } +} + +JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Zstd_decompress( + JNIEnv *env, jclass cls, + jlong ctx, + jlong src_addr, jlong src_len, + jlong dst_addr, jlong dst_cap) { + if (ctx == 0) { + return -1; + } + ZSTD_DCtx *dctx = (ZSTD_DCtx *) (uintptr_t) ctx; + size_t n = ZSTD_decompressDCtx( + dctx, + (void *) (uintptr_t) dst_addr, (size_t) dst_cap, + (const void *) (uintptr_t) src_addr, (size_t) src_len); + if (ZSTD_isError(n)) { + unsigned code = ZSTD_getErrorCode(n); + return -(jlong) code; + } + return (jlong) n; +} diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 986a34dd..578df2a2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -105,7 +105,17 @@ public abstract class WebSocketClient implements QuietCloseable { private CharSequence host; private int port; // QWP version negotiation + // Verbatim header value sent as X-QWP-Accept-Encoding during upgrade, e.g. + // "zstd;level=3,raw". When null, the header is omitted and the server ships + // batches uncompressed. The echoed X-QWP-Content-Encoding response header + // is intentionally not parsed: the RESULT_BATCH decoder branches on + // FLAG_ZSTD in every frame, which is the authoritative signal. + private String qwpAcceptEncoding; private String qwpClientId; + // Client-requested per-batch row cap advertised via X-QWP-Max-Batch-Rows. + // 0 means "omit the header" (server uses its default cap). Server may clamp + // down to its own hard limit. + private int qwpMaxBatchRows; private int qwpMaxVersion = 1; // Opt-in for STATUS_DURABLE_ACK frames; sent as X-QWP-Request-Durable-Ack: true private boolean qwpRequestDurableAck; @@ -378,6 +388,16 @@ public void sendPing(int timeout) { } } + /** + * Sets the value sent as the {@code X-QWP-Accept-Encoding} upgrade header, + * e.g. {@code "zstd;level=3,raw"}. Pass {@code null} to omit the header + * entirely (server ships uncompressed batches). Must be called before + * {@link #upgrade}. + */ + public void setQwpAcceptEncoding(String acceptEncoding) { + this.qwpAcceptEncoding = acceptEncoding; + } + /** * Sets the QWP client identifier sent in the X-QWP-Client-Id upgrade header. */ @@ -385,6 +405,18 @@ public void setQwpClientId(String clientId) { this.qwpClientId = clientId; } + /** + * Sets the client's preferred per-batch row cap, sent in the + * {@code X-QWP-Max-Batch-Rows} upgrade header. {@code 0} (the default) + * omits the header entirely and the server uses its own cap. Positive + * values ask the server to flush batches sooner (lower time-to-first-row + * for streaming consumers, at the cost of more per-batch overhead); the + * server clamps down to its own maximum. + */ + public void setQwpMaxBatchRows(int maxBatchRows) { + this.qwpMaxBatchRows = maxBatchRows; + } + /** * Sets the maximum QWP version this client supports, sent in the X-QWP-Max-Version upgrade header. */ @@ -488,6 +520,16 @@ public void upgrade(CharSequence path, int timeout, CharSequence authorizationHe sendBuffer.putAscii(qwpClientId); sendBuffer.putAscii("\r\n"); } + if (qwpAcceptEncoding != null) { + sendBuffer.putAscii("X-QWP-Accept-Encoding: "); + sendBuffer.putAscii(qwpAcceptEncoding); + sendBuffer.putAscii("\r\n"); + } + if (qwpMaxBatchRows > 0) { + sendBuffer.putAscii("X-QWP-Max-Batch-Rows: "); + sendBuffer.putAscii(Integer.toString(qwpMaxBatchRows)); + sendBuffer.putAscii("\r\n"); + } if (qwpRequestDurableAck) { sendBuffer.putAscii("X-QWP-Request-Durable-Ack: true\r\n"); } @@ -973,7 +1015,7 @@ private void validateUpgradeResponse(int headerEnd) { throw new HttpClientException("Invalid Sec-WebSocket-Accept header"); } - // Extract X-QWP-Version (optional — defaults to 1 if absent) + // Extract X-QWP-Version (optional, defaults to 1 if absent) serverQwpVersion = extractQwpVersion(response); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ColumnView.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ColumnView.java new file mode 100644 index 00000000..215c5607 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ColumnView.java @@ -0,0 +1,455 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.Long256Sink; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Uuid; +import io.questdb.client.std.bytes.DirectByteSequence; +import io.questdb.client.std.str.CharSink; +import io.questdb.client.std.str.DirectUtf8Sequence; + +/** + * Column-pinned view over a {@link QwpColumnBatch}. Resolves the column layout + * once at {@link #of(int)} time, then exposes per-row accessors that take only + * the row index. A column-first scan over N rows therefore pays one layout + * lookup instead of N: + * 
+ *   ColumnView prices = batch.column(2);
+ *   for (int r = 0; r < batch.getRowCount(); r++) {
+ *       sum += prices.getDoubleValue(r);
+ *   }
+ *
+ *

+ * Lifetime. Owned by the batch, reused across rows of the + * pinned column, and valid only for the duration of the surrounding + * {@code onBatch} call. The batch keeps one {@code ColumnView} per column index + * so two views over different columns can be held concurrently; calling + * {@link QwpColumnBatch#column(int)} a second time with the same {@code col} + * returns the same instance and keeps prior bindings of other columns intact. + *

+ * Direct-address surface. The {@code valuesAddr}, + * {@code nullBitmapAddr}, {@code stringBytesAddr}, {@code symbolDict*}, and + * {@code arrayRowAddr} accessors expose raw native pointers into the WebSocket + * payload buffer for SIMD or JNI consumers. The layout of the bytes behind + * these addresses follows the QWP wire format (see + * {@code docs/QWP_EGRESS_EXTENSION.md}); these methods are an expert API and + * may shift as the wire format evolves. + *

+ * Type contract / NULL handling. Each typed accessor delegates + * to the same read path as the corresponding {@code QwpColumnBatch} + * {@code (col, row)} primitive; the contracts in {@link QwpColumnBatch} apply + * unchanged. + */ +public class ColumnView { + + private final QwpColumnBatch batch; + private int bytesPerValue; + private int col; + private QwpColumnLayout layout; + + ColumnView(QwpColumnBatch batch) { + this.batch = batch; + } + + /** + * Address (or 0) of a single ARRAY value's packed bytes. Format is the QWP + * array encoding: 1 byte ndims, ndims x int32 dim sizes, then the flat + * row-major elements. Returns 0 if the row is NULL. Caller must know the + * column is an ARRAY type. + */ + public long arrayRowAddr(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0L; + return layout.arrayRowAddr[row]; + } + + /** + * Returns the parent batch. Use it to read column-level metadata not + * exposed on this view ({@link QwpColumnBatch#getColumnName}, + * {@link QwpColumnBatch#getDecimalScale}, etc.) or to switch to row-first + * iteration without capturing an extra reference. + */ + public QwpColumnBatch batch() { + return batch; + } + + /** + * Per-value stride in bytes for fixed-width types, including GEOHASH (where + * stride is {@code ceil(precisionBits / 8)}). Returns 0 for BOOLEAN + * (bit-packed, 8 values per byte) and -1 for variable-width types + * (STRING, VARCHAR, BINARY, SYMBOL, ARRAY, DOUBLE_ARRAY, LONG_ARRAY). + * Combine with {@link #valuesAddr()} and {@link #nonNullCount()} to walk + * the dense values array directly. + */ + public int bytesPerValue() { + return bytesPerValue; + } + + /** + * Index of the column this view is pinned to. + */ + public int getColumnIndex() { + return col; + } + + /** + * Wire type code for the pinned column. Same value as + * {@link QwpColumnBatch#getColumnWireType(int)}. + */ + public byte getColumnWireType() { + return layout.info.wireType; + } + + /** + * @see QwpColumnBatch#getArrayNDims(int, int) + */ + public int getArrayNDims(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0; + return Unsafe.getUnsafe().getByte(layout.arrayRowAddr[row]) & 0xFF; + } + + /** + * @see QwpColumnBatch#getBinary(int, int) + */ + public byte[] getBinary(int row) { + return batch.getBinary(col, row); + } + + /** + * @see QwpColumnBatch#getBinaryA(int, int) + */ + public DirectByteSequence getBinaryA(int row) { + return batch.getBinaryA(col, row); + } + + /** + * @see QwpColumnBatch#getBinaryB(int, int) + */ + public DirectByteSequence getBinaryB(int row) { + return batch.getBinaryB(col, row); + } + + /** + * @see QwpColumnBatch#getBoolValue(int, int) + */ + public boolean getBoolValue(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return false; + int denseIdx = layout.denseIndex(row); + byte b = Unsafe.getUnsafe().getByte(layout.valuesAddr + (denseIdx >>> 3)); + return (b & (1 << (denseIdx & 7))) != 0; + } + + /** + * @see QwpColumnBatch#getByteValue(int, int) + */ + public byte getByteValue(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0; + return Unsafe.getUnsafe().getByte(layout.valuesAddr + layout.denseIndex(row)); + } + + /** + * @see QwpColumnBatch#getCharValue(int, int) + */ + public char getCharValue(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0; + return (char) Unsafe.getUnsafe().getShort(layout.valuesAddr + 2L * layout.denseIndex(row)); + } + + /** + * @see QwpColumnBatch#getDecimal128High(int, int) + */ + public long getDecimal128High(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0L; + return Unsafe.getUnsafe().getLong(layout.valuesAddr + 16L * layout.denseIndex(row) + 8L); + } + + /** + * @see QwpColumnBatch#getDecimal128Low(int, int) + */ + public long getDecimal128Low(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0L; + return Unsafe.getUnsafe().getLong(layout.valuesAddr + 16L * layout.denseIndex(row)); + } + + /** + * @see QwpColumnBatch#getDoubleArrayElements(int, int) + */ + public double[] getDoubleArrayElements(int row) { + return batch.getDoubleArrayElements(col, row); + } + + /** + * @see QwpColumnBatch#getDoubleValue(int, int) + */ + public double getDoubleValue(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return Double.NaN; + return Unsafe.getUnsafe().getDouble(layout.valuesAddr + 8L * layout.denseIndex(row)); + } + + /** + * @see QwpColumnBatch#getFloatValue(int, int) + */ + public float getFloatValue(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return Float.NaN; + return Unsafe.getUnsafe().getFloat(layout.valuesAddr + 4L * layout.denseIndex(row)); + } + + /** + * @see QwpColumnBatch#getGeohashValue(int, int) + */ + public long getGeohashValue(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0L; + int denseIdx = layout.denseIndex(row); + int bpv = bytesPerValue; + long p = layout.valuesAddr + (long) bpv * denseIdx; + long bits = 0; + for (int b = 0; b < bpv; b++) { + bits |= ((long) (Unsafe.getUnsafe().getByte(p + b) & 0xFF)) << (b * 8); + } + return bits; + } + + /** + * @see QwpColumnBatch#getIntValue(int, int) + */ + public int getIntValue(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0; + return Unsafe.getUnsafe().getInt(layout.valuesAddr + 4L * layout.denseIndex(row)); + } + + /** + * @see QwpColumnBatch#getLong256(int, int, Long256Sink) + */ + public boolean getLong256(int row, Long256Sink sink) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return false; + sink.fromAddress(layout.valuesAddr + 32L * layout.denseIndex(row)); + return true; + } + + /** + * @see QwpColumnBatch#getLong256Word(int, int, int) + */ + public long getLong256Word(int row, int wordIndex) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0L; + return Unsafe.getUnsafe().getLong(layout.valuesAddr + 32L * layout.denseIndex(row) + 8L * wordIndex); + } + + /** + * @see QwpColumnBatch#getLongValue(int, int) + */ + public long getLongValue(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0L; + return Unsafe.getUnsafe().getLong(layout.valuesAddr + 8L * layout.denseIndex(row)); + } + + /** + * @see QwpColumnBatch#getShortValue(int, int) + */ + public short getShortValue(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0; + return Unsafe.getUnsafe().getShort(layout.valuesAddr + 2L * layout.denseIndex(row)); + } + + /** + * @see QwpColumnBatch#getStrA(int, int) + */ + public DirectUtf8Sequence getStrA(int row) { + return batch.getStrA(col, row); + } + + /** + * @see QwpColumnBatch#getStrB(int, int) + */ + public DirectUtf8Sequence getStrB(int row) { + return batch.getStrB(col, row); + } + + /** + * @see QwpColumnBatch#getString(int, int) + */ + public String getString(int row) { + return batch.getString(col, row); + } + + /** + * @see QwpColumnBatch#getString(int, int, CharSink) + */ + public boolean getString(int row, CharSink sink) { + return batch.getString(col, row, sink); + } + + /** + * @see QwpColumnBatch#getSymbol(int, int) + */ + public String getSymbol(int row) { + return batch.getSymbol(col, row); + } + + /** + * @see QwpColumnBatch#getSymbolId(int, int) + */ + public int getSymbolId(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return -1; + return layout.symbolRowIds[row]; + } + + /** + * @see QwpColumnBatch#getUuid(int, int, Uuid) + */ + public boolean getUuid(int row, Uuid sink) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return false; + sink.fromAddress(layout.valuesAddr + 16L * layout.denseIndex(row)); + return true; + } + + /** + * @see QwpColumnBatch#getUuidHi(int, int) + */ + public long getUuidHi(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0L; + return Unsafe.getUnsafe().getLong(layout.valuesAddr + 16L * layout.denseIndex(row) + 8L); + } + + /** + * @see QwpColumnBatch#getUuidLo(int, int) + */ + public long getUuidLo(int row) { + if (QwpColumnBatch.isLayoutNull(layout, row)) return 0L; + return Unsafe.getUnsafe().getLong(layout.valuesAddr + 16L * layout.denseIndex(row)); + } + + /** + * @see QwpColumnBatch#isNull(int, int) + */ + public boolean isNull(int row) { + return QwpColumnBatch.isLayoutNull(layout, row); + } + + /** + * Dense index of {@code row} into the column's packed non-null values, or + * undefined for NULL rows (call {@link #isNull(int)} first). Equivalent to + * looking up {@code QwpColumnBatch.nonNullIndex(col)[row]} but skips the + * per-call lookup of the column array. + */ + public int nonNullIndex(int row) { + return layout.denseIndex(row); + } + + /** + * Number of non-null rows for the pinned column in this batch. Same as + * {@link QwpColumnBatch#nonNullCount(int)}; exposed here so SIMD consumers + * iterating raw {@link #valuesAddr()} bytes have the trip count without a + * second lookup. + */ + public int nonNullCount() { + return layout.nonNullCount; + } + + /** + * Address of the per-row null bitmap. {@code 0} when the column has no + * nulls in this batch, in which case row index equals dense index for all + * rows. Otherwise the bitmap is little-endian byte-packed, 8 rows per byte, + * LSB-first: row {@code r} is NULL iff {@code (bitmap[r >>> 3] & (1 << (r & 7))) != 0}. + * The bitmap is sized to {@code ceil(rowCount / 8)} bytes. + */ + public long nullBitmapAddr() { + return layout.nullBitmapAddr; + } + + /** + * Re-points this flyweight at column {@code col} of the parent batch. + * Resolves the column layout once and caches it; subsequent per-row + * accessors avoid the {@code ObjList.getQuick(col)} that the + * {@link QwpColumnBatch} two-arg primitives pay per call. + */ + public ColumnView of(int col) { + this.col = col; + this.layout = batch.columnLayouts.getQuick(col); + this.bytesPerValue = QwpConstants.getFixedTypeSize(layout.info.wireType); + if (this.bytesPerValue == -1 && layout.info.wireType == QwpConstants.TYPE_GEOHASH) { + this.bytesPerValue = (layout.info.precisionBits + 7) >>> 3; + } + return this; + } + + /** + * Address of the SYMBOL dictionary's packed entries array. Each entry is + * an 8-byte pair {@code (offset:i32 | length:i32<<32)} relative to + * {@link #symbolDictHeapAddr()}. Returns 0 for non-SYMBOL columns. + */ + public long symbolDictEntriesAddr() { + return layout.symbolDictEntriesAddr; + } + + /** + * Address of the SYMBOL dictionary's UTF-8 bytes heap holding all dict + * entries. Returns 0 for non-SYMBOL columns. + */ + public long symbolDictHeapAddr() { + return layout.symbolDictHeapAddr; + } + + /** + * Number of valid entries in the SYMBOL dictionary for this batch. + * Returns 0 for non-SYMBOL columns. + */ + public int symbolDictSize() { + return layout.symbolDictSize; + } + + /** + * Per-row dictionary IDs for a SYMBOL column. {@code result[row]} is the + * dict id for that row; values for NULL rows are unspecified -- check + * {@link #isNull(int)} or {@link #nullBitmapAddr()} first. Returns + * {@code null} for non-SYMBOL columns. Array is owned by the decoder and + * valid only for the surrounding {@code onBatch} call. + */ + public int[] symbolRowIds() { + return layout.symbolRowIds; + } + + /** + * Address of the concatenated UTF-8 bytes for STRING / VARCHAR / BINARY + * values. The {@link #valuesAddr()} for these columns points at the + * {@code (N+1) x int32} offsets array; the bytes themselves live at this + * address. Returns 0 for non-string columns. + */ + public long stringBytesAddr() { + return layout.stringBytesAddr; + } + + /** + * Base address of the column's packed non-null values in the payload + * buffer. Same value as {@link QwpColumnBatch#valuesAddr(int)}; see that + * method's Javadoc for the per-type layout (fixed-width stride, bit-packed + * BOOLEAN, STRING/VARCHAR offsets array, etc.). Combine with + * {@link #bytesPerValue()}, {@link #nonNullCount()}, and + * {@link #nullBitmapAddr()} to drive a SIMD scan. + */ + public long valuesAddr() { + return layout.valuesAddr; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java new file mode 100644 index 00000000..d3083e4c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QueryEvent.java @@ -0,0 +1,110 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Tagged event pushed from the I/O thread onto the user-facing event queue. + * One event per {@code RESULT_BATCH} / {@code RESULT_END} / {@code QUERY_ERROR} + * received from the server, plus a synthetic error event if the connection drops + * mid-query. + */ +public class QueryEvent { + + public static final int KIND_BATCH = 0; + public static final int KIND_END = 1; + public static final int KIND_ERROR = 2; + public static final int KIND_EXEC_DONE = 3; + /** + * Synthesised on the I/O thread when the server closes the socket, + * receiveFrame / decode raises, or the I/O thread abnormally exits. + * Distinct from {@link #KIND_ERROR} (server-emitted {@code QUERY_ERROR}) + * so {@code execute()} can decide whether to trigger failover without + * having to reconstruct the classification from a side-channel latch. + */ + public static final int KIND_TRANSPORT_ERROR = 4; + + public QwpBatchBuffer buffer; // valid for KIND_BATCH (must be released to pool by consumer) + public String errorMessage; // valid for KIND_ERROR + public byte errorStatus; // valid for KIND_ERROR + public int kind; + public short opType; // valid for KIND_EXEC_DONE (matches CompiledQuery.SELECT/INSERT/etc.) + public long rowsAffected; // valid for KIND_EXEC_DONE + public long totalRows; // valid for KIND_END + + public QueryEvent asBatch(QwpBatchBuffer buffer) { + this.kind = KIND_BATCH; + this.buffer = buffer; + return this; + } + + public QueryEvent asEnd(long totalRows) { + this.kind = KIND_END; + this.buffer = null; + this.totalRows = totalRows; + return this; + } + + public QueryEvent asError(byte status, String message) { + this.kind = KIND_ERROR; + this.buffer = null; + this.errorStatus = status; + this.errorMessage = message; + return this; + } + + public QueryEvent asExecDone(short opType, long rowsAffected) { + this.kind = KIND_EXEC_DONE; + this.buffer = null; + this.opType = opType; + this.rowsAffected = rowsAffected; + return this; + } + + public QueryEvent asTransportError(byte status, String message) { + this.kind = KIND_TRANSPORT_ERROR; + this.buffer = null; + this.errorStatus = status; + this.errorMessage = message; + return this; + } + + /** + * Clears object references and resets primitive fields so a pooled event is + * safe to reuse across queries. The I/O thread calls the {@code asX(...)} + * builders after borrowing, which overwrites fields anyway; resetting on + * release still helps because it (1) drops the buffer / message references + * promptly so they are not held past the consumer callback, and (2) keeps + * the event in a consistent "no kind bound" state for diagnostics. + */ + public void reset() { + this.kind = -1; + this.buffer = null; + this.errorMessage = null; + this.errorStatus = 0; + this.opType = 0; + this.rowsAffected = 0; + this.totalRows = 0; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBatchBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBatchBuffer.java new file mode 100644 index 00000000..603386ce --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBatchBuffer.java @@ -0,0 +1,123 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +/** + * Pooled per-batch container owned by the client's I/O thread. A buffer holds a + * native scratch region that a received {@code RESULT_BATCH} payload is memcpy'd + * into, plus the per-column {@link QwpColumnLayout} pool used while decoding and + * the {@link QwpColumnBatch} view that the user's handler sees. + *

+ * Lifecycle: I/O thread takes a buffer from the free pool -> copies the frame + * payload in -> hands the decoder the buffer -> pushes the resulting batch onto + * the event queue. User thread pops, invokes the handler, releases the buffer + * back to the pool. While the user thread owns the buffer the I/O thread is + * free to take a different buffer and decode the next frame. + */ +public class QwpBatchBuffer implements QuietCloseable { + + final QwpColumnBatch batch = new QwpColumnBatch(); + /** + * Per-column layout pool scoped to this buffer. Sized to the max column + * count observed on this buffer across batches; layouts are reused. + */ + final ObjList layoutPool = new ObjList<>(); + private int payloadLen; + private long scratchAddr; + private int scratchCapacity; + + public QwpBatchBuffer(int initialCapacity) { + this.scratchCapacity = initialCapacity; + this.scratchAddr = Unsafe.malloc(initialCapacity, MemoryTag.NATIVE_DEFAULT); + } + + @Override + public void close() { + if (scratchAddr != 0) { + Unsafe.free(scratchAddr, scratchCapacity, MemoryTag.NATIVE_DEFAULT); + scratchAddr = 0; + scratchCapacity = 0; + } + // Layouts own native entries buffers in non-delta SYMBOL mode. Free them + // before the buffer itself is discarded so the allocations don't leak. + for (int i = 0, n = layoutPool.size(); i < n; i++) { + layoutPool.getQuick(i).close(); + } + layoutPool.clear(); + } + + /** + * Copies {@code len} bytes starting at {@code srcAddr} into this buffer's + * native scratch, growing if needed. Call once per incoming frame before + * handing the buffer to the decoder. + */ + public void copyFromPayload(long srcAddr, int len) { + ensureCapacity(len); + Unsafe.getUnsafe().copyMemory(srcAddr, scratchAddr, len); + payloadLen = len; + } + + public int getPayloadLen() { + return payloadLen; + } + + public long getScratchAddr() { + return scratchAddr; + } + + private void ensureCapacity(int required) { + if (required < 0) { + // A negative request cannot be honoured. Reject loudly rather than + // silently wrapping through the doubling loop. + throw new IllegalArgumentException("QwpBatchBuffer required capacity must be non-negative: " + required); + } + if (required <= scratchCapacity) return; + // Start the doubling at max(current, 1) so a buffer constructed with + // initialCapacity=0 can still grow (0 *= 2 would spin forever). Cap the + // double step against Integer.MAX_VALUE because `newCap *= 2` wraps + // negative once newCap passes 2^30, at which point the while loop + // could never reach `required` and would spin indefinitely. + long newCap = Math.max(scratchCapacity, 1); + while (newCap < required) { + newCap <<= 1; + if (newCap > Integer.MAX_VALUE) { + newCap = Integer.MAX_VALUE; + break; + } + } + if (newCap < required) { + throw new OutOfMemoryError("QwpBatchBuffer required capacity " + required + + " exceeds Integer.MAX_VALUE"); + } + int capped = (int) newCap; + scratchAddr = Unsafe.realloc(scratchAddr, scratchCapacity, capped, MemoryTag.NATIVE_DEFAULT); + scratchCapacity = capped; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBindSetter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBindSetter.java new file mode 100644 index 00000000..4c5b39b2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBindSetter.java @@ -0,0 +1,45 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Callback that populates the typed bind parameters of a single + * {@link QwpQueryClient#execute(String, QwpBindSetter, QwpColumnBatchHandler)} + * invocation. The callback runs on the user thread while the client prepares + * the QUERY_REQUEST frame; the encoded bind bytes are then handed to the I/O + * thread for transmission. + *

+ * Implementations must call the typed setters on the supplied + * {@link QwpBindValues} in strictly ascending index order starting at 0, with + * no gaps. The client validates this and throws on violation. + */ +@FunctionalInterface +public interface QwpBindSetter { + /** + * Populates the supplied bind-value sink. Callers must invoke the typed + * setters in ascending index order (0, 1, 2, ...). + */ + void apply(QwpBindValues binds); +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBindValues.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBindValues.java new file mode 100644 index 00000000..ca3a531f --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpBindValues.java @@ -0,0 +1,539 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimals; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +/** + * Typed bind-parameter sink for a single QWP egress query. + *

+ * Writes the per-bind wire layout directly into a reusable native buffer: + * {@code type_code(1B) | null_flag(1B) | [bitmap(1B) if null_flag != 0] | + * [value bytes if !null]}. + *

+ * Non-null: {@code type | 0x00 | value}. NULL: {@code type | 0x01 | 0x01} + * (no value bytes). + *

+ * Indexes must be assigned in strictly ascending order starting at 0. The + * sink tracks the next expected index and throws {@link IllegalStateException} + * on gaps or duplicates. Matches the server's positional bind layout; SQL + * parameter placeholders are 1-based ({@code $1, $2, ...}), indexes here are + * 0-based and map to {@code $(index + 1)}. + *

+ * Multi-byte values are little-endian. DECIMAL scales are validated against + * {@link Decimals#MAX_SCALE}; GEOHASH precisions are validated against the + * [1, 60]-bit range the server enforces. + *

+ * Not thread-safe. One instance per {@link QwpQueryClient}; reset on every + * {@link QwpQueryClient#execute(String, QwpBindSetter, QwpColumnBatchHandler)} + * call. + */ +public final class QwpBindValues implements QuietCloseable { + + /** + * Maximum scale for a DECIMAL128 bind. DECIMAL128 stores up to 38 digits of + * precision, so scale above 38 is mathematically invalid even though the + * wire format can carry any byte value. + */ + private static final int DECIMAL128_MAX_SCALE = 38; + /** + * Maximum scale for a DECIMAL256 bind. DECIMAL256 is the widest decimal + * type and its scale cap matches {@link Decimals#MAX_SCALE} (76). + */ + private static final int DECIMAL256_MAX_SCALE = Decimals.MAX_SCALE; + /** + * Maximum scale for a DECIMAL64 bind. DECIMAL64 stores up to 18 digits of + * precision, so scale above 18 is mathematically invalid. + */ + private static final int DECIMAL64_MAX_SCALE = 18; + /** + * Maximum GEOHASH precision in bits, matching the server's + * {@code ColumnType.GEOLONG_MAX_BITS} check. + */ + private static final int GEOHASH_MAX_BITS = 60; + private static final int GEOHASH_MIN_BITS = 1; + private static final byte NULL_BITMAP = 0x01; + private static final byte NULL_FLAG = 0x01; + private static final byte NON_NULL_FLAG = 0x00; + + private final NativeBufferWriter writer = new NativeBufferWriter(); + private int count; + private int expectedNextIndex; + + @Override + public void close() { + writer.close(); + } + + public QwpBindValues setBoolean(int index, boolean value) { + advance(index); + writeHeader(QwpConstants.TYPE_BOOLEAN, false); + writer.putByte(value ? (byte) 1 : (byte) 0); + return this; + } + + public QwpBindValues setByte(int index, byte value) { + advance(index); + writeHeader(QwpConstants.TYPE_BYTE, false); + writer.putByte(value); + return this; + } + + public QwpBindValues setChar(int index, char value) { + advance(index); + writeHeader(QwpConstants.TYPE_CHAR, false); + writer.putShort((short) value); + return this; + } + + public QwpBindValues setDate(int index, long millisSinceEpoch) { + advance(index); + writeHeader(QwpConstants.TYPE_DATE, false); + writer.putLong(millisSinceEpoch); + return this; + } + + public QwpBindValues setDecimal128(int index, int scale, long lo, long hi) { + checkScale128(scale); + advance(index); + writeHeader(QwpConstants.TYPE_DECIMAL128, false); + writer.putByte((byte) scale); + writer.putLong(lo); + writer.putLong(hi); + return this; + } + + /** + * Convenience overload that takes the scale and limbs from the supplied + * {@link Decimal128}. If the value is NULL (Decimal128's canonical NULL + * sentinel), an explicit DECIMAL128 NULL is encoded preserving + * {@code value.getScale()}; a {@code null} reference encodes with scale 0. + */ + public QwpBindValues setDecimal128(int index, Decimal128 value) { + if (value == null) { + return setNullDecimal128(index, 0); + } + if (Decimal128.isNull(value.getHigh(), value.getLow())) { + return setNullDecimal128(index, value.getScale()); + } + return setDecimal128(index, value.getScale(), value.getLow(), value.getHigh()); + } + + public QwpBindValues setDecimal256(int index, int scale, long ll, long lh, long hl, long hh) { + checkScale256(scale); + advance(index); + writeHeader(QwpConstants.TYPE_DECIMAL256, false); + writer.putByte((byte) scale); + writer.putLong(ll); + writer.putLong(lh); + writer.putLong(hl); + writer.putLong(hh); + return this; + } + + /** + * Convenience overload that takes the scale and limbs from the supplied + * {@link Decimal256}. If the value is NULL (Decimal256's canonical NULL + * sentinel), an explicit DECIMAL256 NULL is encoded preserving + * {@code value.getScale()}; a {@code null} reference encodes with scale 0. + */ + public QwpBindValues setDecimal256(int index, Decimal256 value) { + if (value == null) { + return setNullDecimal256(index, 0); + } + if (Decimal256.isNull(value.getHh(), value.getHl(), value.getLh(), value.getLl())) { + return setNullDecimal256(index, value.getScale()); + } + return setDecimal256(index, value.getScale(), value.getLl(), value.getLh(), value.getHl(), value.getHh()); + } + + public QwpBindValues setDecimal64(int index, int scale, long unscaledValue) { + checkScale64(scale); + advance(index); + writeHeader(QwpConstants.TYPE_DECIMAL64, false); + writer.putByte((byte) scale); + writer.putLong(unscaledValue); + return this; + } + + public QwpBindValues setDouble(int index, double value) { + advance(index); + writeHeader(QwpConstants.TYPE_DOUBLE, false); + writer.putLong(Double.doubleToRawLongBits(value)); + return this; + } + + public QwpBindValues setFloat(int index, float value) { + advance(index); + writeHeader(QwpConstants.TYPE_FLOAT, false); + writer.putInt(Float.floatToRawIntBits(value)); + return this; + } + + /** + * Encodes a GEOHASH bind with the given precision (in bits) and packed + * value. The value is stored little-endian in {@code ceil(precisionBits / 8)} + * bytes. Precision must be in {@code [1, 60]}. + *

+ * {@code value} is masked to {@code precisionBits} before encoding, so bits + * above the declared precision cannot leak into the top wire byte (which + * would otherwise pass through when {@code precisionBits} is not a multiple + * of 8). + */ + public QwpBindValues setGeohash(int index, int precisionBits, long value) { + checkGeohashPrecision(precisionBits); + advance(index); + writeHeader(QwpConstants.TYPE_GEOHASH, false); + writer.putVarint(precisionBits); + long masked = maskGeohashBits(value, precisionBits); + int byteCount = (precisionBits + 7) >>> 3; + for (int b = 0; b < byteCount; b++) { + writer.putByte((byte) (masked >>> (b * 8))); + } + return this; + } + + public QwpBindValues setInt(int index, int value) { + advance(index); + writeHeader(QwpConstants.TYPE_INT, false); + writer.putInt(value); + return this; + } + + public QwpBindValues setLong(int index, long value) { + advance(index); + writeHeader(QwpConstants.TYPE_LONG, false); + writer.putLong(value); + return this; + } + + public QwpBindValues setLong256(int index, long l0, long l1, long l2, long l3) { + advance(index); + writeHeader(QwpConstants.TYPE_LONG256, false); + writer.putLong(l0); + writer.putLong(l1); + writer.putLong(l2); + writer.putLong(l3); + return this; + } + + /** + * Binds an explicit NULL with the given QWP wire type. The type code must + * be one of the supported scalar bind types; ARRAY, BINARY, and IPv4 are + * rejected because the server decoder does not accept them as binds. + *

+ * DECIMAL64/128/256 NULLs are encoded with scale 0 and GEOHASH NULLs with + * precision {@value #GEOHASH_MIN_BITS} bit; use {@link #setNullDecimal64}, + * {@link #setNullDecimal128}, {@link #setNullDecimal256}, or + * {@link #setNullGeohash} when the scale/precision matters (it becomes part + * of the bound variable's type on the server). + */ + public QwpBindValues setNull(int index, byte qwpTypeCode) { + checkBindType(qwpTypeCode); + if (qwpTypeCode == QwpConstants.TYPE_DECIMAL64) { + return setNullDecimal64(index, 0); + } + if (qwpTypeCode == QwpConstants.TYPE_DECIMAL128) { + return setNullDecimal128(index, 0); + } + if (qwpTypeCode == QwpConstants.TYPE_DECIMAL256) { + return setNullDecimal256(index, 0); + } + if (qwpTypeCode == QwpConstants.TYPE_GEOHASH) { + return setNullGeohash(index, GEOHASH_MIN_BITS); + } + advance(index); + writeHeader(qwpTypeCode, true); + return this; + } + + /** + * Binds an explicit NULL with DECIMAL128 type and the given scale. The + * server reads the scale byte regardless of null, so the scale must be + * supplied even for NULL (it becomes part of the bound variable's type). + */ + public QwpBindValues setNullDecimal128(int index, int scale) { + checkScale128(scale); + advance(index); + writeHeader(QwpConstants.TYPE_DECIMAL128, true); + writer.putByte((byte) scale); + return this; + } + + /** + * Binds an explicit NULL with DECIMAL256 type and the given scale. See + * {@link #setNullDecimal128} for the rationale. + */ + public QwpBindValues setNullDecimal256(int index, int scale) { + checkScale256(scale); + advance(index); + writeHeader(QwpConstants.TYPE_DECIMAL256, true); + writer.putByte((byte) scale); + return this; + } + + /** + * Binds an explicit NULL with DECIMAL64 type and the given scale. See + * {@link #setNullDecimal128} for the rationale. + */ + public QwpBindValues setNullDecimal64(int index, int scale) { + checkScale64(scale); + advance(index); + writeHeader(QwpConstants.TYPE_DECIMAL64, true); + writer.putByte((byte) scale); + return this; + } + + /** + * Binds an explicit NULL with GEOHASH type and the given precision (bits). + * The server reads the precision_bits varint regardless of null, so + * precision must be supplied even for NULL (it becomes part of the bound + * variable's type). + */ + public QwpBindValues setNullGeohash(int index, int precisionBits) { + checkGeohashPrecision(precisionBits); + advance(index); + writeHeader(QwpConstants.TYPE_GEOHASH, true); + writer.putVarint(precisionBits); + return this; + } + + public QwpBindValues setShort(int index, short value) { + advance(index); + writeHeader(QwpConstants.TYPE_SHORT, false); + writer.putShort(value); + return this; + } + + public QwpBindValues setTimestampMicros(int index, long microsSinceEpoch) { + advance(index); + writeHeader(QwpConstants.TYPE_TIMESTAMP, false); + writer.putLong(microsSinceEpoch); + return this; + } + + public QwpBindValues setTimestampNanos(int index, long nanosSinceEpoch) { + advance(index); + writeHeader(QwpConstants.TYPE_TIMESTAMP_NANOS, false); + writer.putLong(nanosSinceEpoch); + return this; + } + + public QwpBindValues setUuid(int index, long lo, long hi) { + advance(index); + writeHeader(QwpConstants.TYPE_UUID, false); + writer.putLong(lo); + writer.putLong(hi); + return this; + } + + /** + * Convenience overload. Encodes {@link UUID#getLeastSignificantBits()} as + * the lo limb and {@link UUID#getMostSignificantBits()} as the hi limb, + * matching how QuestDB's UUID type is laid out internally. + */ + public QwpBindValues setUuid(int index, UUID uuid) { + if (uuid == null) { + return setNull(index, QwpConstants.TYPE_UUID); + } + return setUuid(index, uuid.getLeastSignificantBits(), uuid.getMostSignificantBits()); + } + + /** + * Encodes a VARCHAR bind. A {@code null} value is written as a typed NULL. + * Strings are encoded as: {@code u32 offset0=0 | u32 length_bytes | UTF-8 bytes}. + * The length must fit in a signed int32 (the server rejects negative + * lengths). + */ + public QwpBindValues setVarchar(int index, CharSequence value) { + if (value == null) { + return setNull(index, QwpConstants.TYPE_VARCHAR); + } + advance(index); + writeHeader(QwpConstants.TYPE_VARCHAR, false); + // Fast path for ASCII-only; fall back to UTF-8 re-encode for non-ASCII. + int charLen = value.length(); + if (isAscii(value, charLen)) { + writer.putInt(0); // offset[0] + writer.putInt(charLen); // offset[1] = UTF-8 length (== charLen for ASCII) + writer.ensureCapacity(charLen); + long addr = writer.getWriteAddress(); + for (int i = 0; i < charLen; i++) { + Unsafe.getUnsafe().putByte(addr + i, (byte) value.charAt(i)); + } + writer.skip(charLen); + return this; + } + byte[] utf8 = value.toString().getBytes(StandardCharsets.UTF_8); + writer.putInt(0); + writer.putInt(utf8.length); + writer.ensureCapacity(utf8.length); + long addr = writer.getWriteAddress(); + for (int i = 0; i < utf8.length; i++) { + Unsafe.getUnsafe().putByte(addr + i, utf8[i]); + } + writer.skip(utf8.length); + return this; + } + + /** + * Number of bytes of encoded bind payload currently in the buffer. Used + * by {@link QwpQueryClient} to hand the payload to the I/O thread; not + * intended for user code. + */ + public long bufferLen() { + return writer.getPosition(); + } + + /** + * Native pointer to the encoded bind payload. Used by + * {@link QwpQueryClient} to hand the payload to the I/O thread; not + * intended for user code. Valid only until the next {@link #reset()} + * (implicit at the start of every {@code execute} call). + */ + public long bufferPtr() { + return writer.getBufferPtr(); + } + + /** + * Number of binds that have been written since the last {@link #reset()}. + * Used by {@link QwpQueryClient} to emit the {@code bind_count} varint; + * not intended for user code. + */ + public int count() { + return count; + } + + /** + * Clears prior state so this instance can accumulate binds for a new + * query. Called by {@link QwpQueryClient} at the start of every + * {@code execute}; not intended for user code. + */ + public void reset() { + writer.reset(); + count = 0; + expectedNextIndex = 0; + } + + private static void checkGeohashPrecision(int precisionBits) { + if (precisionBits < GEOHASH_MIN_BITS || precisionBits > GEOHASH_MAX_BITS) { + throw new IllegalArgumentException( + "GEOHASH precision must be in [" + GEOHASH_MIN_BITS + ", " + GEOHASH_MAX_BITS + + "], got " + precisionBits); + } + } + + private static boolean isAscii(CharSequence value, int charLen) { + for (int i = 0; i < charLen; i++) { + if (value.charAt(i) >= 0x80) { + return false; + } + } + return true; + } + + private static long maskGeohashBits(long value, int precisionBits) { + return precisionBits >= 64 ? value : value & ((1L << precisionBits) - 1L); + } + + private void advance(int index) { + if (index != expectedNextIndex) { + throw new IllegalStateException( + "bind index out of order: expected " + expectedNextIndex + ", got " + index); + } + if (count >= QwpConstants.MAX_COLUMNS_PER_TABLE) { + throw new IllegalStateException( + "too many binds: exceeds " + QwpConstants.MAX_COLUMNS_PER_TABLE); + } + expectedNextIndex++; + count++; + } + + private void checkBindType(byte type) { + switch (type) { + case QwpConstants.TYPE_BOOLEAN: + case QwpConstants.TYPE_BYTE: + case QwpConstants.TYPE_SHORT: + case QwpConstants.TYPE_CHAR: + case QwpConstants.TYPE_INT: + case QwpConstants.TYPE_LONG: + case QwpConstants.TYPE_FLOAT: + case QwpConstants.TYPE_DOUBLE: + case QwpConstants.TYPE_DATE: + case QwpConstants.TYPE_TIMESTAMP: + case QwpConstants.TYPE_TIMESTAMP_NANOS: + case QwpConstants.TYPE_UUID: + case QwpConstants.TYPE_LONG256: + case QwpConstants.TYPE_GEOHASH: + case QwpConstants.TYPE_VARCHAR: + case QwpConstants.TYPE_DECIMAL64: + case QwpConstants.TYPE_DECIMAL128: + case QwpConstants.TYPE_DECIMAL256: + return; + default: + throw new IllegalArgumentException( + "unsupported bind type 0x" + Integer.toHexString(type & 0xFF)); + } + } + + private void checkScale128(int scale) { + if (scale < 0 || scale > DECIMAL128_MAX_SCALE) { + throw new IllegalArgumentException( + "DECIMAL128 scale must be in [0, " + DECIMAL128_MAX_SCALE + "], got " + scale); + } + } + + private void checkScale256(int scale) { + if (scale < 0 || scale > DECIMAL256_MAX_SCALE) { + throw new IllegalArgumentException( + "DECIMAL256 scale must be in [0, " + DECIMAL256_MAX_SCALE + "], got " + scale); + } + } + + private void checkScale64(int scale) { + if (scale < 0 || scale > DECIMAL64_MAX_SCALE) { + throw new IllegalArgumentException( + "DECIMAL64 scale must be in [0, " + DECIMAL64_MAX_SCALE + "], got " + scale); + } + } + + private void writeHeader(byte type, boolean isNull) { + writer.putByte(type); + if (isNull) { + writer.putByte(NULL_FLAG); + writer.putByte(NULL_BITMAP); + } else { + writer.putByte(NON_NULL_FLAG); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatch.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatch.java new file mode 100644 index 00000000..6ea2a029 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatch.java @@ -0,0 +1,755 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.Long256Sink; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Uuid; +import io.questdb.client.std.bytes.DirectByteSequence; +import io.questdb.client.std.bytes.DirectByteSlice; +import io.questdb.client.std.str.CharSink; +import io.questdb.client.std.str.DirectUtf8Sequence; +import io.questdb.client.std.str.DirectUtf8String; + +import java.nio.charset.StandardCharsets; + +/** + * Column-major view over one decoded {@code RESULT_BATCH}. Handed to + * {@link QwpColumnBatchHandler#onBatch} and valid only for the duration of that + * callback; all pointers and row indices become stale once control returns to + * the decoder. To retain a value past the callback, copy it out. + *

+ * Threading: not thread-safe. The batch is produced on the + * client's I/O thread and consumed on whichever thread calls + * {@code onBatch}. Dispatching work to another thread requires copying values + * first. + *

+ * Indexing: {@code col} and {@code row} are 0-based. Out-of-range + * values surface as {@link ArrayIndexOutOfBoundsException}. + *

+ * Type contract: each typed accessor is valid only for a specific + * wire type (or small family of compatible wire types). Cross-check with + * {@link #getColumnWireType(int)} before dispatching. Calling an accessor whose + * type doesn't match the column's wire type yields undefined results -- + * the underlying native read uses a stride that matches the declared type, so + * you will silently get bytes at the wrong offset. + *

+ * NULL handling: call {@link #isNull(int, int)} when NULL vs. + * value must be distinguished. When a typed accessor is called on a NULL row + * the returned value is a type-dependent zero: {@code 0} / {@code 0L} for + * integer and boolean accessors, {@code Double.NaN} / {@code Float.NaN} for + * floating-point, {@code null} for reference types, {@code false} for sink + * accessors. Sink accessors leave the sink's previous contents untouched on + * NULL. Per-accessor NULL behaviour is not repeated in method docs; this + * paragraph is the contract. + *

+ * Five zero-allocation idioms -- pick whichever matches the + * wire type of the column: + *

    + *
  • Primitive accessors ({@link #getLongValue}, {@link #getIntValue}, + * {@link #getDoubleValue}, {@link #getBoolValue}, etc.) read fixed-width + * values straight from the payload buffer with one native load.
    • + *
  • UTF-8 views ({@link #getStrA}, {@link #getStrB}) return reusable + * {@link DirectUtf8Sequence} flyweights pointing into the payload (or into + * the symbol dict heap for SYMBOL columns). Each call re-points the view, + * so hold at most two live views at once via the A/B pair.
    • + *
  • Sink accessors ({@link #getLong256(int, int, Long256Sink)}, + * {@link #getUuid(int, int, Uuid)}, + * {@link #getString(int, int, CharSink)}) copy multi-word or variable-width + * values into a caller-owned sink in a single call. Reuse the sink across + * rows.
    • + *
  • Symbol cache ({@link #getSymbol}, {@link #getSymbolForId}, + * {@link #getSymbolId}) materialises each distinct SYMBOL dict entry into + * a {@link String} at most once per batch, regardless of how many rows + * reference it.
    • + *
  • Row / column views ({@link #row(int)}, {@link #column(int)}, + * {@link #forEachRow(RowCallback)}) wrap the {@code (col, row)} primitives + * with single-arg accessors. {@link RowView} pins the current row; + * {@link ColumnView} pins the column once and exposes a direct-address + * surface ({@link ColumnView#valuesAddr()}, {@link ColumnView#nullBitmapAddr()}, + * etc.) for SIMD or JNI consumers. Both flyweights are batch-owned and + * reused.
    • + *
+ * The heap-allocating convenience accessors {@link #getString(int, int)}, + * {@link #getBinary(int, int)}, and {@link #getDoubleArrayElements(int, int)} + * allocate per call and are meant for low-volume access only. + */ +public class QwpColumnBatch { + + final ObjList columnLayouts = new ObjList<>(); + // BINARY views -- re-pointed per call, never re-allocated. + private final DirectByteSlice binaryA = new DirectByteSlice(); + private final DirectByteSlice binaryB = new DirectByteSlice(); + // One ColumnView per column index, lazily created by column(int). Slots are + // re-pointed in place across batches; the list grows but never shrinks so a + // wide-then-narrow query sequence keeps reusing the same instances. + private final ObjList columnViews = new ObjList<>(); + // Reusable views for zero-alloc UTF-8 access. strA and strB are dual views + // (same pattern as QuestDB Record.getStrA/getStrB) so callers can compare + // two cells without one overwriting the other. + private final DirectUtf8String strA = new DirectUtf8String(); + private final DirectUtf8String strB = new DirectUtf8String(); + long batchSeq; + int columnCount; + ObjList columns; + long payloadAddr; + long payloadLimit; + long requestId; + int rowCount; + // Lazily created on first row(int) / forEachRow call; reused for the batch lifetime. + private RowView rowView; + + /** + * Server-assigned monotonic sequence number for this batch within the query. + * First batch is 0. Useful for cross-referencing client-side timings with + * server logs or for detecting batch gaps. + */ + public long batchSeq() { + return batchSeq; + } + + /** + * Returns the {@link ColumnView} for {@code col}, pinned and ready to read. + * The view is owned by the batch and cached per column index, so calling + * {@code column(0)} and {@code column(1)} returns two distinct instances + * that can be held side-by-side. Calling {@code column(c)} a second time + * with the same {@code c} returns the same instance with its layout + * pointer refreshed against the current batch state. + */ + public ColumnView column(int col) { + if (columnViews.size() <= col) { + columnViews.setPos(col + 1); + } + ColumnView view = columnViews.getQuick(col); + if (view == null) { + view = new ColumnView(this); + columnViews.setQuick(col, view); + } + return view.of(col); + } + + /** + * Iterates rows in this batch and invokes {@code callback} for each one. + * The {@link RowView} handed to the callback is re-pointed in place across + * iterations -- do not retain it past {@link RowCallback#onRow(RowView)}. + * Throwing from the callback aborts iteration and propagates the exception. + */ + public void forEachRow(RowCallback callback) { + if (rowView == null) { + rowView = new RowView(this); + } + int n = rowCount; + for (int r = 0; r < n; r++) { + rowView.of(r); + callback.onRow(rowView); + } + } + + /** + * Returns the dimensionality of the ARRAY value at {@code (col, row)}, or 0 if + * the row is null. Caller must know the column is an ARRAY type. + */ + public int getArrayNDims(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0; + return Unsafe.getUnsafe().getByte(l.arrayRowAddr[row]) & 0xFF; + } + + /** + * Heap-allocating convenience. Returns the raw bytes of a BINARY value. Allocates + * a new {@code byte[]} per call; on the hot path prefer {@link #getBinaryA} which + * returns a reusable native view. + */ + public byte[] getBinary(int col, int row) { + io.questdb.client.std.bytes.DirectByteSequence v = lookupBinaryBytes(col, row, binaryA); + if (v == null) return null; + int size = v.size(); + byte[] bytes = new byte[size]; + for (int i = 0; i < size; i++) { + bytes[i] = v.byteAt(i); + } + return bytes; + } + + /** + * Zero-allocation BINARY view. Returns a {@link DirectByteSequence} + * pointing into the WebSocket payload buffer. The view is invalidated by the next call to + * {@link #getBinaryB} on this batch or once the enclosing + * {@code onBatch} callback returns. + */ + public DirectByteSequence getBinaryA(int col, int row) { + return lookupBinaryBytes(col, row, binaryA); + } + + /** + * Dual of {@link #getBinaryA}; use when you need to hold two binary views concurrently. + */ + public io.questdb.client.std.bytes.DirectByteSequence getBinaryB(int col, int row) { + return lookupBinaryBytes(col, row, binaryB); + } + + /** + * Returns a single BOOLEAN value. Caller must know the column is BOOLEAN. + * Values are bit-packed on the wire (8 per byte). + */ + public boolean getBoolValue(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return false; + int denseIdx = l.denseIndex(row); + // Bit-packed: 8 values per byte, LSB-first + byte b = Unsafe.getUnsafe().getByte(l.valuesAddr + (denseIdx >>> 3)); + return (b & (1 << (denseIdx & 7))) != 0; + } + + /** + * Returns a single BYTE value. The caller must know the column is BYTE. + */ + public byte getByteValue(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0; + return Unsafe.getUnsafe().getByte(l.valuesAddr + l.denseIndex(row)); + } + + /** + * Returns a single CHAR value. Caller must know the column is CHAR. + */ + public char getCharValue(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0; + return (char) Unsafe.getUnsafe().getShort(l.valuesAddr + 2L * l.denseIndex(row)); + } + + /** + * Number of columns in the result schema. Matches the projection of the SELECT. + */ + public int getColumnCount() { + return columnCount; + } + + /** + * Column name as declared by the schema (e.g. the SELECT alias or underlying + * column identifier). Same across every batch of the query. + */ + public String getColumnName(int col) { + return columns.getQuick(col).name; + } + + /** + * QWP wire type code for the column, used to choose the right typed accessor. + * See {@link io.questdb.client.cutlass.qwp.protocol.QwpConstants} for the + * {@code TYPE_*} constants. + */ + public byte getColumnWireType(int col) { + return columns.getQuick(col).wireType; + } + + /** + * Returns the high 64 bits of a DECIMAL128 value. Combine with {@link #getDecimal128Low}. + */ + public long getDecimal128High(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0L; + return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.denseIndex(row) + 8L); + } + + /** + * Returns the low 64 bits of a DECIMAL128 value. + */ + public long getDecimal128Low(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0L; + return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.denseIndex(row)); + } + + /** + * Scale (number of fractional digits) for DECIMAL64 / DECIMAL128 / DECIMAL256 + * columns. Same across every row in the batch. Undefined for non-DECIMAL columns. + */ + public int getDecimalScale(int col) { + return columns.getQuick(col).scale; + } + + /** + * Returns the flattened elements of a DOUBLE_ARRAY value in row-major order. + * Heap-allocating convenience; use {@link #getArrayNDims} to discover + * dimensionality separately if you need it. + */ + public double[] getDoubleArrayElements(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return null; + long rowAddr = l.arrayRowAddr[row]; + int nDims = Unsafe.getUnsafe().getByte(rowAddr) & 0xFF; + int elements = 1; + for (int d = 0; d < nDims; d++) { + elements *= Unsafe.getUnsafe().getInt(rowAddr + 1 + 4L * d); + } + double[] out = new double[elements]; + long base = rowAddr + 1 + 4L * nDims; + for (int i = 0; i < elements; i++) { + out[i] = Unsafe.getUnsafe().getDouble(base + 8L * i); + } + return out; + } + + /** + * Returns a single DOUBLE value. Caller must know the column is DOUBLE. + */ + public double getDoubleValue(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return Double.NaN; + return Unsafe.getUnsafe().getDouble(l.valuesAddr + 8L * l.denseIndex(row)); + } + + /** + * Returns a single FLOAT value. Caller must know the column is FLOAT. + */ + public float getFloatValue(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return Float.NaN; + return Unsafe.getUnsafe().getFloat(l.valuesAddr + 4L * l.denseIndex(row)); + } + + /** + * Precision (in bits) of a GEOHASH column, in the range 1..60. Same across + * every row in the batch. Undefined for non-GEOHASH columns. + */ + public int getGeohashPrecisionBits(int col) { + return columns.getQuick(col).precisionBits; + } + + /** + * Returns a GEOHASH value packed into a long (up to 60 bits of precision). + * Caller must know the column is GEOHASH; use {@link #getGeohashPrecisionBits} + * to retrieve the bit width. + */ + public long getGeohashValue(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0L; + int denseIdx = l.denseIndex(row); + int bytesPerValue = (l.info.precisionBits + 7) >>> 3; + long p = l.valuesAddr + (long) bytesPerValue * denseIdx; + long bits = 0; + for (int b = 0; b < bytesPerValue; b++) { + bits |= ((long) (Unsafe.getUnsafe().getByte(p + b) & 0xFF)) << (b * 8); + } + return bits; + } + + /** + * Returns a single INT value. Caller must know the column is INT or IPv4. + */ + public int getIntValue(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0; + return Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * l.denseIndex(row)); + } + + /** + * Zero-allocation read of a LONG256 or DECIMAL256 value. Copies all four + * 64-bit words into {@code sink} in a single call. Returns {@code true} + * on a hit, {@code false} for NULL rows (the sink is left untouched). + * Prefer this over four {@link #getLong256Word} calls on the hot path -- + * one virtual dispatch instead of four, one address computation instead + * of four. + */ + public boolean getLong256(int col, int row, Long256Sink sink) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return false; + sink.fromAddress(l.valuesAddr + 32L * l.denseIndex(row)); + return true; + } + + /** + * Returns one of the four 64-bit words of a LONG256 or DECIMAL256 value. + * {@code wordIndex} 0 is least significant, 3 is most significant. For + * bulk reads prefer {@link #getLong256(int, int, Long256Sink)}. + */ + public long getLong256Word(int col, int row, int wordIndex) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0L; + return Unsafe.getUnsafe().getLong(l.valuesAddr + 32L * l.denseIndex(row) + 8L * wordIndex); + } + + /** + * Returns an 8-byte LONG / TIMESTAMP / DATE / DECIMAL64 value. + * Caller must know the column is a LONG-family type. + */ + public long getLongValue(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0L; + return Unsafe.getUnsafe().getLong(l.valuesAddr + 8L * l.denseIndex(row)); + } + + /** + * Number of rows in this batch, including NULL rows. The server caps this + * to its {@code max_batch_rows} setting (typically 4096), so a query that + * returns more rows arrives split across multiple {@code onBatch} calls. + */ + public int getRowCount() { + return rowCount; + } + + /** + * Returns a SHORT value without type dispatch. Caller must know the column is SHORT. + */ + public short getShortValue(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0; + return Unsafe.getUnsafe().getShort(l.valuesAddr + 2L * l.denseIndex(row)); + } + + /** + * Zero-allocation UTF-8 view over the STRING / VARCHAR / SYMBOL value at + * {@code (col, row)}. For STRING / VARCHAR the view points into the payload + * buffer; for SYMBOL it points into the per-batch symbol dictionary heap + * (same lifetime either way -- both are invalidated when {@code onBatch} + * returns). The returned view is also invalidated by the next call to + * {@code getStrA} on this batch; use {@link #getStrB} for the dual slot + * when two concurrent views are needed. + */ + public DirectUtf8Sequence getStrA(int col, int row) { + return lookupStringBytes(col, row, strA); + } + + /** + * Dual of {@link #getStrA}; use when you need to hold two string views + * concurrently (e.g. row-vs-row comparisons within one batch). + */ + public DirectUtf8Sequence getStrB(int col, int row) { + return lookupStringBytes(col, row, strB); + } + + /** + * Heap-allocating convenience. Returns a {@link String} for STRING / SYMBOL / + * VARCHAR columns. For STRING / VARCHAR each call allocates a new String; on + * the hot path prefer {@link #getStrA} or {@link #getString(int, int, CharSink)}. + * For SYMBOL the result is cached per dict entry (see {@link #getSymbol}) so + * repeated calls against rows sharing a dict entry return the same String + * instance. + */ + public String getString(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return null; + if (l.info.wireType == QwpConstants.TYPE_SYMBOL) { + return lookupCachedSymbol(l, l.symbolRowIds[row]); + } + DirectUtf8Sequence v = lookupStringBytes(col, row, strA); + if (v == null) return null; + int size = v.size(); + byte[] bytes = new byte[size]; + for (int i = 0; i < size; i++) { + bytes[i] = v.byteAt(i); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + /** + * Zero-allocation variant of {@link #getString(int, int)}. Writes the STRING / + * SYMBOL / VARCHAR value at {@code (col, row)} into {@code sink} -- UTF-8 bytes + * pass straight through to a {@link io.questdb.client.std.str.Utf8Sink}; a + * {@link io.questdb.client.std.str.Utf16Sink} transcodes to UTF-16 en route. + * Returns {@code true} when a value was written, {@code false} when the row is + * NULL (the sink is left untouched). + */ + public boolean getString(int col, int row, CharSink sink) { + DirectUtf8Sequence v = lookupStringBytes(col, row, strA); + if (v == null) return false; + sink.put(v); + return true; + } + + /** + * Materialises the SYMBOL value at {@code (col, row)} as a {@link String}. + * The result is cached per dict entry on the batch, so a scan over N rows + * that share K distinct symbols allocates at most K Strings -- the same + * String instance is returned for every row pointing at the same dict + * entry. + */ + public String getSymbol(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return null; + return lookupCachedSymbol(l, l.symbolRowIds[row]); + } + + /** + * Number of distinct entries in the SYMBOL column's dictionary for this batch. + * Caller must know the column is SYMBOL. In delta mode this equals the size + * of the connection-scoped dictionary at the time the batch was decoded. + */ + public int getSymbolDictSize(int col) { + return columnLayouts.getQuick(col).symbolDictSize; + } + + /** + * Materialises the dict entry at {@code dictId} as a {@link String}, with the + * same per-entry caching as {@link #getSymbol}. Use together with + * {@link #getSymbolId} to walk rows by id instead of value -- e.g. to key a + * {@code HashMap} on dict id without ever allocating a String. + * Caller must know the column is SYMBOL. + */ + public String getSymbolForId(int col, int dictId) { + return lookupCachedSymbol(columnLayouts.getQuick(col), dictId); + } + + /** + * Dict id for the SYMBOL value at {@code (col, row)}, or {@code -1} for NULL + * rows. Ids are stable across a batch and allow id-based processing (see + * {@link #getSymbolForId}). Caller must know the column is SYMBOL. + */ + public int getSymbolId(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return -1; + return l.symbolRowIds[row]; + } + + /** + * Zero-allocation read of a UUID value. Copies both 64-bit words into + * {@code sink} in a single call. Returns {@code true} on a hit, + * {@code false} for NULL rows (the sink is left untouched). Prefer this + * over paired {@link #getUuidLo} / {@link #getUuidHi} calls on the hot + * path -- one virtual dispatch instead of two, one address computation + * instead of two. + */ + public boolean getUuid(int col, int row, Uuid sink) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return false; + sink.fromAddress(l.valuesAddr + 16L * l.denseIndex(row)); + return true; + } + + /** + * Returns the high 64 bits of a UUID value. For bulk reads prefer + * {@link #getUuid(int, int, Uuid)}. + */ + public long getUuidHi(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0L; + return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.denseIndex(row) + 8L); + } + + /** + * Returns the low 64 bits of a UUID value. + */ + public long getUuidLo(int col, int row) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (isLayoutNull(l, row)) return 0L; + return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.denseIndex(row)); + } + + /** + * True if the cell is NULL on the wire. + *

+ * Note on type-specific sentinels (see {@code docs/QWP_EGRESS_EXTENSION.md} sec 11.5): + * QuestDB stores NULL as a sentinel value for several types -- {@code Long.MIN_VALUE} + * for LONG/INT/etc., {@code 0.0.0.0} for IPv4, {@code -1} for GEOHASH, and crucially + * {@code NaN} for FLOAT and DOUBLE. Egress preserves these conventions: a row carrying + * NaN in a DOUBLE column will return {@code true} from this method. Callers who need + * to distinguish "real NaN" from "explicit NULL" cannot do so over the wire -- both + * map to the same null bitmap bit. + */ + public boolean isNull(int col, int row) { + return isLayoutNull(columnLayouts.getQuick(col), row); + } + + /** + * Number of non-null rows in the column for this batch. Equal to + * {@link #getRowCount()} minus the NULL-row count. + */ + public int nonNullCount(int col) { + return columnLayouts.getQuick(col).nonNullCount; + } + + /** + * Per-row lookup table. {@code result[row]} is the dense index within the + * column's non-null values, or -1 if the row is NULL. Array length equals + * {@link #getRowCount()}. Valid only during the current {@code onBatch} + * callback; do not retain. + *

+ * Most callers don't need this -- the typed accessors + * ({@link #getLongValue}, {@link #getDoubleValue}, etc.) already resolve + * dense indices internally. Use only when writing custom readers against + * the raw column-address API. + *

+ * For columns with no nulls the decoder skips populating this array, so + * this accessor lazy-materialises an identity mapping on demand; the + * result is cached until the next batch. + */ + public int[] nonNullIndex(int col) { + QwpColumnLayout l = columnLayouts.getQuick(col); + if (l.nullBitmapAddr == 0 && l.nonNullIdx == null) { + int[] arr = new int[rowCount]; + for (int i = 0; i < rowCount; i++) arr[i] = i; + l.nonNullIdx = arr; + } + return l.nonNullIdx; + } + + /** + * Starting address of the RESULT_BATCH payload this view was decoded from. + * Equal to the WebSocket payload pointer when in-place decode is in use. + * Intended for accounting (byte counters) rather than data access. + */ + public long payloadAddr() { + return payloadAddr; + } + + /** + * Exclusive upper bound of the RESULT_BATCH payload bytes. See + * {@link #payloadAddr()}. + */ + public long payloadLimit() { + return payloadLimit; + } + + /** + * Client-assigned request id this batch belongs to. Matches the id the I/O + * thread attached to the {@code QUERY_REQUEST} frame. Useful for correlating + * batches with {@code execute} calls in diagnostics. + */ + public long requestId() { + return requestId; + } + + /** + * Returns the cached {@link RowView} pointed at {@code row}. Use as an + * escape hatch from {@link #forEachRow(RowCallback)} when you need to + * drive iteration manually (e.g., interleaved with another data source). + * Single shared instance per batch; two calls overwrite each other. + */ + public RowView row(int row) { + if (rowView == null) { + rowView = new RowView(this); + } + return rowView.of(row); + } + + /** + * Address of the column's packed non-null values in the payload buffer. + * Layout depends on the wire type: + *

    + *
  • Fixed-width (LONG, INT, DOUBLE, UUID, LONG256, etc.): contiguous values, index by {@code nonNullIndex(col)[row] * sizeBytes}.
    • + *
  • BOOLEAN: bit-packed, 8 values per byte, LSB-first; index by {@code nonNullIndex(col)[row]}.
    • + *
  • STRING / VARCHAR: points to the (N+1) x uint32 offsets array; use for the bytes region.
    • + *
  • GEOHASH: {@code ceil(precisionBits / 8)} bytes per value; index by {@code nonNullIndex(col)[row] * bytesPerValue}.
    • + *
  • DECIMAL64/128/256: the scale byte has already been consumed; this is the first unscaled value.
    • + *
  • SYMBOL: not meaningful -- use {@link #getStrA} / {@link #getStrB} instead.
    • + *
  • ARRAY: not meaningful -- use the per-row {@code arrayRowAddr} accessors (forthcoming).
    • + *
+ */ + public long valuesAddr(int col) { + return columnLayouts.getQuick(col).valuesAddr; + } + + /** + * Resolves the {@code (col, row)} cell for a BINARY column and points the supplied + * slice at the underlying bytes in the payload buffer. Returns {@code null} for NULL + * rows or if the column is not BINARY. + */ + private io.questdb.client.std.bytes.DirectByteSequence lookupBinaryBytes( + int col, int row, io.questdb.client.std.bytes.DirectByteSlice view) { + if (isNull(col, row)) return null; + QwpColumnLayout l = columnLayouts.getQuick(col); + if (l.info.wireType != QwpConstants.TYPE_BINARY) return null; + int denseIdx = l.denseIndex(row); + int startOff = Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * denseIdx); + int endOff = Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * (denseIdx + 1)); + return view.of(l.stringBytesAddr + startOff, endOff - startOff); + } + + /** + * Lazily materialises and caches the String for a SYMBOL dict entry. The + * cache lives on the {@link QwpColumnLayout} and survives across batches in + * delta mode, where {@code dictId} resolves to the same bytes for the life + * of the connection-scoped dict. When the decoder re-stamps + * {@link QwpColumnLayout#symbolDictVersion} (non-delta batch, or CACHE_RESET + * bump of the dict generation), this method wipes the cache on the first + * lookup and catches {@link QwpColumnLayout#symbolCacheVersion} up to the + * current version. + */ + private String lookupCachedSymbol(QwpColumnLayout l, int dictId) { + ObjList cache = l.symbolStringCache; + if (l.symbolCacheVersion != l.symbolDictVersion) { + cache.clear(); + l.symbolCacheVersion = l.symbolDictVersion; + } + if (cache.size() < l.symbolDictSize) { + cache.setPos(l.symbolDictSize); + } + String s = cache.getQuick(dictId); + if (s == null) { + long packed = Unsafe.getUnsafe().getLong(l.symbolDictEntriesAddr + ((long) dictId << 3)); + long start = l.symbolDictHeapAddr + (packed & 0xFFFFFFFFL); + int len = (int) (packed >>> 32); + byte[] bytes = new byte[len]; + for (int i = 0; i < len; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(start + i); + } + s = new String(bytes, StandardCharsets.UTF_8); + cache.setQuick(dictId, s); + } + return s; + } + + /** + * Resolves the {@code (col, row)} cell for STRING / VARCHAR / SYMBOL columns and + * points the supplied view at the underlying bytes in the payload buffer. + * Returns {@code null} for NULL rows or unsupported wire types. + */ + private DirectUtf8Sequence lookupStringBytes(int col, int row, DirectUtf8String view) { + if (isNull(col, row)) return null; + QwpColumnLayout l = columnLayouts.getQuick(col); + byte wt = l.info.wireType; + int denseIdx = l.denseIndex(row); + if (wt == QwpConstants.TYPE_VARCHAR) { + int startOff = Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * denseIdx); + int endOff = Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * (denseIdx + 1)); + return view.of(l.stringBytesAddr + startOff, l.stringBytesAddr + endOff); + } + if (wt == QwpConstants.TYPE_SYMBOL) { + int dictIdx = l.symbolRowIds[row]; + // Single 64-bit load: low 32 bits = offset into dict heap, high 32 = length. + // No ObjList.getQuick, no DirectUtf8String deref -- pure pointer arithmetic. + long packed = Unsafe.getUnsafe().getLong(l.symbolDictEntriesAddr + ((long) dictIdx << 3)); + long start = l.symbolDictHeapAddr + (packed & 0xFFFFFFFFL); + long end = start + (packed >>> 32); + return view.of(start, end); + } + return null; + } + + /** + * Fast null check once the layout is in hand. Inlining pattern used by all the + * typed accessors: load layout once, check bitmap, read value. Eliminates the + * second {@code ObjList.getQuick(col)} that separate {@code isNull(col,row)} would cost. + * Also reused by {@link ColumnView} so it can keep the layout pointer cached. + */ + static boolean isLayoutNull(QwpColumnLayout l, int row) { + if (l.nullBitmapAddr == 0) return false; + byte bm = Unsafe.getUnsafe().getByte(l.nullBitmapAddr + (row >>> 3)); + return (bm & (1 << (row & 7))) != 0; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java new file mode 100644 index 00000000..eb418b75 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java @@ -0,0 +1,106 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Callback interface for consuming a streamed QWP egress query result. + *

+ * Invoked by {@link QwpQueryClient#execute(String, QwpColumnBatchHandler)}: + * once per {@code RESULT_BATCH} frame via {@link #onBatch(QwpColumnBatch)}, + * then exactly once via either {@link #onEnd(long)} or + * {@link #onError(byte, String)} (or {@link #onExecDone} for non-SELECT queries). + *

+ * The {@link QwpColumnBatch} passed to {@link #onBatch} is valid only for the + * duration of that call. Copy any values you need to retain. + *

+ * Exception contract: if any callback method throws, the + * exception propagates out of the {@link QwpQueryClient#execute} call on the + * caller's thread and no further callbacks fire for that query. The connection + * remains usable for subsequent queries. + */ +public interface QwpColumnBatchHandler { + + /** + * Invoked for each {@code RESULT_BATCH} received. Throwing from here aborts + * the query (see the class-level exception contract). + * + * @param batch column-major view over the batch; valid until {@code onBatch} returns + */ + void onBatch(QwpColumnBatch batch); + + /** + * Invoked exactly once after the last batch, upon successful completion of the query. + * + * @param totalRows server-reported total row count (0 if not tracked) + */ + void onEnd(long totalRows); + + /** + * Invoked exactly once if the query fails at any point, instead of + * {@link #onEnd} / {@link #onExecDone}. + * + * @param status one of the QWP status codes (e.g., {@code STATUS_PARSE_ERROR}) + * @param message server-supplied error message (may be empty) + */ + void onError(byte status, String message); + + /** + * Invoked when {@link QwpQueryClient#execute} has transparently reconnected + * to another endpoint after a transport failure and is about to re-submit + * the query (with {@code failover=on}, the default). + *

+ * {@code newNode} is the {@link QwpServerInfo} of the endpoint the client + * just bound to, or {@code null} if the new server negotiated the v1 + * protocol (no SERVER_INFO frame). + *

+ * After this callback fires, {@link #onBatch} will be invoked again with + * {@code batch_seq} restarting at 0 on the new connection. Handlers that + * accumulate rows across batches should discard whatever they built up + * from the previous attempt. The default implementation is a no-op, safe + * for handlers that don't care (for example, simple row-count aggregators + * that are idempotent against replays) and for applications that set + * {@code failover=off}. + */ + default void onFailoverReset(QwpServerInfo newNode) { + } + + /** + * Invoked in place of {@link #onBatch} + {@link #onEnd} when the query was + * a non-SELECT (DDL, INSERT, UPDATE, etc.). No batches are delivered for + * such queries -- the server executes the statement and replies with a + * single {@code EXEC_DONE}. + *

+ * Override this method in any handler that may see non-SELECT SQL (e.g. a + * generic query runner or a test harness that issues {@code CREATE TABLE} + * before querying). SELECT-only handlers can rely on the default no-op and + * will only ever see {@link #onBatch} + {@link #onEnd} / {@link #onError}. + * + * @param opType matches one of {@code CompiledQuery.SELECT} / {@code INSERT} / + * {@code UPDATE} / {@code CREATE_TABLE} / etc. (server-side constants) + * @param rowsAffected rows inserted / updated / deleted; 0 for pure DDL + */ + default void onExecDone(short opType, long rowsAffected) { + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnLayout.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnLayout.java new file mode 100644 index 00000000..3a75d7c5 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnLayout.java @@ -0,0 +1,221 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; + +/** + * Per-column parsed layout for one batch. Holds native pointers INTO the + * currently-active WS payload buffer plus pre-computed per-row indices for + * O(1) access. Reused across batches to eliminate allocations on the hot path + * (pooled arrays grow to max observed size and never shrink). + */ +public class QwpColumnLayout implements QuietCloseable { + + /** + * SYMBOL: lazy String cache indexed by dict ID. Populated on first + * {@code getSymbol}/{@code getSymbolForId}/{@code getString}(SYMBOL) call + * so a query over 10M rows with N distinct symbols materialises only N + * Strings per batch. The cache survives across batches in delta mode (the + * connection dict is stable) and invalidates lazily when + * {@link #symbolDictVersion} disagrees with {@link #symbolCacheVersion} -- + * see {@code QwpColumnBatch#lookupCachedSymbol}. + */ + final ObjList symbolStringCache = new ObjList<>(); + /** + * ARRAY: per-row starting offset (absolute address) of the array bytes. -1 for NULL rows. + */ + long[] arrayRowAddr; + /** + * ARRAY: per-row length in bytes of the array payload. + */ + int[] arrayRowLen; + /** + * Schema column metadata (name, wire type, scale, precisionBits). + */ + QwpEgressColumnInfo info; + /** + * Absolute address of the first byte after this column's data -- used to walk to the next column. + */ + long nextAddr; + /** + * Count of non-null rows in this column. + */ + int nonNullCount; + /** + * Per-row lookup: {@code nonNullIdx[row]} is the dense index of row {@code row} within + * the non-null values, or -1 if the row is NULL. Sized to {@code rowCount}. + * Pool-owned; re-used across batches. + */ + int[] nonNullIdx; + /** + * Absolute payload address of the null bitmap, or 0 if the column has no NULL rows. + */ + long nullBitmapAddr; + /** + * STRING / VARCHAR: absolute address of the concatenated UTF-8 bytes (right after the offsets array). + */ + long stringBytesAddr; + /** + * Last {@link #symbolDictVersion} the {@link #symbolStringCache} was known + * valid under. Updated at cache read time (not write time) -- if the dict + * has been re-stamped since the last lookup, the cache is wiped and this + * field catches up to the current version before the lookup proceeds. + */ + long symbolCacheVersion; + /** + * SYMBOL: absolute address of the native entries array. Each entry is a packed + * 8-byte pair {@code (offset:i32 | length:i32<<32)} relative to + * {@link #symbolDictHeapAddr}. Access in the hot path is a single 64-bit + * load + two int extractions, no object dereferences. + */ + long symbolDictEntriesAddr; + /** + * SYMBOL: absolute address of the UTF-8 bytes heap holding all dict entries. In + * delta mode this points into the decoder's connection-scoped native heap; in + * non-delta mode it points into the payload buffer (wire bytes directly). + */ + long symbolDictHeapAddr; + /** + * SYMBOL: number of valid entries referenced by {@link #symbolDictEntriesAddr}. + */ + int symbolDictSize; + /** + * Version of the dict currently bound to this layout. Stamped by the + * decoder at parse time. Encoding: + *

    + *
  • delta mode: {@code connDictGeneration << 1} (bit 0 clear)
    • + *
  • non-delta: {@code dictBase | 1} (bit 0 set, address changes each batch)
    • + *
+ * The split low bit stops a delta generation count from colliding with a + * non-delta address token. Consumers compare this against + * {@link #symbolCacheVersion} to decide whether the cache is still valid. + */ + long symbolDictVersion; + /** + * SYMBOL: per-row dictionary ID. Sized to {@code rowCount}; NULL rows are + * left with stale values -- use {@link #nonNullIdx}/null-check first. + */ + int[] symbolRowIds; + /** + * Absolute payload address where this column's non-null values start. For + * fixed-width types this is the dense values array. For strings/varchars + * it's the offsets array. For symbols it's where the dict starts; the + * per-row IDs are materialised into {@link #symbolRowIds} during parse. + */ + long valuesAddr; + /** + * SYMBOL non-delta only: native buffer owned by this layout that holds the + * per-batch packed entries when the column carries its own dict. In delta + * mode this is 0 and {@link #symbolDictEntriesAddr} points at the decoder's + * shared array instead. + */ + private long ownedEntriesAddr; + private int ownedEntriesCapacity; + /** + * TIMESTAMP / TIMESTAMP_NANOS / DATE Gorilla decode buffer owned by this + * layout. Populated when the per-column encoding discriminator is + * {@code 0x01}; {@link #valuesAddr} is then pointed at this buffer so the + * user-facing {@code getLong(col, row)} path sees decoded int64s. Unused + * (0) when the column was shipped uncompressed. + */ + private long timestampDecodeAddr; + private int timestampDecodeCapacity; + + public void clear() { + info = null; + valuesAddr = 0; + nullBitmapAddr = 0; + nonNullCount = 0; + stringBytesAddr = 0; + symbolDictHeapAddr = 0; + symbolDictEntriesAddr = 0; + symbolDictSize = 0; + // symbolStringCache is intentionally NOT wiped here. Lazy invalidation + // via symbolDictVersion / symbolCacheVersion keeps connection-stable + // dict entries cached across batches and wipes on the first lookup + // after a dict-version mismatch (non-delta batch or CACHE_RESET). + nextAddr = 0; + } + + @Override + public void close() { + if (ownedEntriesAddr != 0) { + Unsafe.free(ownedEntriesAddr, ownedEntriesCapacity, MemoryTag.NATIVE_DEFAULT); + ownedEntriesAddr = 0; + ownedEntriesCapacity = 0; + } + if (timestampDecodeAddr != 0) { + Unsafe.free(timestampDecodeAddr, timestampDecodeCapacity, MemoryTag.NATIVE_DEFAULT); + timestampDecodeAddr = 0; + timestampDecodeCapacity = 0; + } + } + + /** + * Returns the dense index of {@code row} into the non-null values array. + * For columns with no nulls in this batch ({@code nullBitmapAddr == 0}), + * dense index equals row and {@link #nonNullIdx} is left unread (the + * decoder skips the per-row array fill on this path). Otherwise the + * pre-computed slot is returned. + *

+ * Caller MUST have null-checked the cell first via the surrounding + * {@code isNull} / {@code isLayoutNull} guard -- this method does not + * detect null rows on its own. + */ + public int denseIndex(int row) { + return nullBitmapAddr == 0 ? row : nonNullIdx[row]; + } + + /** + * Ensures the per-layout owned entries buffer is at least {@code requiredBytes} + * and returns its address. Used by non-delta mode where the column carries its + * own dictionary inline and we need a dedicated buffer for the packed entries. + */ + long ensureOwnedEntriesAddr(int requiredBytes) { + if (ownedEntriesCapacity < requiredBytes) { + int newCap = Math.max(ownedEntriesCapacity * 2, Math.max(64, requiredBytes)); + ownedEntriesAddr = Unsafe.realloc(ownedEntriesAddr, ownedEntriesCapacity, newCap, MemoryTag.NATIVE_DEFAULT); + ownedEntriesCapacity = newCap; + } + return ownedEntriesAddr; + } + + /** + * Ensures the per-layout Gorilla decode buffer is at least {@code requiredBytes} + * and returns its address. + */ + long ensureTimestampDecodeAddr(int requiredBytes) { + if (timestampDecodeCapacity < requiredBytes) { + int newCap = Math.max(timestampDecodeCapacity * 2, Math.max(64, requiredBytes)); + timestampDecodeAddr = Unsafe.realloc(timestampDecodeAddr, timestampDecodeCapacity, newCap, MemoryTag.NATIVE_DEFAULT); + timestampDecodeCapacity = newCap; + } + return timestampDecodeAddr; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDecodeException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDecodeException.java new file mode 100644 index 00000000..3926914f --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpDecodeException.java @@ -0,0 +1,35 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Thrown by {@link QwpResultBatchDecoder} when an inbound frame is malformed + * or references unknown connection state. + */ +public class QwpDecodeException extends Exception { + public QwpDecodeException(String message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressColumnInfo.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressColumnInfo.java new file mode 100644 index 00000000..fa2f72dd --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressColumnInfo.java @@ -0,0 +1,46 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Per-column metadata recorded when a schema is registered on a client connection. + * Internal to the QWP client; callers read column attributes through the forwarder + * methods on {@link QwpColumnBatch} (for example {@link QwpColumnBatch#getColumnName}). + */ +class QwpEgressColumnInfo { + String name; + int precisionBits; // valid only for GEOHASH; set per-batch by QwpResultBatchDecoder.parseColumn + int scale; // valid only for DECIMAL*; set per-batch by QwpResultBatchDecoder.parseColumn + byte wireType; + + void of(String name, byte wireType) { + this.name = name; + this.wireType = wireType; + // scale / precisionBits come from the per-batch column data section, not the schema. + // Reset here in case a schema slot is reused for a different schema. + this.scale = 0; + this.precisionBits = 0; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java new file mode 100644 index 00000000..0569c631 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java @@ -0,0 +1,764 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.Misc; +import io.questdb.client.std.Unsafe; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Dedicated I/O thread that owns the client's {@link WebSocketClient} and drives + * receive + decode off the user thread. The user thread submits a query via + * {@link #submitQuery} and drains events via {@link #takeEvent} / {@link #releaseBuffer}; + * meanwhile the I/O thread is free to read and decode the next batch in parallel. + *

+ * A small pool of {@link QwpBatchBuffer} instances (default: 4) holds decoded + * batches in flight. When the pool is exhausted the I/O thread blocks on + * {@link #freeBuffers} until the user releases a buffer. This gives natural + * back-pressure -- if the consumer is slow, the I/O thread stops reading and + * the kernel's TCP window closes on the server side. + */ +public class QwpEgressIoThread implements Runnable, WebSocketFrameHandler { + + private static final int DEFAULT_BUFFER_CAPACITY = 64 * 1024; + private static final long POLL_TIMEOUT_MS = 100; + private static final Object RELEASE_TOKEN = new Object(); + private final QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + // Pool of reusable QueryEvent instances flowing back from the user thread + // to the I/O thread. Producer = user thread (calls {@link #releaseEvent} + // after a handler returns). Consumer = I/O thread (calls {@link #borrowEvent} + // before publishing a RESULT_BATCH). Pre-filled at construction so the + // per-batch publish path stays allocation-free after warmup. + private final QwpSpscQueue eventPool; + // Events delivered from I/O thread (producer) to user thread (consumer): + // RESULT_BATCH / RESULT_END / EXEC_DONE / QUERY_ERROR. Hot path of every + // query. Purpose-built SPSC with spin-then-park avoids the ~2 microseconds + // per offer/take that j.u.c.ArrayBlockingQueue burns on its ReentrantLock + + // Condition pair for an empty queue hand-off. + private final QwpSpscQueue events; + // Pool of pre-allocated buffers. I/O thread takes, user thread releases. + // Kept on ArrayBlockingQueue: the take uses a timeout to poll pending + // cancels while waiting, a pattern the SPSC queue doesn't offer. + private final BlockingQueue freeBuffers; + // Pending CANCEL requestId set by the user thread via {@link #requestCancel}. + // The I/O thread polls this between {@code receiveFrame} iterations; when + // non-negative, it emits a CANCEL frame for that requestId and resets to -1. + // Using AtomicLong (not volatile long) to guarantee 64-bit atomicity on all + // supported JVMs. + private final AtomicLong pendingCancelRequestId = new AtomicLong(-1L); + // One-slot release latch: user thread offers a token from releaseBuffer + // (producer), I/O thread drains it before returning from onBinaryMessage + // (consumer). Holds the payload bytes in the WebSocket recv buffer steady + // for the duration of the user handler, since in-place decode makes batch + // pointers alias those bytes. Same SPSC reasoning as {@link #events} -- + // the I/O thread typically parks on this for the duration of the user's + // onBatch callback, which is microseconds for a no-op consumer. + private final QwpSpscQueue pendingRelease = new QwpSpscQueue<>(1); + // Single-slot request queue (Phase-1 allows one in-flight query). + private final BlockingQueue requests = new ArrayBlockingQueue<>(1); + private final NativeBufferWriter sendScratch = new NativeBufferWriter(); + // Notified once when the I/O thread detects a transport- or protocol-level + // terminal failure (onClose, truncated/unknown frames, send/receive + // throwing). Distinct from per-query QUERY_ERROR, which leaves the + // connection healthy and is delivered only through the events queue. + private final TerminalFailureListener terminalFailureListener; + private final WebSocketClient wsClient; + // Per-query credit state (accessed only from the I/O thread). + // creditEnabled == (initialCredit > 0); controls whether we emit CREDIT + // replenish frames after each batch release. + // Set true by closePool() before it drains freeBuffers / events, so any + // user thread still inside releaseBuffer (or one that returns from onBatch + // after close has run) sees closed == true and frees the buffer in place + // instead of stranding it in a queue nobody will iterate again. + // Volatile because the writer (close-caller thread) and reader (user + // thread inside releaseBuffer) are different threads. + private volatile boolean closed; + private boolean creditEnabled; + private boolean currentQueryDone; + private long currentRequestId = -1L; + private volatile boolean shutdown; + + public QwpEgressIoThread(WebSocketClient wsClient, int bufferPoolSize, TerminalFailureListener terminalFailureListener) { + this.wsClient = wsClient; + this.terminalFailureListener = terminalFailureListener; + this.freeBuffers = new ArrayBlockingQueue<>(bufferPoolSize); + // +2 reserves slots for a trailing RESULT_END and a synthetic error + // frame the I/O thread may emit on shutdown, so a full buffer pool + // never stalls the producer. + this.events = new QwpSpscQueue<>(bufferPoolSize + 2); + // Pool capacity must be at least (events capacity) + 2 so that even when + // the events queue is full, the I/O thread can still borrow one event + // (being filled between borrow and offer) while the user thread holds + // another (between take and release). Pre-fill so the steady-state + // batch-publish path never allocates. + int eventPoolCapacity = bufferPoolSize + 4; + this.eventPool = new QwpSpscQueue<>(eventPoolCapacity); + for (int i = 0; i < eventPoolCapacity; i++) { + eventPool.offer(new QueryEvent()); + } + // Track allocations as we go so a partial-pool failure (e.g. native + // OOM at iteration K) frees the K-1 buffers already in freeBuffers. + // Without this, the half-built QwpEgressIoThread instance escapes the + // failed constructor and is unreachable from QwpQueryClient.close(), + // leaking those buffers' native scratches for the JVM's lifetime. + try { + for (int i = 0; i < bufferPoolSize; i++) { + // add() throws on capacity violation, which is what we want here: + // the queue was just allocated with capacity == bufferPoolSize so + // every insertion must succeed. A silent drop via offer() would + // leave the pool smaller than advertised. + freeBuffers.add(new QwpBatchBuffer(DEFAULT_BUFFER_CAPACITY)); + } + } catch (Throwable t) { + for (QwpBatchBuffer b : freeBuffers) { + b.close(); + } + freeBuffers.clear(); + // sendScratch and decoder are field initializers that run BEFORE + // this constructor body. NativeBufferWriter allocates 8 KiB native + // immediately on construction; QwpResultBatchDecoder allocates + // lazily on first decode but still owns native scratch over its + // lifetime. Free both here so a partial-pool failure (native OOM + // mid-loop) doesn't leak them along with the half-built instance. + Misc.free(sendScratch); + Misc.free(decoder); + throw t; + } + } + + /** + * Decodes a QUERY_ERROR payload into a {@link QueryEvent}. Visible for testing. + * Bound-checks msgLen against the actual payload: a hostile or buggy server + * can encode msgLen=0xFFFF with a tiny payload, which would otherwise read + * up to 65 KiB of native memory beyond the frame and surface it to the user + * callback as a String. + */ + public static QueryEvent decodeError(long payload, int payloadLen) { + long payloadEnd = payload + payloadLen; + long p = payload + QwpConstants.HEADER_SIZE + 1 + 8; + if (p + 1 + 2 > payloadEnd) { + return new QueryEvent().asError(WebSocketResponse.STATUS_INTERNAL_ERROR, "QUERY_ERROR frame truncated before msg_len"); + } + byte status = Unsafe.getUnsafe().getByte(p); + p += 1; + int msgLen = Unsafe.getUnsafe().getShort(p) & 0xFFFF; + p += 2; + if (p + msgLen > payloadEnd) { + return new QueryEvent().asError(WebSocketResponse.STATUS_INTERNAL_ERROR, + "QUERY_ERROR msg_len " + msgLen + " exceeds frame remainder " + (payloadEnd - p)); + } + byte[] bytes = new byte[msgLen]; + for (int i = 0; i < msgLen; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(p + i); + } + return new QueryEvent().asError(status, new String(bytes, StandardCharsets.UTF_8)); + } + + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + if (payloadLen < QwpConstants.HEADER_SIZE + 1) { + emitTerminalTransportError("server sent truncated frame"); + // Stop the receive loop; the framing is broken and any further bytes + // would be misinterpreted relative to the expected message boundary. + currentQueryDone = true; + return; + } + byte msgKind = Unsafe.getUnsafe().getByte(payloadPtr + QwpConstants.HEADER_SIZE); + if (msgKind == QwpEgressMsgKind.RESULT_BATCH) { + handleResultBatch(payloadPtr, payloadLen); + } else if (msgKind == QwpEgressMsgKind.RESULT_END) { + decodeAndEmitResultEnd(payloadPtr, payloadLen); + currentQueryDone = true; + } else if (msgKind == QwpEgressMsgKind.EXEC_DONE) { + decodeAndEmitExecDone(payloadPtr, payloadLen); + currentQueryDone = true; + } else if (msgKind == QwpEgressMsgKind.QUERY_ERROR) { + decodeAndEmitError(payloadPtr, payloadLen); + currentQueryDone = true; + } else if (msgKind == QwpEgressMsgKind.CACHE_RESET) { + // Server reached a configured soft cap on the connection-scoped + // SYMBOL dict or schema-fingerprint cache. Drop the indicated + // caches on this side so the next RESULT_BATCH's deltaStart and + // schema-reference ids line up with the server's fresh counter. + // No user-visible event -- CACHE_RESET never arrives between the + // RESULT_BATCH / RESULT_END / EXEC_DONE / QUERY_ERROR of a query + // and the user callback, only after it. + handleCacheReset(payloadPtr, payloadLen); + } else { + emitTerminalTransportError("unknown msg_kind 0x" + Integer.toHexString(msgKind & 0xFF)); + currentQueryDone = true; + } + } + + @Override + public void onClose(int code, String reason) { + String msg = "server closed connection: code=" + code + " reason=" + reason; + notifyTerminalFailure(msg); + if (!currentQueryDone) { + events.offer(new QueryEvent().asTransportError(WebSocketResponse.STATUS_INTERNAL_ERROR, msg)); + currentQueryDone = true; + } + } + + /** + * Releases a buffer back to the I/O thread pool. Call after the user + * handler finishes processing a {@code KIND_BATCH} event. + *

+ * Also signals the release latch so the I/O thread, parked at the end of + * {@code handleResultBatch}, can resume and let the WebSocket recv buffer + * compact past the consumed frame. + */ + public void releaseBuffer(QwpBatchBuffer buffer) { + if (closed) { + // closePool already drained and cleared freeBuffers; offering this + // buffer back into the pool would strand it (no consumer left to + // close it). Free its native scratch in place. The pendingRelease + // queue is also abandoned -- the I/O thread that would have read + // the token is gone. + buffer.close(); + return; + } + // Invariant: at most bufferPoolSize buffers exist, so a slot is always + // free when the user releases one. If offer ever fails we'd leak the + // buffer's native scratch; close in place so an invariant violation + // surfaces as a broken buffer rather than a slow native-memory leak. + if (!freeBuffers.offer(buffer)) { + buffer.close(); + return; + } + pendingRelease.offer(RELEASE_TOKEN); + // Close race: closePool may have set closed AND drained freeBuffers + // between our closed-check above and our offer. In that window the + // buffer landed in a cleared-and-abandoned queue with no consumer. + // Re-check after offer: if closed is now true and the buffer is + // still in freeBuffers, remove it and close in place. remove() is + // atomic on ArrayBlockingQueue, so if closePool's drain beat us to + // it, remove returns false and we skip the double-close. + if (closed && freeBuffers.remove(buffer)) { + buffer.close(); + } + } + + /** + * Returns a consumed event to the pool for reuse. Called by the user thread + * after the handler has processed the event. No-op when {@code event} is + * null. When the pool is already at capacity (e.g. a cold path allocated a + * fresh fallback event), the extra event is dropped and GC reclaims it. + */ + public void releaseEvent(QueryEvent event) { + if (event == null) { + return; + } + event.reset(); + eventPool.offer(event); + } + + /** + * Queues a CANCEL frame for {@code requestId} to be sent by the I/O thread + * between the next two {@code receiveFrame} iterations (typically within + * {@link #POLL_TIMEOUT_MS}). Safe to call from any thread. If a CANCEL for + * the same (or another) requestId is already pending, the newer id wins -- + * multiple concurrent cancels coalesce into one send. + */ + public void requestCancel(long requestId) { + pendingCancelRequestId.set(requestId); + } + + @Override + public void run() { + try { + while (!shutdown) { + QueryRequest req; + try { + req = requests.poll(POLL_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException ie) { + break; + } + if (req == null) continue; + + // Per-query state; accessed only from the I/O thread. + currentQueryDone = false; + currentRequestId = req.requestId; + creditEnabled = req.initialCredit > 0L; + sendQueryRequest(req); + + while (!currentQueryDone && !shutdown) { + // onBinaryMessage (on this same thread) sets currentQueryDone. + wsClient.receiveFrame(this, (int) POLL_TIMEOUT_MS); + drainPendingCancel(); + } + } + } catch (Throwable t) { + String msg = "I/O thread failure: " + t.getMessage(); + notifyTerminalFailure(msg); + emitTransportErrorBlocking(msg); + } finally { + // Wake any user thread blocked on events.take(). Without this, a close() + // (or any abnormal exit) while a user thread is mid-execute() would let + // takeEvent() block forever -- once the I/O thread is gone, no further + // events arrive on the queue. + if (!currentQueryDone) { + String msg = shutdown + ? "I/O thread shut down with query in flight" + : "I/O thread terminated with query in flight"; + if (shutdown) { + // User-driven close. Emit a plain query error (KIND_ERROR): the + // user initiated the shutdown, we do not want failover to fire on + // their behalf. + emitError(WebSocketResponse.STATUS_INTERNAL_ERROR, msg); + } else { + notifyTerminalFailure(msg); + emitTransportErrorBlocking(msg); + } + currentQueryDone = true; + } + } + } + + /** + * Signals shutdown. Does not join the thread -- caller handles that. + */ + public void shutdown() { + shutdown = true; + } + + /** + * Blocking submission of a query. Called by the user thread. {@code initialCredit} + * is the client-advertised send-ahead budget in bytes -- 0 means "unbounded" + * (server streams without flow control, Phase-1 default); any positive value + * asks the server to suspend once it has emitted that many bytes and wait + * for a CREDIT frame. + *

+ * The {@code bindPayloadPtr} / {@code bindPayloadLen} range holds the + * pre-encoded per-bind wire bytes (type code + null flag + value, one + * sequence per bind) produced on the user thread by + * {@link QwpBindValues}. The I/O thread reads this range synchronously + * during {@link #sendQueryRequest} and does not retain a reference after + * the send completes. {@code bindCount} is the number of binds the + * payload contains; zero when the user supplied no binds. + */ + public void submitQuery( + String sql, + long requestId, + long initialCredit, + int bindCount, + long bindPayloadPtr, + long bindPayloadLen + ) throws InterruptedException { + requests.put(new QueryRequest(sql, requestId, initialCredit, bindCount, bindPayloadPtr, bindPayloadLen)); + } + + /** + * Blocking pop of the next event. Called by the user thread during {@code execute()}. + */ + public QueryEvent takeEvent() throws InterruptedException { + return events.take(); + } + + /** + * Takes a pre-allocated {@link QueryEvent} from the pool for the I/O thread + * to populate before offering to {@link #events}. Falls back to a fresh + * allocation if the pool is momentarily empty -- under normal flow the pool + * is sized above the events-queue capacity so the fallback never fires, but + * defensive allocation keeps the I/O thread making progress if accounting + * drifts on a cold path. + */ + private QueryEvent borrowEvent() { + QueryEvent ev = eventPool.poll(); + if (ev != null) { + return ev; + } + return new QueryEvent(); + } + + private void decodeAndEmitError(long payload, int payloadLen) { + QueryEvent ev = decodeError(payload, payloadLen); + events.offer(ev); + } + + /** + * EXEC_DONE body: msg_kind(1) + requestId(8) + op_type(1) + rows_affected(varint). + * Parses all fields, surfaces as a {@link QueryEvent#KIND_EXEC_DONE} event. + */ + private void decodeAndEmitExecDone(long payload, int payloadLen) { + long p = payload + QwpConstants.HEADER_SIZE + 1 + 8; + long limit = payload + payloadLen; + if (p + 1 > limit) { + emitTerminalTransportError("EXEC_DONE frame truncated before op_type"); + return; + } + byte opType = Unsafe.getUnsafe().getByte(p++); + long rowsAffected = 0; + int shift = 0; + boolean terminated = false; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + rowsAffected |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + terminated = true; + break; + } + shift += 7; + if (shift > 63) { + emitTerminalTransportError("EXEC_DONE rows_affected varint overflow"); + return; + } + } + // The loop also exits when {@code p == limit} on a frame that ended + // mid-varint without a terminator byte. Without this guard, a buggy or + // hostile server could truncate the frame and the partial value would + // surface to the user handler as if the EXEC_DONE completed normally. + if (!terminated) { + emitTerminalTransportError("EXEC_DONE frame truncated mid rows_affected varint"); + return; + } + events.offer(new QueryEvent().asExecDone(opType, rowsAffected)); + } + + /** + * RESULT_END body: msg_kind(1) + requestId(8) + final_seq(varint) + total_rows(varint). + * We only need total_rows. Both varint loops cap at 10 bytes so a hostile + * server cannot drive {@code shift} past 63 (where Java's {@code <<} masks + * the count to 6 bits and silently wraps high bytes back into the low bits, + * producing a wildly wrong total). A frame that ends mid-varint surfaces + * as a terminal transport error rather than a partial-value RESULT_END. + */ + private void decodeAndEmitResultEnd(long payload, int payloadLen) { + long p = payload + QwpConstants.HEADER_SIZE + 1 + 8; + long limit = payload + payloadLen; + int seqBytes = 0; + boolean seqTerminated = false; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + if ((b & 0x80) == 0) { + seqTerminated = true; + break; + } + if (++seqBytes > 9) { + // Continuation bit set on byte 10 of the final_seq varint -- + // malformed. Surface as terminal so the caller cannot proceed + // with a desynced cursor. + emitTerminalTransportError("RESULT_END final_seq varint overflow"); + return; + } + } + if (!seqTerminated) { + emitTerminalTransportError("RESULT_END frame truncated mid final_seq varint"); + return; + } + long total = 0; + int shift = 0; + boolean totalTerminated = false; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + total |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + totalTerminated = true; + break; + } + shift += 7; + if (shift > 63) { + // Same overflow guard decodeAndEmitExecDone uses; without it, + // byte 10 of total_rows could land in the sign bit and bytes + // 11+ would silently wrap. + emitTerminalTransportError("RESULT_END total_rows varint overflow"); + return; + } + } + if (!totalTerminated) { + emitTerminalTransportError("RESULT_END frame truncated mid total_rows varint"); + return; + } + events.offer(new QueryEvent().asEnd(total)); + } + + /** + * Flushes the pending cancel (if any) to the wire. Called from the I/O + * thread at every loop boundary so a cancel set by a user thread reaches + * the server regardless of whether the I/O thread was waiting on a frame + * or on a free buffer. + */ + private void drainPendingCancel() { + long id = pendingCancelRequestId.getAndSet(-1L); + if (id >= 0L) { + sendCancel(id); + } + } + + private void emitError(byte status, String message) { + events.offer(new QueryEvent().asError(status, message)); + } + + /** + * Emits a {@code KIND_TRANSPORT_ERROR} event AND latches the client's + * terminal-failure state. Use for transport- or protocol-level faults + * where the WebSocket is no longer usable (server close, send/recv + * exception, decoder out of sync). Per-query {@code QUERY_ERROR} still + * goes through {@link #emitError}, since the connection remains healthy + * after one. The event-kind split means {@code execute()} classifies + * the failure by the event kind, not by peeking at the terminal-failure + * latch -- the latch stays strictly for short-circuiting subsequent + * {@code execute()} calls on a broken client. + */ + private void emitTerminalTransportError(String message) { + notifyTerminalFailure(message); + events.offer(new QueryEvent().asTransportError(WebSocketResponse.STATUS_INTERNAL_ERROR, message)); + } + + /** + * Like {@link #emitError} but emits a {@code KIND_TRANSPORT_ERROR} event + * rather than {@code KIND_ERROR}, and retries until the event is enqueued. + * Used on transport-failure paths where the WebSocket is torn down and + * the user thread needs to be woken so failover can kick in. + */ + private void emitTransportErrorBlocking(String message) { + QueryEvent ev = new QueryEvent().asTransportError(WebSocketResponse.STATUS_INTERNAL_ERROR, message); + while (!events.offer(ev)) { + // Events queue is bounded; the consumer side (user thread) drains + // steadily under a cooperative shutdown. Yield between retries so + // we don't busy-spin while the consumer is scheduled. + Thread.yield(); + if (Thread.currentThread().isInterrupted()) { + // Restore the interrupt and abandon the error publish. Caller + // paths that rely on the error reaching the user may see a + // silent drop -- acceptable under explicit interrupt. + return; + } + } + } + + /** + * Decodes a {@code CACHE_RESET} frame body and clears the indicated + * connection-scoped caches on the client side. Body is a single byte mask: + * bit 0 = SYMBOL dict, bit 1 = schema-fingerprint cache. + */ + private void handleCacheReset(long payloadPtr, int payloadLen) { + int bodyStart = QwpConstants.HEADER_SIZE + 1; // msg_kind byte consumed by caller + if (payloadLen < bodyStart + 1) { + emitTerminalTransportError("CACHE_RESET frame truncated before reset_mask"); + currentQueryDone = true; + return; + } + byte resetMask = Unsafe.getUnsafe().getByte(payloadPtr + bodyStart); + decoder.applyCacheReset(resetMask); + } + + private void handleResultBatch(long payloadPtr, int payloadLen) { + QwpBatchBuffer buf; + try { + // Poll rather than take so a pending cancel set by the user thread + // still gets flushed while the I/O thread is waiting for the user + // to release a buffer. Without this, an app that never releases + // (or sleeps in its handler) would wedge the cancel path -- the + // server never sees the CANCEL and keeps on streaming. + buf = freeBuffers.poll(POLL_TIMEOUT_MS, TimeUnit.MILLISECONDS); + while (buf == null && !shutdown) { + drainPendingCancel(); + buf = freeBuffers.poll(POLL_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } + if (buf == null) return; + } catch (InterruptedException ie) { + return; + } + // Decode in place: column layouts reference payloadPtr (the WebSocket recv + // buffer) directly, skipping the previous per-batch memcpy into buf.scratchAddr. + try { + decoder.decode(buf, payloadPtr, payloadLen); + } catch (QwpDecodeException e) { + // Same invariant as releaseBuffer: a slot is always free for a buf + // we took out of the pool moments ago. Close-on-failure is a + // defensive guard against future refactors breaking that invariant. + if (!freeBuffers.offer(buf)) { + buf.close(); + } + // A decode failure leaves the client-side decoder out of step with + // the server's byte stream: the next frame cannot be trusted. + emitTerminalTransportError("decode failure: " + e.getMessage()); + currentQueryDone = true; + return; + } + events.offer(borrowEvent().asBatch(buf)); + // Park on the release latch. Returning sooner would let receiveFrame + // compact the WebSocket recv buffer, overwriting the bytes that the + // user-visible column pointers still reference. User thread's + // releaseBuffer offers the token that unblocks this take. + // + // Wait uninterruptibly: if close() interrupts us here we MUST NOT + // unwind back to tryParseFrame, because tryParseFrame's tail + // advances recvReadPos and runs compactRecvBuffer (Vect.memmove) + // over the bytes the user's column pointers still alias inside + // recv-buf. A premature unwind on interrupt is the close()-during- + // onBatch UAF: the user thread's onBatch reads garbage mid-call. + // Stay parked; close()'s 5-second join timeout will abandon the + // I/O thread (the documented timeout-leak path), but only AFTER + // the user's onBatch has finished -- no UAF. + boolean interrupted = false; + while (true) { + try { + pendingRelease.take(); + break; + } catch (InterruptedException ie) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + // Credit replenish: the user is done with the batch, so the recv-buffer + // bytes are free. Tell the server it can stream {@code payloadLen} more + // bytes. No-op when the query wasn't started under flow control. + if (creditEnabled) { + sendCredit(currentRequestId, payloadLen); + } + } + + private void notifyTerminalFailure(String message) { + if (terminalFailureListener != null) { + try { + terminalFailureListener.onTerminalFailure(WebSocketResponse.STATUS_INTERNAL_ERROR, message); + } catch (Throwable ignored) { + // Listener must not bring down the I/O thread. A first-failure-wins + // CAS in the listener cannot throw in practice; defensive anyway. + } + } + } + + /** + * Builds and transmits a CANCEL frame on the WebSocket. Wire format: + * {@code msg_kind(0x14) + request_id(u64)}. + */ + private void sendCancel(long requestId) { + sendScratch.reset(); + sendScratch.putByte(QwpEgressMsgKind.CANCEL); + sendScratch.putLong(requestId); + wsClient.sendBinary(sendScratch.getBufferPtr(), sendScratch.getPosition()); + sendScratch.reset(); + } + + /** + * Builds and transmits a CREDIT frame on the WebSocket. Wire format: + * {@code msg_kind(0x15) + request_id(u64) + additional_bytes(varint)}. + */ + private void sendCredit(long requestId, long additionalBytes) { + sendScratch.reset(); + sendScratch.putByte(QwpEgressMsgKind.CREDIT); + sendScratch.putLong(requestId); + sendScratch.putVarint(additionalBytes); + wsClient.sendBinary(sendScratch.getBufferPtr(), sendScratch.getPosition()); + sendScratch.reset(); + } + + /** + * Builds and transmits a QUERY_REQUEST frame on the WebSocket. + * The bind payload, if any, is copied straight from the user thread's + * {@link QwpBindValues} scratch into the send buffer; no per-bind encoding + * happens on this thread. + */ + private void sendQueryRequest(QueryRequest req) { + byte[] sqlBytes = req.sql.getBytes(StandardCharsets.UTF_8); + sendScratch.reset(); + sendScratch.putByte(QwpEgressMsgKind.QUERY_REQUEST); + sendScratch.putLong(req.requestId); + sendScratch.putVarint(sqlBytes.length); + for (byte b : sqlBytes) { + sendScratch.putByte(b); + } + sendScratch.putVarint(req.initialCredit); // 0 = unbounded (Phase-1 default) + sendScratch.putVarint(req.bindCount); + if (req.bindCount > 0 && req.bindPayloadLen > 0) { + sendScratch.putBlockOfBytes(req.bindPayloadPtr, req.bindPayloadLen); + } + wsClient.sendBinary(sendScratch.getBufferPtr(), sendScratch.getPosition()); + sendScratch.reset(); + } + + /** + * Frees native scratch owned by the pool. Call after the thread has terminated. + *

+ * Drains any unconsumed batch events still in the events queue and closes + * their buffers. Without this, a close() that races with an in-flight query + * would leak the {@link QwpBatchBuffer} native scratches that were enqueued + * but never consumed. + *

+ * Pushes a final sentinel error onto the events queue so any user thread + * blocked on {@link #takeEvent} (or that returns from a handler after the + * pool has been drained) wakes up with a clear error rather than blocking + * forever on an empty queue. + */ + void closePool() { + // Set closed BEFORE draining so a user thread that returns from + // onBatch concurrently and calls releaseBuffer sees closed=true and + // frees the buffer in place rather than offering it into a queue + // we're about to clear. Volatile write pairs with the volatile read + // in releaseBuffer. + closed = true; + Misc.free(sendScratch); + Misc.free(decoder); + QueryEvent ev; + while ((ev = events.poll()) != null) { + if (ev.kind == QueryEvent.KIND_BATCH && ev.buffer != null) { + ev.buffer.close(); + } + } + for (QwpBatchBuffer b : freeBuffers) { + b.close(); + } + freeBuffers.clear(); + // The events queue capacity is bufferPoolSize + 2 with no consumer competing + // for slots after the I/O thread has joined, so offer is guaranteed to succeed. + events.offer(new QueryEvent().asError(WebSocketResponse.STATUS_INTERNAL_ERROR, "QwpQueryClient closed")); + } + + @FunctionalInterface + public interface TerminalFailureListener { + void onTerminalFailure(byte status, String message); + } + + private static final class QueryRequest { + final int bindCount; + final long bindPayloadLen; + final long bindPayloadPtr; + final long initialCredit; + final long requestId; + final String sql; + + QueryRequest(String sql, long requestId, long initialCredit, int bindCount, long bindPayloadPtr, long bindPayloadLen) { + this.sql = sql; + this.requestId = requestId; + this.initialCredit = initialCredit; + this.bindCount = bindCount; + this.bindPayloadPtr = bindPayloadPtr; + this.bindPayloadLen = bindPayloadLen; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java new file mode 100644 index 00000000..21be618a --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java @@ -0,0 +1,91 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * QWP egress message-kind discriminator bytes. Mirrors the server-side constants + * in {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind}. First byte of every + * egress payload identifies which of the egress message types it carries. + */ +public final class QwpEgressMsgKind { + /** + * Server -> client. Connection-scoped cache reset. Body: {@code reset_mask:u8} + * with bit 0 = SYMBOL dict, bit 1 = schema-fingerprint cache. Sent between + * queries when a cache hits its server-side soft cap. Recipient clears the + * indicated caches; subsequent RESULT_BATCH delta sections start fresh. + */ + public static final byte CACHE_RESET = 0x17; + public static final byte CANCEL = 0x14; + public static final byte CREDIT = 0x15; + /** + * Server -> client. Ack for a successful non-SELECT query. Body: + * {@code request_id:u64, op_type:u8, rows_affected:varint}. + */ + public static final byte EXEC_DONE = 0x16; + public static final byte QUERY_ERROR = 0x13; + public static final byte QUERY_REQUEST = 0x10; + /** + * Reset mask bit: clear the connection-scoped SYMBOL dict. + */ + public static final byte RESET_MASK_DICT = 0x01; + /** + * Reset mask bit: clear the connection-scoped schema-fingerprint cache. + */ + public static final byte RESET_MASK_SCHEMAS = 0x02; + public static final byte RESULT_BATCH = 0x11; + public static final byte RESULT_END = 0x12; + /** + * Role value on {@code SERVER_INFO.role}: the authoritative write node. + */ + public static final byte ROLE_PRIMARY = 1; + /** + * Role value on {@code SERVER_INFO.role}: promotion-in-progress. Clients + * insisting on primary-only reads may still route here; clients needing + * write-visible reads should wait for {@link #ROLE_PRIMARY}. + */ + public static final byte ROLE_PRIMARY_CATCHUP = 3; + /** + * Role value on {@code SERVER_INFO.role}: the node is a read-only replica + * that pulls WAL segments from the shared object store. Reads may lag the + * primary by the replication poll interval plus transport time. + */ + public static final byte ROLE_REPLICA = 2; + /** + * Role value on {@code SERVER_INFO.role}: no replication is configured. The + * standalone OSS default; behaves like a primary for routing purposes. + */ + public static final byte ROLE_STANDALONE = 0; + /** + * Server -> client. Unsolicited frame delivered as the first QWP message + * on every v2 WebSocket connection. Body (little-endian): {@code + * msg_kind:u8, role:u8, epoch:u64, capabilities:u32, server_wall_ns:i64, + * cluster_id:u16_len+utf8, node_id:u16_len+utf8}. The byte-value 0x17 is + * claimed by {@link #CACHE_RESET}; SERVER_INFO lives at 0x18. + */ + public static final byte SERVER_INFO = 0x18; + + private QwpEgressMsgKind() { + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java new file mode 100644 index 00000000..f6202799 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -0,0 +1,1813 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.ClientTlsConfiguration; +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.impl.ConfStringParser; +import io.questdb.client.std.Chars; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Zstd; +import io.questdb.client.std.str.StringSink; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +/** + * QWP egress (query results) client. + *

+ * Connection shape: one WebSocket to {@code /read/v1}, one dedicated I/O thread + * that owns the socket and the decoder. The user thread submits a query and + * drains result batches via the supplied {@link QwpColumnBatchHandler}; the + * I/O thread reads and decodes ahead so that decoding of batch {@code N+1} + * overlaps with the user's processing of batch {@code N}. + *

+ * Thread safety: not thread-safe for concurrent queries on the same client. + * One {@link #execute} at a time. Opening one client per query-issuing thread + * is the recommended pattern. + *

+ * Multi-endpoint routing: the connection string accepts a comma-separated list + * of {@code addr=host:port[,host:port...]} endpoints plus a {@code target=} + * filter ({@code any} | {@code primary} | {@code replica}). {@link #connect()} + * walks the list in order, reads the server's role from the v2 + * {@code SERVER_INFO} frame, and picks the first endpoint matching the target. + * When every endpoint reports a role the filter rejects, {@link #connect()} + * throws {@link QwpRoleMismatchException} with the last observed info attached + * so callers can distinguish "no primary available" from "all endpoints down". + *

+ * Failover on transport failure: with {@code failover=on} (the default) + * {@link #execute} transparently reconnects to another endpoint when the I/O + * thread reports a transport- or protocol-level terminal failure mid-stream, + * and re-submits the query. The user handler observes a + * {@link QwpColumnBatchHandler#onFailoverReset} callback just before replayed + * batches start arriving (batch_seq restarts at 0 on the new node), so + * accumulating handlers can discard rows the old connection already delivered. + * {@code failover=off} restores the pre-v2 behaviour -- terminal failures + * surface immediately through {@link QwpColumnBatchHandler#onError}. + *

+ * Terminal-failure latching: transport- or protocol-level faults detected by + * the I/O thread (server close, truncated/unknown frames, send/recv exceptions) + * latch a sticky terminal failure on the client. With failover on, the next + * {@link #execute} transparently reconnects; with failover off, subsequent + * {@link #execute} calls short-circuit via + * {@link QwpColumnBatchHandler#onError} with the stored status/message. Per- + * query {@code QUERY_ERROR} responses are NOT terminal -- the connection + * remains usable for the next query. + *

+ * Status byte convention on {@link QwpColumnBatchHandler#onError}: server- + * emitted {@code QUERY_ERROR} frames surface with the server's status code + * ({@link WebSocketResponse#STATUS_PARSE_ERROR}, {@code STATUS_INTERNAL_ERROR}, + * {@link QwpConstants#STATUS_CANCELLED}, {@link QwpConstants#STATUS_LIMIT_EXCEEDED}, + * etc.). Failures detected client-side (closed client, bind encoding error, + * truncated / unknown frame, decoder out of sync, I/O thread interrupt) all + * surface with {@link WebSocketResponse#STATUS_INTERNAL_ERROR} and the + * specific cause in the message. + */ +public class QwpQueryClient implements QuietCloseable { + + public static final String DEFAULT_ENDPOINT_PATH = "/read/v1"; + public static final int DEFAULT_WS_PORT = 9000; + /** + * Hard ceiling on {@link #withMaxBatchRows}. Matches the client decoder's + * own {@code MAX_ROWS_PER_BATCH} safety cap so a user cannot ask for a + * per-batch row count the decoder would itself refuse. The server enforces + * its own (typically smaller) cap independently; this is just the client's + * sanity bound. + */ + public static final int MAX_BATCH_ROWS_UPPER_BOUND = 1_048_576; + public static final int QWP_MAX_VERSION = QwpConstants.MAX_SUPPORTED_VERSION; + public static final String TARGET_ANY = "any"; + public static final String TARGET_PRIMARY = "primary"; + public static final String TARGET_REPLICA = "replica"; + /** + * Default initial backoff between failover attempts, in milliseconds. + * Doubled on each subsequent retry up to + * {@link #DEFAULT_FAILOVER_MAX_BACKOFF_MS}. Keeps a tight reconnect + * storm from overwhelming an already-struggling cluster; small enough + * that a one-off transient hiccup barely adds latency to the replay. + */ + private static final long DEFAULT_FAILOVER_INITIAL_BACKOFF_MS = 50L; + /** + * Default cap on the number of {@code executeOnce} invocations per call + * to {@link #execute}. Counts the initial attempt plus every failover + * retry. Each attempt walks the endpoint list independently, so this + * bound is normally well above the expected cluster size. Override + * per-client via {@link #withFailoverMaxAttempts(int)} or the + * {@code failover_max_attempts=} connection-string key. + */ + private static final int DEFAULT_FAILOVER_MAX_ATTEMPTS = 8; + /** + * Default ceiling on the exponential failover backoff. A minutes-long + * outage that burns through {@code failoverMaxAttempts} still completes + * in bounded wall time: with 8 attempts and a 1 s cap the client gives + * up after roughly 5 s of cumulative sleep. + */ + private static final long DEFAULT_FAILOVER_MAX_BACKOFF_MS = 1_000L; + private static final int DEFAULT_IO_BUFFER_POOL_SIZE = 4; + /** + * How long {@link #connect()} waits to read the v2 {@code SERVER_INFO} frame + * from each endpoint before giving up and moving to the next. 5 seconds is + * comfortable on a WAN; the server writes SERVER_INFO into the same send + * buffer as the 101 upgrade response so under normal conditions the frame + * is already in the client's kernel recv buffer by the time this wait starts. + */ + private static final int DEFAULT_SERVER_INFO_TIMEOUT_MS = 5_000; + private static final Logger LOG = LoggerFactory.getLogger(QwpQueryClient.class); + // Reusable typed bind-value sink. Populated on the user thread by the + // {@link QwpBindSetter} passed to execute(); the pre-encoded bytes are + // handed to the I/O thread via QueryRequest. Allocated once per client to + // keep the per-query path zero-alloc after warmup. Freed in close() on the + // clean-join path; leaked together with the buffer pool on the timeout path. + private final QwpBindValues bindValues = new QwpBindValues(); + // Gates {@link #close()} so concurrent or repeat invocations are a no-op + // after the first. Without this, a second close() would run through the + // full shutdown path again and double-free {@link #bindValues} native + // scratch. + private final AtomicBoolean closedFlag = new AtomicBoolean(); + private final List endpoints = new ArrayList<>(); + private String authorizationHeader; + private int bufferPoolSize = DEFAULT_IO_BUFFER_POOL_SIZE; + private String clientId; + private int compressionLevel = 3; + // User-facing compression preference from the connection string. "raw" is + // the library default -- no compression, no handshake header, no server- + // side CPU burn on payloads where the network isn't the bottleneck + // (colocated clients, loopback). Clients wanting zstd must opt in via + // {@code compression=zstd} (demands zstd) or {@code compression=auto} + // (advertises zstd,raw and lets the server pick). + private String compressionPreference = "raw"; + // Published by connect() / reconnectSkippingIndex() and read by cancel(), + // close(), and the pre-connect-guard on the configuration setters. Volatile + // both for the standard happens-before relationship against pre-connect + // configuration writes and so a second thread calling cancel() or close() + // observes the freshly-bound state without explicit synchronisation. + private volatile boolean connected; + // Index of the endpoint the current connection is bound to. -1 when + // disconnected. Used by the failover path to skip the current endpoint + // when picking the next one to try. Volatile because execute() on the + // user thread reads + rewrites across a cleanup-reconnect handoff and + // cancel() from another thread may also read it for diagnostics. + private volatile int currentEndpointIndex = -1; + // One-shot terminal-failure listener held by the currently-bound I/O + // thread. Every new QwpEgressIoThread gets a fresh instance wired to it + // at construction time; {@link #cleanupFailedConnect} orphans the + // outgoing instance so a late callback from an I/O thread that failed + // to join cannot pollute the new connection's terminalFailure latch. + private volatile GenerationListener currentGenerationListener; + // Written on the user thread at entry to {@link #execute} and cleared on exit. + // Read by {@link #cancel} from any thread. {@code volatile} to guarantee the + // user thread's write is visible to a concurrent cancel caller; 64-bit writes + // are atomic under {@code volatile long}. + private volatile long currentRequestId = -1L; + private String endpointPath = DEFAULT_ENDPOINT_PATH; + // True by default: on transport failure during execute(), reconnect to + // another endpoint and replay the query. Callers that prefer to see the + // error themselves opt out via {@code failover=off} in the connection + // string or {@link #withFailover(boolean)}. Volatile as a paired write + // with the connected flag so post-connect mutation is consistently + // rejected across threads. + private volatile boolean failoverEnabled = true; + private long failoverInitialBackoffMs = DEFAULT_FAILOVER_INITIAL_BACKOFF_MS; + private int failoverMaxAttempts = DEFAULT_FAILOVER_MAX_ATTEMPTS; + private long failoverMaxBackoffMs = DEFAULT_FAILOVER_MAX_BACKOFF_MS; + // Credit-flow send-ahead budget. 0 = unbounded (Phase-1 default, no CREDIT + // bookkeeping on either side). A positive value puts the stream under byte- + // based flow control: the server emits at most this many bytes of result + // payload before it parks, and the client auto-replenishes by the size of + // each batch as the user releases it. + private long initialCreditBytes; + // Volatile so a cancel() call from a thread other than the one that ran + // connect() sees the published reference (and a concurrent null-out from + // close() is observed without a stale-reference race). The thread-safety + // contract documented on cancel() relies on this. + // ioThreadHandle is volatile and written BEFORE the ioThread volatile + // write in connectToEndpoint. Together this gives close() (called from a + // third thread) the guarantee that a non-null read of ioThread implies a + // visible non-null ioThreadHandle. Without volatile here, a third-thread + // close could observe ioThread != null && ioThreadHandle == null, skip + // the join, and then closePool() / Misc.free under a still-running daemon + // -- crashing the JVM on the next decode cycle. + private volatile QwpEgressIoThread ioThread; + private volatile Thread ioThreadHandle; + private boolean lastCloseTimedOut; + // Client preference for server-side per-batch row cap. 0 means "unset", + // server uses its default. Set via {@code max_batch_rows=N} in the + // connection string or {@link #withMaxBatchRows}. Smaller values give + // streaming consumers earlier access to the first rows at the cost of + // more per-batch overhead; larger values amortise fixed costs over more + // rows. Server may clamp down to its own hard cap. + private int maxBatchRows; + private int negotiatedQwpVersion; + private long nextRequestId = 1; + // Decoded SERVER_INFO from the current connection's handshake. Null before + // connect() has succeeded, and on connections that negotiated v1 (which + // doesn't emit the frame). Volatile so the I/O thread's read on the + // {@code onFailoverReset} path sees the latest reconnect. + private volatile QwpServerInfo serverInfo; + private int serverInfoTimeoutMs = DEFAULT_SERVER_INFO_TIMEOUT_MS; + // Maximum time close() will wait for the I/O thread to exit before giving up + // and leaking the (daemon) thread + its native buffer pool + WebSocket socket. + // 5 seconds is generous given the I/O thread polls on a 100 ms cadence; if + // it overshoots this, something is seriously wrong (e.g., user handler stuck + // in onBatch). Volatile (not final) so tests can reflectively shorten it to + // hit the timeout branch in under a second instead of spending five. + @SuppressWarnings("FieldMayBeFinal") + private volatile long shutdownJoinMs = 5_000; + // Routing filter: one of {@link #TARGET_ANY}, {@link #TARGET_PRIMARY}, + // {@link #TARGET_REPLICA}. Applied in {@link #connect()} against the role + // byte from SERVER_INFO; endpoints that don't match are skipped. + // {@link #TARGET_PRIMARY} accepts STANDALONE as well so OSS deployments + // (which don't have a replication role) are not accidentally excluded. + // Volatile so the post-connect mutation guard observes the live value + // across threads. + private volatile String target = TARGET_ANY; + private boolean tlsEnabled; + // Only meaningful when tlsEnabled. Default is full validation against the JVM's trust store. + private int tlsValidationMode = ClientTlsConfiguration.TLS_VALIDATION_MODE_FULL; + private char[] trustStorePassword; + private String trustStorePath; + private WebSocketClient webSocketClient; + + private QwpQueryClient(String host, int port) { + this.endpoints.add(new Endpoint(host, port)); + } + + /** + * Builds a query client from a connection-string of the same shape used by + * {@link io.questdb.client.Sender#fromConfig(CharSequence)}: {@code ::key=value;key=value;...}. + *

+ * Supported schemas: + *

    + *
  • {@code ws::} -- plain WebSocket.
    • + *
  • {@code wss::} -- WebSocket over TLS. See {@code tls_verify} / {@code tls_roots} / + * {@code tls_roots_password} for trust configuration.
    • + *
+ * Supported keys: + *
    + *
  • {@code addr=host[:port][,host[:port]...]} -- required. Comma-separated list of + * WebSocket endpoints; {@link #connect()} walks them in order and stops at the + * first matching {@code target=}. Default port on each entry is {@value #DEFAULT_WS_PORT}.
    • + *
  • {@code target=any|primary|replica} -- endpoint filter applied against the role + * byte from the v2 {@code SERVER_INFO} frame. Default {@code any}. {@code primary} + * accepts {@code PRIMARY}, {@code PRIMARY_CATCHUP} and {@code STANDALONE}.
    • + *
  • {@code failover=on|off} -- default {@code on}. On transport failure during + * {@link #execute}, reconnect to another endpoint and re-submit the query. + * The user handler sees {@link QwpColumnBatchHandler#onFailoverReset} before + * replayed batches begin arriving (batch_seq restarts at 0 on the new node).
    • + *
  • {@code path=/read/v1} -- egress endpoint. Default {@value #DEFAULT_ENDPOINT_PATH}.
    • + *
  • {@code auth=} -- sent verbatim as the HTTP {@code Authorization} header during the upgrade handshake. + * Mutually exclusive with {@code username}/{@code password} and {@code token}.
    • + *
  • {@code username=;password=} -- HTTP Basic authentication. Server verifies the credentials + * against the same user store the Postgres wire protocol uses, so a user created via + * {@code CREATE USER ... WITH PASSWORD ...} can log in unchanged. + * Both keys must be present together; mutually exclusive with {@code auth} and {@code token}.
    • + *
  • {@code token=} -- HTTP Bearer authentication with an OIDC access token (sent as + * {@code Authorization: Bearer }). Mutually exclusive with {@code auth} and + * {@code username}/{@code password}.
    • + *
  • {@code client_id=} -- sent as the {@code X-QWP-Client-Id} header.
    • + *
  • {@code buffer_pool_size=N} -- depth of the I/O thread's batch buffer pool. Default 4.
    • + *
  • {@code compression=zstd|raw|auto} -- compression codec the client + * asks the server to use for RESULT_BATCH bodies. {@code auto} + * (default) advertises {@code zstd,raw} so the server picks zstd + * when it supports it and falls back to raw otherwise.
    • + *
  • {@code compression_level=N} -- zstd level hint, clamped server-side + * to [1, 9]. Default 3. Ignored when {@code compression=raw}.
    • + *
  • {@code tls_verify=on|unsafe_off} -- TLS certificate validation. Default is {@code on}. + * Only allowed with the {@code wss::} schema. {@code unsafe_off} disables hostname and + * certificate chain validation; use only for testing.
    • + *
  • {@code tls_roots=} -- path to a custom trust store (PKCS12 or JKS). Must be + * paired with {@code tls_roots_password}. Only allowed with {@code wss::}.
    • + *
  • {@code tls_roots_password=} -- password for the custom trust store.
    • + *
+ * Examples: + * 
+     *   ws::addr=localhost:9000;
+     *   ws::addr=db.internal:9000;path=/read/v1;auth=Bearer abc123;client_id=dashboard/2.0;
+     *   ws::addr=db-a:9000,db-b:9000,db-c:9000;target=primary;failover=on;
+     *
+ */ + public static QwpQueryClient fromConfig(CharSequence configurationString) { + if (configurationString == null || configurationString.length() == 0) { + throw new IllegalArgumentException("configuration string cannot be empty"); + } + StringSink sink = new StringSink(); + int pos = ConfStringParser.of(configurationString, sink); + if (pos < 0) { + throw new IllegalArgumentException("invalid configuration string: " + sink); + } + boolean tls; + if (Chars.equals("ws", sink)) { + tls = false; + } else if (Chars.equals("wss", sink)) { + tls = true; + } else { + throw new IllegalArgumentException( + "unsupported schema [schema=" + sink + ", supported-schemas=[ws, wss]]"); + } + + List parsedEndpoints = null; + String path = DEFAULT_ENDPOINT_PATH; + String target = TARGET_ANY; + Boolean failover = null; + Integer failoverMaxAttempts = null; + Long failoverBackoffInitialMs = null; + Long failoverBackoffMaxMs = null; + String auth = null; + String username = null; + String password = null; + String token = null; + String cid = null; + int poolSize = DEFAULT_IO_BUFFER_POOL_SIZE; + // Default matches the field initializer in QwpQueryClient: raw wire, + // zstd opt-in. + String compression = "raw"; + int compressionLevel = 3; + int maxBatchRows = 0; // 0 = omit header, server uses its default + // TLS validation mode: null means "unset in config". Explicit values kick in only when tls is true. + Integer tlsValidation = null; + String tlsRoots = null; + String tlsRootsPassword = null; + + while (ConfStringParser.hasNext(configurationString, pos)) { + pos = ConfStringParser.nextKey(configurationString, pos, sink); + if (pos < 0) { + throw new IllegalArgumentException("invalid configuration string [error=" + sink + "]"); + } + String key = sink.toString(); + pos = ConfStringParser.value(configurationString, pos, sink); + if (pos < 0) { + throw new IllegalArgumentException("invalid configuration string [error=" + sink + "]"); + } + String value = sink.toString(); + switch (key) { + case "addr": + parsedEndpoints = parseEndpointList(value); + break; + case "target": + if (!TARGET_ANY.equals(value) && !TARGET_PRIMARY.equals(value) && !TARGET_REPLICA.equals(value)) { + throw new IllegalArgumentException( + "invalid target: " + value + " (expected any, primary, or replica)"); + } + target = value; + break; + case "failover": + if ("on".equals(value)) { + failover = Boolean.TRUE; + } else if ("off".equals(value)) { + failover = Boolean.FALSE; + } else { + throw new IllegalArgumentException("invalid failover: " + value + " (expected on or off)"); + } + break; + case "failover_max_attempts": + try { + failoverMaxAttempts = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid failover_max_attempts: " + value); + } + if (failoverMaxAttempts < 1) { + throw new IllegalArgumentException("failover_max_attempts must be >= 1"); + } + break; + case "failover_backoff_initial_ms": + try { + failoverBackoffInitialMs = Long.parseLong(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid failover_backoff_initial_ms: " + value); + } + if (failoverBackoffInitialMs < 0L) { + throw new IllegalArgumentException("failover_backoff_initial_ms must be >= 0"); + } + break; + case "failover_backoff_max_ms": + try { + failoverBackoffMaxMs = Long.parseLong(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid failover_backoff_max_ms: " + value); + } + if (failoverBackoffMaxMs < 0L) { + throw new IllegalArgumentException("failover_backoff_max_ms must be >= 0"); + } + break; + case "path": + path = value; + break; + case "auth": + auth = value; + break; + case "username": + username = value; + break; + case "password": + password = value; + break; + case "token": + token = value; + break; + case "client_id": + cid = value; + break; + case "buffer_pool_size": + try { + poolSize = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid buffer_pool_size: " + value); + } + if (poolSize < 1) { + throw new IllegalArgumentException("buffer_pool_size must be >= 1"); + } + break; + case "compression": + if (!"zstd".equals(value) && !"raw".equals(value) && !"auto".equals(value)) { + throw new IllegalArgumentException( + "unsupported compression: " + value + " (expected zstd, raw, or auto)"); + } + compression = value; + break; + case "compression_level": + try { + compressionLevel = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid compression_level: " + value); + } + if (compressionLevel < 1 || compressionLevel > 22) { + throw new IllegalArgumentException("compression_level must be in [1, 22]"); + } + break; + case "max_batch_rows": + try { + maxBatchRows = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid max_batch_rows: " + value); + } + if (maxBatchRows < 1 || maxBatchRows > MAX_BATCH_ROWS_UPPER_BOUND) { + throw new IllegalArgumentException( + "max_batch_rows must be in [1, " + MAX_BATCH_ROWS_UPPER_BOUND + "]"); + } + break; + case "tls_verify": + if ("on".equals(value)) { + tlsValidation = ClientTlsConfiguration.TLS_VALIDATION_MODE_FULL; + } else if ("unsafe_off".equals(value)) { + tlsValidation = ClientTlsConfiguration.TLS_VALIDATION_MODE_NONE; + } else { + throw new IllegalArgumentException( + "invalid tls_verify: " + value + " (expected on or unsafe_off)"); + } + break; + case "tls_roots": + tlsRoots = value; + break; + case "tls_roots_password": + tlsRootsPassword = value; + break; + default: + throw new IllegalArgumentException("unknown configuration key: " + key); + } + } + if (parsedEndpoints == null || parsedEndpoints.isEmpty()) { + throw new IllegalArgumentException("missing required key: addr"); + } + boolean hasBasic = username != null || password != null; + if (hasBasic && (username == null || password == null)) { + throw new IllegalArgumentException("both username and password must be provided together"); + } + int authModesSet = (auth != null ? 1 : 0) + (hasBasic ? 1 : 0) + (token != null ? 1 : 0); + if (authModesSet > 1) { + throw new IllegalArgumentException( + "auth, username/password, and token are mutually exclusive"); + } + if (!tls && (tlsValidation != null || tlsRoots != null || tlsRootsPassword != null)) { + throw new IllegalArgumentException( + "tls_verify/tls_roots/tls_roots_password require the wss:: schema"); + } + if ((tlsRoots == null) != (tlsRootsPassword == null)) { + throw new IllegalArgumentException( + "tls_roots and tls_roots_password must be provided together"); + } + if (failoverBackoffInitialMs != null + && failoverBackoffMaxMs != null + && failoverBackoffMaxMs < failoverBackoffInitialMs) { + throw new IllegalArgumentException( + "failover_backoff_max_ms must be >= failover_backoff_initial_ms"); + } + Endpoint first = parsedEndpoints.get(0); + QwpQueryClient client = new QwpQueryClient(first.host, first.port); + for (int i = 1; i < parsedEndpoints.size(); i++) { + client.endpoints.add(parsedEndpoints.get(i)); + } + client.withTarget(target); + if (failover != null) { + client.withFailover(failover); + } + if (failoverMaxAttempts != null) { + client.withFailoverMaxAttempts(failoverMaxAttempts); + } + if (failoverBackoffInitialMs != null || failoverBackoffMaxMs != null) { + long initial = failoverBackoffInitialMs != null + ? failoverBackoffInitialMs + : DEFAULT_FAILOVER_INITIAL_BACKOFF_MS; + long max = failoverBackoffMaxMs != null + ? failoverBackoffMaxMs + : Math.max(initial, DEFAULT_FAILOVER_MAX_BACKOFF_MS); + client.withFailoverBackoff(initial, max); + } + client.withEndpointPath(path); + client.withBufferPoolSize(poolSize); + client.withCompression(compression, compressionLevel); + if (tls) { + if (tlsRoots != null) { + client.withTrustStore(tlsRoots, tlsRootsPassword.toCharArray()); + } else if (tlsValidation != null && tlsValidation == ClientTlsConfiguration.TLS_VALIDATION_MODE_NONE) { + client.withInsecureTls(); + } else { + client.withTls(); + } + } + if (auth != null) client.withAuthorization(auth); + if (hasBasic) client.withBasicAuth(username, password); + if (token != null) client.withBearerToken(token); + if (cid != null) client.withClientId(cid); + if (maxBatchRows > 0) client.withMaxBatchRows(maxBatchRows); + return client; + } + + /** + * Creates a plain-text (non-TLS) QWP query client against {@code host:port} + * with all other settings at their defaults. For TLS, authentication, + * custom paths, compression, buffer pool sizing, or multi-endpoint routing + * use {@link #fromConfig(CharSequence)} with a connection string instead. + * + * @see #fromConfig(CharSequence) + */ + public static QwpQueryClient newPlainText(CharSequence host, int port) { + return new QwpQueryClient(host.toString(), port); + } + + /** + * Asks the server to cancel the currently executing query. No-op if no query + * is in flight. Safe to call from a thread other than the one blocked inside + * {@link #execute}. The server replies to the active query with a + * {@code QUERY_ERROR} whose status byte is {@code STATUS_CANCELLED}; the + * handler's {@code onError} (on the execute-ing thread) will see it. + */ + public void cancel() { + QwpEgressIoThread io = ioThread; + long id = currentRequestId; + if (io != null && id >= 0L) { + io.requestCancel(id); + } + } + + /** + * Shutdown order: signal the I/O thread, interrupt it to wake it from any blocking + * {@code wsClient.receiveFrame(...)} or queue poll, wait for it to exit, then free + * the buffer pool and close the underlying socket. + *

+ * If the I/O thread fails to exit within {@link #shutdownJoinMs} (default 5 s), this + * method does not free the buffer pool or close the WebSocket -- both are + * still in use by the thread, and freeing them would race into a JVM-killing + * use-after-free. The thread is a daemon, so the JVM still exits normally; the + * resources leak for the lifetime of the process. A warning is recorded by setting + * {@link #lastCloseTimedOut} (queryable via {@link #wasLastCloseTimedOut}) so callers + * can detect and report the condition. + *

+ * Threading contract: {@code close()} must be called from a thread + * other than the one currently inside a batch handler, AND the user must finish + * any in-flight {@code execute()} before calling {@code close()}. Calling + * {@code close()} concurrently with a {@code handler.onBatch(...)} that is still + * dereferencing batch column pointers can free the WebSocket recv buffer under + * those pointers and SIGSEGV the JVM. The interrupt-driven I/O thread shutdown + * does NOT detect or wait for a still-running user handler -- the timeout-based + * leak fallback only protects against an unresponsive I/O thread, not an + * unresponsive user thread. {@link #cancel()} is the right way to ask an + * in-flight {@code execute()} to return; close after it does. + */ + @Override + public void close() { + if (!closedFlag.compareAndSet(false, true)) { + // Second (or concurrent) close call. Without this guard, two + // closes would each Misc.free(bindValues.writer) the shared native + // scratch, double-freeing it. + return; + } + connected = false; + lastCloseTimedOut = false; + try { + if (ioThread != null) { + ioThread.shutdown(); + // Wake the thread from any blocking poll / recv so it sees the shutdown flag promptly. + if (ioThreadHandle != null) { + ioThreadHandle.interrupt(); + boolean joined; + try { + ioThreadHandle.join(shutdownJoinMs); + joined = !ioThreadHandle.isAlive(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + // Don't free anything else -- preserve clean shutdown semantics on the next + // attempt. bindValues still closes via the enclosing finally because its + // bytes are copied into sendScratch at submit time and no other thread + // references it after that. + return; + } + if (!joined) { + // Daemon thread is still running -- buffer pool and WebSocketClient may + // be in use. Leak them rather than risk a SIGSEGV by freeing under it. + // Log at ERROR so operators notice; wasLastCloseTimedOut() is the + // programmatic hook for monitoring or tests that want to assert on it. + LOG.error("QwpQueryClient close timed out after {} ms; leaking I/O thread, " + + "buffer pool, and WebSocket to avoid freeing them from under " + + "a running daemon. Common cause: a batch handler that never " + + "returns (e.g. blocking I/O or deadlock).", + shutdownJoinMs); + lastCloseTimedOut = true; + ioThread = null; + ioThreadHandle = null; + webSocketClient = null; + return; + } + } + ioThread.closePool(); + ioThread = null; + ioThreadHandle = null; + } + if (webSocketClient != null) { + webSocketClient.close(); + webSocketClient = null; + } + } finally { + // bindValues owns native scratch that is not shared with the I/O thread + // (submitQuery copies its bytes into sendScratch), so it is safe to free + // even when we otherwise leak the I/O thread and buffer pool. + bindValues.close(); + } + } + + /** + * Opens a connection that satisfies the configured {@code target} filter and + * performs the WebSocket upgrade. Must be called before any query is submitted. + *

+ * Walks the endpoint list in order: for each entry it opens the TCP socket, + * runs the HTTP upgrade, reads the v2 {@code SERVER_INFO} frame, and accepts + * the endpoint if the server's role matches the configured target. An + * endpoint that matches becomes the bound connection and the I/O thread is + * spawned. An endpoint whose role doesn't match is closed and the walk + * continues to the next entry. Transport failures on a specific endpoint + * are treated the same way -- the walk continues. + *

+ * If every endpoint is tried and none matches the target, throws + * {@link QwpRoleMismatchException} carrying the last {@link QwpServerInfo} + * observed so callers can distinguish "no primary available" from "all + * endpoints unreachable" (the latter surfaces as a plain + * {@link HttpClientException}). + */ + public void connect() { + if (connected) { + return; + } + QwpServerInfo lastObservedMismatch = null; + boolean sawV1Mismatch = false; + Throwable lastTransportError = null; + for (int i = 0; i < endpoints.size(); i++) { + Endpoint ep = endpoints.get(i); + try { + connectToEndpoint(ep); + } catch (RuntimeException e) { + lastTransportError = e; + LOG.warn("QwpQueryClient connect failed for {}:{} -- {}", ep.host, ep.port, e.getMessage()); + cleanupFailedConnect(); + continue; + } + QwpServerInfo info = serverInfo; + if (!TARGET_ANY.equals(target) && info == null) { + // v1 server (no SERVER_INFO frame) cannot satisfy a specific-role + // filter. target=primary/replica asks for a role guarantee; a + // silent no-SERVER_INFO bind would give the caller false + // confidence that the connected endpoint is the role they asked + // for. + sawV1Mismatch = true; + LOG.info("QwpQueryClient {}:{} negotiated v1 (no SERVER_INFO) and target={} requires v2; trying next", + ep.host, ep.port, target); + cleanupFailedConnect(); + continue; + } + if (info != null && !matchesTarget(info.getRole(), target)) { + lastObservedMismatch = info; + LOG.info("QwpQueryClient {}:{} role={} does not match target={}, trying next", + ep.host, ep.port, QwpServerInfo.roleName(info.getRole()), target); + cleanupFailedConnect(); + continue; + } + currentEndpointIndex = i; + connected = true; + return; + } + if (lastObservedMismatch != null) { + throw new QwpRoleMismatchException( + target, + lastObservedMismatch, + "no endpoint matches target=" + target + "; last observed role=" + + QwpServerInfo.roleName(lastObservedMismatch.getRole()) + + " cluster=" + lastObservedMismatch.getClusterId() + ); + } + if (sawV1Mismatch) { + throw new QwpRoleMismatchException( + target, + null, + "no endpoint matches target=" + target + + "; at least one endpoint negotiated v1 and cannot supply a role" + ); + } + throw new HttpClientException( + "all QWP endpoints unreachable [count=" + endpoints.size() + + ", lastError=" + (lastTransportError == null ? "" : lastTransportError.getMessage()) + ']'); + } + + /** + * Executes {@code sql} and drives the supplied handler through the result stream. + *

+ * Blocks the calling thread until the server sends {@code RESULT_END} or + * {@code QUERY_ERROR}. While the user thread is inside {@code handler.onBatch}, + * the I/O thread keeps reading and decoding ahead up to the configured buffer-pool depth. + *

+ * With {@code failover=on} (the default), a transport failure mid-stream + * triggers a transparent reconnect to another endpoint and re-submission of + * the query. The user handler observes + * {@link QwpColumnBatchHandler#onFailoverReset} just before the replayed + * batches begin arriving on the new connection, and any rows delivered + * before the reset should be discarded by the handler. + */ + public void execute(String sql, QwpColumnBatchHandler handler) { + execute(sql, null, handler); + } + + /** + * Executes {@code sql} with typed bind parameters supplied by the given + * {@link QwpBindSetter}. The setter runs on the calling thread before the + * query is dispatched; bind values must be assigned in strictly ascending + * index order starting at 0, matching the SQL placeholders ({@code $1, $2, ...}). + *

+ * Passing the same SQL text on subsequent calls hits the server's + * SQL-text-keyed factory cache -- the factory compiles once and binds + * supply the per-call values. Interpolating values into the SQL string + * defeats this reuse. + */ + public void execute(String sql, QwpBindSetter binds, QwpColumnBatchHandler handler) { + if (!connected) { + throw new IllegalStateException("QwpQueryClient not connected; call connect() first"); + } + int attempt = 0; + while (true) { + attempt++; + FailoverProbeHandler probe = new FailoverProbeHandler(handler); + executeOnce(sql, binds, probe); + if (!probe.transportFailureIntercepted) { + return; + } + if (!failoverEnabled) { + // failover disabled: surface the transport failure to the user + // and leave the client in a broken state (per documented contract). + handler.onError(probe.interceptedStatus, probe.interceptedMessage); + return; + } + if (attempt >= failoverMaxAttempts) { + int failovers = Math.max(0, attempt - 1); + handler.onError(probe.interceptedStatus, + "transport failure after " + attempt + " execute attempt" + + (attempt == 1 ? "" : "s") + " (" + + failovers + " failover reconnect" + + (failovers == 1 ? "" : "s") + "); last error: " + + probe.interceptedMessage); + return; + } + // Snapshot the endpoint we were just bound to before cleanup + // clobbers currentEndpointIndex. reconnectSkippingIndex uses it + // to start the walk at the NEXT entry -- without this, a transport + // failure against a primary whose server is still accepting new + // sockets (our debug hook does exactly that, but so would a brief + // WebSocket hiccup in production) would pick the same primary + // back up on reconnect and never exercise the replica. + int failedIndex = currentEndpointIndex; + // Tear the broken connection down before reconnecting. We don't want + // to leak the current I/O thread or WebSocket. cleanupFailedConnect + // also orphans the outgoing generation's terminal-failure listener + // so a late callback from the dying I/O thread cannot pollute the + // new connection's state. + cleanupFailedConnect(); + connected = false; + // Exponential backoff between failover reconnects, doubling per + // attempt and capped at failoverMaxBackoffMs. attempt=1 is the + // original execute and never sleeps; attempt=2 sleeps the initial + // amount, attempt=3 double, etc. Sleep is interruptible: a thread + // interrupt aborts failover and surfaces as an onError so a + // blocking execute() can still be cancelled by the user. + if (failoverInitialBackoffMs > 0L) { + long base = failoverInitialBackoffMs << Math.min(attempt - 1, 30); + long delay = Math.min(base, failoverMaxBackoffMs); + if (delay > 0L) { + try { + Thread.sleep(delay); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + handler.onError(probe.interceptedStatus, + "failover interrupted while backing off after attempt " + + attempt + "; last error: " + probe.interceptedMessage); + return; + } + } + } + try { + reconnectSkippingIndex(failedIndex); + } catch (RuntimeException reconnectErr) { + handler.onError(probe.interceptedStatus, + "failover reconnect failed after " + attempt + " attempt" + + (attempt == 1 ? "" : "s") + " [last error: " + + probe.interceptedMessage + ", reconnect error: " + + reconnectErr.getMessage() + ']'); + return; + } + handler.onFailoverReset(serverInfo); + // loop back: next iteration re-executes the query on the fresh connection + } + } + + /** + * Returns the current compression preference: one of {@code raw} (the + * library default, no compression), {@code zstd} (demand zstd), or + * {@code auto} (advertise zstd and raw, let the server pick). Useful for + * introspection and for tests that pin the default. + */ + public String getCompressionPreference() { + return compressionPreference; + } + + public int getNegotiatedQwpVersion() { + return negotiatedQwpVersion; + } + + /** + * Returns the {@link QwpServerInfo} decoded from the currently-bound + * server's {@code SERVER_INFO} frame, or {@code null} if the server + * negotiated the v1 protocol (no frame sent) or the client is not + * connected. The value is refreshed on every successful failover reconnect. + */ + public QwpServerInfo getServerInfo() { + return serverInfo; + } + + public boolean isConnected() { + return connected; + } + + /** + * Returns true if the most recent {@link #close()} call abandoned the I/O thread + * because it failed to exit within the join timeout. The native buffer pool and + * WebSocket socket are leaked for the lifetime of the JVM; the daemon I/O thread + * keeps running until process exit. + */ + public boolean wasLastCloseTimedOut() { + return lastCloseTimedOut; + } + + public void withAuthorization(String authorizationHeader) { + checkPreConnect("withAuthorization"); + this.authorizationHeader = authorizationHeader; + } + + /** + * Configures HTTP Basic authentication for the WebSocket upgrade request. + * The server verifies the credentials against the same user store the + * Postgres wire protocol uses, so a user created via + * {@code CREATE USER ... WITH PASSWORD ...} can authenticate here unchanged. + * Must be called before {@link #connect}. + */ + public QwpQueryClient withBasicAuth(String username, String password) { + checkPreConnect("withBasicAuth"); + if (username == null || password == null) { + throw new IllegalArgumentException("username and password must not be null"); + } + String credentials = username + ":" + password; + this.authorizationHeader = "Basic " + Base64.getEncoder() + .encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); + return this; + } + + /** + * Configures HTTP Bearer authentication with an OIDC access token for the + * WebSocket upgrade request. The server verifies the token via the + * configured OIDC provider and resolves the principal (and any groups) + * from the token's claims. Must be called before {@link #connect}. + */ + public QwpQueryClient withBearerToken(String token) { + checkPreConnect("withBearerToken"); + if (token == null) { + throw new IllegalArgumentException("token must not be null"); + } + this.authorizationHeader = "Bearer " + token; + return this; + } + + /** + * Overrides the default I/O buffer pool depth (4). Larger pools let the + * I/O thread decode further ahead of the consumer at the cost of memory; + * smaller pools reduce memory but may stall the I/O thread on slow consumers. + * Must be called before {@link #connect()}. + */ + public void withBufferPoolSize(int size) { + checkPreConnect("withBufferPoolSize"); + if (size < 1) throw new IllegalArgumentException("bufferPoolSize must be >= 1"); + this.bufferPoolSize = size; + } + + /** + * Overrides the {@code X-QWP-Client-Id} header sent during the upgrade handshake. + * Must be called before {@link #connect()}. + */ + public void withClientId(String clientId) { + checkPreConnect("withClientId"); + this.clientId = clientId; + } + + /** + * Programmatic equivalent of the {@code compression=} / {@code compression_level=} + * connection-string keys. {@code preference} is one of {@code zstd}, + * {@code raw} (default), or {@code auto}. {@code level} is the zstd + * compression level hint passed to the server; clamped server-side to + * [1, 9]. Must be called before {@link #connect}. + */ + public void withCompression(String preference, int level) { + checkPreConnect("withCompression"); + if (!"zstd".equals(preference) && !"raw".equals(preference) && !"auto".equals(preference)) { + throw new IllegalArgumentException( + "unsupported compression: " + preference + " (expected zstd, raw, or auto)"); + } + if (level < 1 || level > 22) { + throw new IllegalArgumentException("compression level must be in [1, 22]"); + } + this.compressionPreference = preference; + this.compressionLevel = level; + } + + public void withEndpointPath(String endpointPath) { + checkPreConnect("withEndpointPath"); + this.endpointPath = endpointPath; + } + + /** + * Programmatic equivalent of the {@code failover=} connection-string key. + * Default is {@code true}: transport failures during {@link #execute} are + * transparently retried against another endpoint. + */ + public void withFailover(boolean enabled) { + checkPreConnect("withFailover"); + this.failoverEnabled = enabled; + } + + /** + * Configures the exponential backoff applied between failover reconnect + * attempts. {@code initialMs} is the delay before the first retry (the + * second overall execute attempt); each subsequent retry doubles the + * delay up to {@code maxMs}. A zero {@code initialMs} disables backoff + * entirely -- retries fire back to back, which is fine for fast LAN + * clusters but risks hammering a struggling one during a real outage. + * Defaults: initial {@value #DEFAULT_FAILOVER_INITIAL_BACKOFF_MS} ms, + * max {@value #DEFAULT_FAILOVER_MAX_BACKOFF_MS} ms. + */ + public void withFailoverBackoff(long initialMs, long maxMs) { + checkPreConnect("withFailoverBackoff"); + if (initialMs < 0) throw new IllegalArgumentException("failoverInitialBackoffMs must be >= 0"); + if (maxMs < initialMs) { + throw new IllegalArgumentException("failoverMaxBackoffMs must be >= failoverInitialBackoffMs"); + } + this.failoverInitialBackoffMs = initialMs; + this.failoverMaxBackoffMs = maxMs; + } + + /** + * Configures the maximum number of {@code executeOnce} invocations per + * {@link #execute} call. Counts the initial attempt plus every failover + * retry. Default {@value #DEFAULT_FAILOVER_MAX_ATTEMPTS}. + */ + public QwpQueryClient withFailoverMaxAttempts(int attempts) { + checkPreConnect("withFailoverMaxAttempts"); + if (attempts < 1) throw new IllegalArgumentException("failoverMaxAttempts must be >= 1"); + this.failoverMaxAttempts = attempts; + return this; + } + + /** + * Opts the next {@link #execute} into credit-based flow control with + * {@code bytes} of initial send-ahead budget. The server streams at most + * {@code bytes} of result payload before pausing; the client auto- + * replenishes by the size of each batch after the user's handler releases + * it. Passing {@code 0} (the default) disables flow control entirely + * (unbounded -- Phase-1 behaviour). + *

+ * Must be called before {@link #connect}. + */ + public QwpQueryClient withInitialCredit(long bytes) { + checkPreConnect("withInitialCredit"); + if (bytes < 0) throw new IllegalArgumentException("initial credit must be >= 0"); + this.initialCreditBytes = bytes; + return this; + } + + /** + * Enables TLS with certificate validation disabled. Intended for testing only -- + * production code should use {@link #withTls} or {@link #withTrustStore}. + * Must be called before {@link #connect}. + */ + public QwpQueryClient withInsecureTls() { + checkPreConnect("withInsecureTls"); + this.tlsEnabled = true; + this.tlsValidationMode = ClientTlsConfiguration.TLS_VALIDATION_MODE_NONE; + this.trustStorePath = null; + this.trustStorePassword = null; + return this; + } + + /** + * Asks the server to cap each {@code RESULT_BATCH} at {@code rows} rows. + * Useful for latency-sensitive streaming consumers that want to start + * processing the first row as soon as possible -- a smaller cap flushes + * the first batch sooner, at the cost of more per-batch overhead (WS + * header, send syscall, schema-reference decode). The server clamps down + * to its own hard limit; a value of {@code 0} (default) omits the header + * and the server uses its own cap. + *

+ * Must be called before {@link #connect}. + */ + public QwpQueryClient withMaxBatchRows(int rows) { + checkPreConnect("withMaxBatchRows"); + if (rows < 1 || rows > MAX_BATCH_ROWS_UPPER_BOUND) { + throw new IllegalArgumentException( + "max_batch_rows must be in [1, " + MAX_BATCH_ROWS_UPPER_BOUND + "]"); + } + this.maxBatchRows = rows; + return this; + } + + /** + * Overrides the {@link #DEFAULT_SERVER_INFO_TIMEOUT_MS} wait for the v2 + * {@code SERVER_INFO} frame. Must be called before {@link #connect}. + */ + public void withServerInfoTimeout(int ms) { + checkPreConnect("withServerInfoTimeout"); + if (ms < 1) throw new IllegalArgumentException("serverInfoTimeoutMs must be >= 1"); + this.serverInfoTimeoutMs = ms; + } + + /** + * Programmatic equivalent of the {@code target=} connection-string key. + * One of {@link #TARGET_ANY} (default), {@link #TARGET_PRIMARY}, + * {@link #TARGET_REPLICA}. + */ + public void withTarget(String target) { + checkPreConnect("withTarget"); + if (!TARGET_ANY.equals(target) && !TARGET_PRIMARY.equals(target) && !TARGET_REPLICA.equals(target)) { + throw new IllegalArgumentException( + "invalid target: " + target + " (expected any, primary, or replica)"); + } + this.target = target; + } + + /** + * Enables TLS with full certificate validation against the JVM's default trust store. + * Must be called before {@link #connect}. + */ + public QwpQueryClient withTls() { + checkPreConnect("withTls"); + this.tlsEnabled = true; + this.tlsValidationMode = ClientTlsConfiguration.TLS_VALIDATION_MODE_FULL; + this.trustStorePath = null; + this.trustStorePassword = null; + return this; + } + + /** + * Enables TLS with full certificate validation against the given custom trust store. + * Must be called before {@link #connect}. + * + * @param trustStorePath filesystem path to a PKCS12 or JKS trust store + * @param trustStorePassword password for the trust store + */ + public QwpQueryClient withTrustStore(String trustStorePath, char[] trustStorePassword) { + checkPreConnect("withTrustStore"); + if (trustStorePath == null || trustStorePassword == null) { + throw new IllegalArgumentException("trustStorePath and trustStorePassword must not be null"); + } + this.tlsEnabled = true; + this.tlsValidationMode = ClientTlsConfiguration.TLS_VALIDATION_MODE_FULL; + this.trustStorePath = trustStorePath; + this.trustStorePassword = trustStorePassword; + return this; + } + + private static String defaultClientId() { + return "questdb-java-egress/1.0.0"; + } + + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + private static boolean matchesTarget(byte role, String target) { + if (TARGET_ANY.equals(target)) { + return true; + } + if (TARGET_PRIMARY.equals(target)) { + return role == QwpEgressMsgKind.ROLE_PRIMARY + || role == QwpEgressMsgKind.ROLE_PRIMARY_CATCHUP + || role == QwpEgressMsgKind.ROLE_STANDALONE; + } + if (TARGET_REPLICA.equals(target)) { + return role == QwpEgressMsgKind.ROLE_REPLICA; + } + return true; + } + + /** + * Parses an {@code addr=} value that may be a single {@code host[:port]} + * or a comma-separated list of such entries. A single entry without a + * port falls back to {@link #DEFAULT_WS_PORT}. The port (when present) + * must be in {@code [1, 65535]}. + *

+ * IPv6 addresses must be wrapped in brackets when carrying a port, per + * RFC 3986: {@code [::1]:9000}, {@code [fe80::1]}. An unbracketed entry + * containing more than one colon is treated as a bare IPv6 host with + * the default port (no syntactic way to distinguish {@code host:port} + * from a bare IPv6 address otherwise; users wanting a custom port on + * IPv6 must bracket). + */ + private static List parseEndpointList(String value) { + List list = new ArrayList<>(); + int start = 0; + int len = value.length(); + for (int i = 0; i <= len; i++) { + if (i == len || value.charAt(i) == ',') { + if (i == start) { + throw new IllegalArgumentException("empty addr entry"); + } + String entry = value.substring(start, i).trim(); + if (entry.isEmpty()) { + throw new IllegalArgumentException("empty addr entry"); + } + String host; + int port; + if (entry.charAt(0) == '[') { + // Bracketed IPv6: [host] or [host]:port. + int closeBracket = entry.indexOf(']'); + if (closeBracket < 0) { + throw new IllegalArgumentException( + "missing closing ']' in IPv6 addr entry: " + entry); + } + host = entry.substring(1, closeBracket); + if (closeBracket == entry.length() - 1) { + port = DEFAULT_WS_PORT; + } else if (entry.charAt(closeBracket + 1) != ':') { + throw new IllegalArgumentException( + "expected ':' after ']' in IPv6 addr entry: " + entry); + } else { + port = parsePort(entry.substring(closeBracket + 2), entry); + } + } else if (entry.indexOf(':') != entry.lastIndexOf(':')) { + // Multi-colon, unbracketed: treat as bare IPv6 host with + // the default port. Custom port on IPv6 requires brackets. + host = entry; + port = DEFAULT_WS_PORT; + } else { + int colon = entry.indexOf(':'); + if (colon < 0) { + host = entry; + port = DEFAULT_WS_PORT; + } else { + host = entry.substring(0, colon); + port = parsePort(entry.substring(colon + 1), entry); + } + } + if (host.isEmpty()) { + throw new IllegalArgumentException("empty host in addr entry: " + entry); + } + list.add(new Endpoint(host, port)); + start = i + 1; + } + } + return list; + } + + /** + * Parses {@code portStr} into a TCP port in the inclusive range + * {@code [1, 65535]}. Surrounding whitespace is tolerated so config + * strings hand-edited around the {@code :} don't surface as opaque + * "invalid port" errors. {@code entry} is the full + * {@code host[:port]} fragment, used only for the error message. + */ + private static int parsePort(String portStr, String entry) { + int port; + try { + port = Integer.parseInt(portStr.trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("invalid port in addr: " + entry); + } + if (port < 1 || port > 65535) { + throw new IllegalArgumentException( + "port out of range in addr: " + entry + " (must be 1-65535)"); + } + return port; + } + + /** + * Builds the {@code X-QWP-Accept-Encoding} header value from the user's + * preference. {@code raw} (the library default) omits the header entirely + * so servers that don't know about compression see an unchanged handshake. + * {@code zstd} asks for zstd first and falls back to raw. {@code auto} + * advertises both and lets the server pick -- useful for cross-DC clients + * where the bandwidth/CPU trade-off is worthwhile; an explicit opt-in. + */ + private String buildAcceptEncodingHeader() { + if ("raw".equals(compressionPreference)) { + return null; + } + return "zstd;level=" + compressionLevel + ",raw"; + } + + /** + * Guard for configuration setters that only apply at connect time. + * Calling them after {@link #connect()} has bound to an endpoint is a + * programming error: the new value would silently not apply until the + * next reconnect, which is almost never what the caller wants. Throw + * early with a clear message instead. + */ + private void checkPreConnect(String setterName) { + if (connected) { + throw new IllegalStateException(setterName + + " must be called before connect(); the current value has already taken effect on the bound connection"); + } + } + + /** + * Tears down the half-built connection state left behind by a failed + * endpoint attempt (transport error, role mismatch, SERVER_INFO decode + * error). Idempotent; safe to call when no connection state has been + * created yet. + */ + private void cleanupFailedConnect() { + // Orphan the outgoing generation's listener FIRST. Any callback from + // the I/O thread after this point (including from a thread that never + // exits the join below) becomes a no-op. Without this step, a late + // terminalFailure notification could land in the new generation's + // latch and cause spurious failover on the next execute(). + if (currentGenerationListener != null) { + currentGenerationListener.orphan(); + currentGenerationListener = null; + } + boolean ioThreadJoined = true; + if (ioThread != null) { + ioThread.shutdown(); + if (ioThreadHandle != null) { + ioThreadHandle.interrupt(); + try { + ioThreadHandle.join(shutdownJoinMs); + ioThreadJoined = !ioThreadHandle.isAlive(); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + ioThreadJoined = false; + } + } + if (ioThreadJoined) { + ioThread.closePool(); + } else { + // Daemon thread still alive -- buffer pool and WebSocket may be + // in use. Leak them rather than risk a SIGSEGV by freeing them + // from under a still-running consumer. lastCloseTimedOut is + // surfaced via the existing {@link #wasLastCloseTimedOut()} + // accessor so monitoring and tests can observe the condition. + LOG.error("QwpQueryClient failover cleanup timed out after {} ms; leaking " + + "I/O thread, buffer pool, and WebSocket to avoid freeing them " + + "from under a running daemon. Common cause: a batch handler " + + "that never returns.", + shutdownJoinMs); + lastCloseTimedOut = true; + } + ioThread = null; + ioThreadHandle = null; + } + // Only close the socket if the I/O thread has actually exited; otherwise + // the daemon may still be reading from it. + if (ioThreadJoined && webSocketClient != null) { + webSocketClient.close(); + } + webSocketClient = null; + serverInfo = null; + currentEndpointIndex = -1; + } + + private void connectToEndpoint(Endpoint ep) { + if (tlsEnabled) { + webSocketClient = WebSocketClientFactory.newTlsInstance( + new ClientTlsConfiguration(trustStorePath, trustStorePassword, tlsValidationMode)); + } else { + webSocketClient = WebSocketClientFactory.newPlainTextInstance(); + } + webSocketClient.setQwpMaxVersion(QWP_MAX_VERSION); + webSocketClient.setQwpClientId(clientId != null ? clientId : defaultClientId()); + webSocketClient.setQwpAcceptEncoding(buildAcceptEncodingHeader()); + webSocketClient.setQwpMaxBatchRows(maxBatchRows); + webSocketClient.connect(ep.host, ep.port); + webSocketClient.upgrade(endpointPath, authorizationHeader); + negotiatedQwpVersion = webSocketClient.getServerQwpVersion(); + + // v2 servers send SERVER_INFO as the first WebSocket frame after the + // upgrade response. Consume it synchronously on the user thread before + // spawning the I/O thread, so the role filter can run without any + // cross-thread synchronisation and so a mismatched role doesn't waste + // the I/O thread setup + teardown. + if (negotiatedQwpVersion >= QwpConstants.VERSION_2) { + serverInfo = receiveServerInfoSync(); + } else { + serverInfo = null; + } + + // Early probe: if we told the server we can accept zstd, make sure the + // bundled native library actually provides the decompression symbols + // before we start accepting batches. Without this, a client jar built + // without the zstd submodule would only discover the missing symbols + // mid-stream when it hits the first FLAG_ZSTD frame, and the error + // would surface as an opaque "I/O thread failure: ..." callback on the + // user handler. Fail loud here instead so the cause is obvious. + if (!"raw".equals(compressionPreference)) { + probeZstdAvailable(); + } + + // Wire a fresh generation-scoped listener into this I/O thread. Each + // listener owns its own terminal-failure latch, so even if a dying + // I/O thread slips a late onTerminalFailure callback past the orphan + // flag, the write lands on a ref nobody reads rather than poisoning + // the next connection's state. + GenerationListener listener = new GenerationListener(); + currentGenerationListener = listener; + // Publish ioThreadHandle BEFORE ioThread. Both fields are volatile; + // a third-thread close() that reads ioThread != null is guaranteed + // (via the volatile happens-before edge on the write below) to also + // see the non-null ioThreadHandle. Without this ordering, close() + // could observe (ioThread != null, ioThreadHandle == null), skip the + // join, and call closePool() / Misc.free with the daemon still alive. + QwpEgressIoThread newIoThread = new QwpEgressIoThread(webSocketClient, bufferPoolSize, listener); + Thread handle = new Thread(newIoThread, "qwp-egress-io"); + handle.setDaemon(true); + handle.start(); + ioThreadHandle = handle; + ioThread = newIoThread; + } + + /** + * Inner loop for a single query attempt. Driven by {@link #execute}; wraps + * the user's handler in a {@link FailoverProbeHandler} so that the outer + * loop can intercept transport failures before they reach the user. + */ + private void executeOnce(String sql, QwpBindSetter binds, FailoverProbeHandler probe) { + // Cache the I/O thread reference at entry: close() may null the field while + // we are inside this loop, so reading the field per-iteration would NPE + // exactly when the user is mid-execute() and close() races. The queue and + // pool the cached reference owns are still drained safely by closePool() + // before close() returns. + QwpEgressIoThread io = ioThread; + if (io == null) { + probe.onError(WebSocketResponse.STATUS_INTERNAL_ERROR, "QwpQueryClient is closed"); + return; + } + GenerationListener listener = currentGenerationListener; + TerminalFailure tf = listener != null ? listener.get() : null; + if (tf != null) { + // I/O thread already reported a transport- or protocol-level failure + // on a previous call. Tag it as a transport failure so the failover + // wrapper can take over instead of surfacing to the user as a final + // error. Going through markTransportFailure (not probe.onError) + // keeps the classification explicit -- probe.onError always means + // server-emitted QUERY_ERROR. + probe.markTransportFailure(tf.status, tf.message); + return; + } + bindValues.reset(); + if (binds != null) { + try { + binds.apply(bindValues); + } catch (RuntimeException e) { + // Surface user-side bind errors through the handler contract rather + // than propagating out of execute. Keeps error handling consistent + // with the rest of the API and avoids leaving the client in a half- + // prepared state for the next call. Bind encoding failures are NOT + // transport failures -- they're deterministic on the user thread -- + // so they must not trigger failover. + bindValues.reset(); + probe.deliverFinal( + "bind encoding failed: " + e.getMessage()); + return; + } + } + long requestId = nextRequestId++; + currentRequestId = requestId; + try { + io.submitQuery(sql, requestId, initialCreditBytes, bindValues.count(), bindValues.bufferPtr(), bindValues.bufferLen()); + while (true) { + QueryEvent ev = io.takeEvent(); + try { + switch (ev.kind) { + case QueryEvent.KIND_BATCH: + try { + probe.onBatch(ev.buffer.batch); + } finally { + io.releaseBuffer(ev.buffer); + } + break; + case QueryEvent.KIND_END: + probe.onEnd(ev.totalRows); + return; + case QueryEvent.KIND_EXEC_DONE: + probe.onExecDone(ev.opType, ev.rowsAffected); + return; + case QueryEvent.KIND_ERROR: + // Server-emitted QUERY_ERROR. Connection remains healthy; + // pass straight through to the user. Never triggers failover. + probe.onError(ev.errorStatus, ev.errorMessage); + return; + case QueryEvent.KIND_TRANSPORT_ERROR: + // Transport / protocol-level failure synthesised by the + // I/O thread. Tag as a transport failure so the outer + // execute loop can decide whether to replay (failover=on) + // or surface as a final onError (failover=off). + probe.markTransportFailure(ev.errorStatus, ev.errorMessage); + return; + default: + probe.onError(WebSocketResponse.STATUS_INTERNAL_ERROR, "unknown event kind " + ev.kind); + return; + } + } finally { + // Return the event to the I/O thread's pool so steady-state + // publishes stay allocation-free. Safe even if the handler + // threw: releaseEvent clears the event's fields and offers + // it to the pool, with overflow silently dropped. + io.releaseEvent(ev); + } + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + // Interrupt on the user thread is not a transport failure; surface directly. + probe.deliverFinal("interrupted while waiting for server response"); + } finally { + currentRequestId = -1L; + } + } + + /** + * Allocates and immediately frees a {@code ZSTD_DCtx} so that any + * {@link UnsatisfiedLinkError} from a client build that doesn't include + * the bundled libzstd surfaces synchronously on the user thread at + * {@code connect()} time. Closes the just-opened WebSocket on failure so + * the caller doesn't inherit a half-open socket. + */ + private void probeZstdAvailable() { + long dctx; + try { + dctx = Zstd.createDCtx(); + } catch (UnsatisfiedLinkError e) { + LOG.error("zstd JNI symbols missing from libquestdb; aborting connect", e); + if (webSocketClient != null) { + webSocketClient.close(); + webSocketClient = null; + } + throw new HttpClientException("this client build does not support zstd compression -- " + + "libquestdb was built without the zstd submodule. Rebuild the native library " + + "with 'git submodule update --init --recursive' and 'cmake --build', or set " + + "compression=raw on the connection string to skip the probe. " + + "[cause=" + e.getMessage() + "]"); + } + if (dctx == 0) { + LOG.error("zstd createDCtx returned 0 (native allocation failure); aborting connect"); + if (webSocketClient != null) { + webSocketClient.close(); + webSocketClient = null; + } + throw new HttpClientException("zstd decompression context allocation failed; " + + "cannot accept compressed batches. Set compression=raw on the connection " + + "string to disable compression, or retry once memory pressure subsides."); + } + Zstd.freeDCtx(dctx); + } + + private QwpServerInfo receiveServerInfoSync() { + ServerInfoReceiver receiver = new ServerInfoReceiver(); + long deadlineMs = System.currentTimeMillis() + serverInfoTimeoutMs; + while (receiver.info == null + && receiver.decodeError == null + && receiver.closeCode < 0 + && receiver.unexpectedFrame == null) { + int remaining = (int) (deadlineMs - System.currentTimeMillis()); + if (remaining <= 0) { + throw new HttpClientException("timeout waiting for SERVER_INFO frame"); + } + webSocketClient.receiveFrame(receiver, remaining); + } + if (receiver.closeCode >= 0) { + throw new HttpClientException("server closed connection before SERVER_INFO [code=" + + receiver.closeCode + ", reason=" + receiver.closeReason + ']'); + } + if (receiver.decodeError != null) { + throw new HttpClientException("SERVER_INFO decode failed: " + receiver.decodeError); + } + if (receiver.unexpectedFrame != null) { + throw new HttpClientException("unexpected frame before SERVER_INFO: " + receiver.unexpectedFrame); + } + return receiver.info; + } + + /** + * Walks the endpoint list starting at the entry right after the one that + * just failed, wrapping around so every other endpoint gets a + * try. The failed endpoint itself is deliberately not + * retried: a transport failure is likely to repeat immediately on the same + * socket ({@code PeerDisconnectedException} from a server that's still + * accepting new connections but torn the old one down, for example), and + * a retry would just burn an attempt. The outer {@link #execute} loop + * can revisit the failed endpoint on a subsequent failover attempt if + * every other endpoint is also unreachable. + *

+ * On success, leaves the client in the same state {@link #connect()} + * produces: {@code connected=true}, {@code ioThread} spawned, + * {@code serverInfo} populated. On exhaustion, raises the same exceptions + * as {@link #connect()}. + */ + private void reconnectSkippingIndex(int failedIndex) { + int total = endpoints.size(); + // When failedIndex is known, walk the other (total - 1) entries only. + // When it is not (defensive path: currentEndpointIndex was never set + // before the failure), fall back to the full (total) walk. + int startFrom = failedIndex < 0 ? 0 : failedIndex + 1; + int stepCount = failedIndex < 0 ? total : total - 1; + QwpServerInfo lastMismatch = null; + boolean sawV1Mismatch = false; + Throwable lastError = null; + for (int step = 0; step < stepCount; step++) { + int i = (startFrom + step) % total; + Endpoint ep = endpoints.get(i); + try { + connectToEndpoint(ep); + } catch (RuntimeException e) { + lastError = e; + cleanupFailedConnect(); + continue; + } + QwpServerInfo info = serverInfo; + if (!TARGET_ANY.equals(target) && info == null) { + // v1 cannot satisfy a specific-role filter (see the matching + // branch in connect()). + sawV1Mismatch = true; + cleanupFailedConnect(); + continue; + } + if (info != null && !matchesTarget(info.getRole(), target)) { + lastMismatch = info; + cleanupFailedConnect(); + continue; + } + currentEndpointIndex = i; + connected = true; + return; + } + if (lastMismatch != null) { + throw new QwpRoleMismatchException(target, lastMismatch, + "no endpoint matches target=" + target + " on failover; last observed role=" + + QwpServerInfo.roleName(lastMismatch.getRole())); + } + if (sawV1Mismatch) { + throw new QwpRoleMismatchException(target, null, + "no endpoint matches target=" + target + + " on failover; at least one endpoint negotiated v1 and cannot supply a role"); + } + throw new HttpClientException( + "all QWP endpoints unreachable on failover [count=" + total + + ", lastError=" + (lastError == null ? "" : lastError.getMessage()) + ']'); + } + + /** + * Test-only / diagnostics hook: injects a synthetic terminal failure + * through the current generation's listener. Production code does not + * call this -- transport failures arrive through the I/O thread's own + * callback path. Package-private so it does not leak into the public + * client API; tests in a different package reach it via reflection to + * simulate the I/O thread reporting a failure without actually tearing + * the WebSocket down. + */ + @SuppressWarnings("unused") + void recordTerminalFailure(byte status, String message) { + GenerationListener listener = currentGenerationListener; + if (listener != null) { + listener.onTerminalFailure(status, message); + } + } + + private static final class Endpoint { + final String host; + final int port; + + Endpoint(String host, int port) { + this.host = host; + this.port = port; + } + } + + /** + * Wraps the user's handler so {@link #execute} can classify a transport + * failure (surfaced as {@link QueryEvent#KIND_TRANSPORT_ERROR}) and route + * it into the failover loop instead of to the user. Server-emitted + * {@code QUERY_ERROR} frames ({@link QueryEvent#KIND_ERROR}) go straight + * through {@code onError} to the user. The two paths are kept strictly + * separate at the event-dispatch layer so the classification is driven by + * the I/O thread's decision at emit time, not reconstructed from a side- + * channel latch. + *

+ * Per-query state is fresh for every call of {@link #execute}; the caller + * instantiates one probe per attempt. + */ + private static final class FailoverProbeHandler implements QwpColumnBatchHandler { + final QwpColumnBatchHandler delegate; + String interceptedMessage; + byte interceptedStatus; + boolean transportFailureIntercepted; + + FailoverProbeHandler(QwpColumnBatchHandler delegate) { + this.delegate = delegate; + } + + @Override + public void onBatch(QwpColumnBatch batch) { + delegate.onBatch(batch); + } + + @Override + public void onEnd(long totalRows) { + delegate.onEnd(totalRows); + } + + @Override + public void onError(byte status, String message) { + // Server-emitted QUERY_ERROR. Pass straight through. Transport + // failures are delivered via markTransportFailure, not here. + delegate.onError(status, message); + } + + @Override + public void onExecDone(short opType, long rowsAffected) { + delegate.onExecDone(opType, rowsAffected); + } + + /** + * Bypass the interception logic and deliver the error straight to the + * user. Used for failures that are not transport-related (bind-encode + * errors, interrupts) so they don't trigger a spurious failover. + */ + void deliverFinal(String message) { + delegate.onError(WebSocketResponse.STATUS_INTERNAL_ERROR, message); + } + + /** + * Records a transport-level failure so the outer {@code execute()} + * loop can decide whether to replay or surface to the user. Does + * NOT call any user callback -- the user sees either a clean replay + * (failover=on + reconnect succeeds) or a single final {@code onError} + * (failover=off or reconnect exhausted). + */ + void markTransportFailure(byte status, String message) { + transportFailureIntercepted = true; + interceptedStatus = status; + interceptedMessage = message; + } + } + + /** + * One-shot terminal-failure listener scoped to a single {@link + * QwpEgressIoThread} instance. Each instance owns its own + * {@link AtomicReference} latch so an orphaned-but-in-flight callback + * writes into its own generation's slot and cannot poison a successor + * generation. The orphan flag is advisory: a late callback from a + * dying I/O thread observes it and short-circuits; if it squeaks past + * the check and into the CAS, the write lands on a ref that nobody + * reads, so no poisoning occurs either way. + *

+ * {@link #cleanupFailedConnect} calls {@link #orphan()} on the outgoing + * listener before creating the next generation. + */ + private static final class GenerationListener implements QwpEgressIoThread.TerminalFailureListener { + private final AtomicReference target = new AtomicReference<>(); + private volatile boolean orphaned; + + @Override + public void onTerminalFailure(byte status, String message) { + if (orphaned) { + return; + } + target.compareAndSet(null, new TerminalFailure(status, message)); + } + + TerminalFailure get() { + return target.get(); + } + + void orphan() { + this.orphaned = true; + } + } + + /** + * Buffers the inbound {@code SERVER_INFO} frame on the user thread during + * {@link QwpQueryClient#connect()}. Exactly one of {@link #info}, + * {@link #decodeError}, {@link #closeCode} gets set per connect attempt; + * the outer loop polls the fields to decide whether to accept the endpoint, + * try the next one, or raise. + */ + private static final class ServerInfoReceiver implements WebSocketFrameHandler { + int closeCode = -1; + String closeReason; + String decodeError; + QwpServerInfo info; + String unexpectedFrame; + + @Override + public void onBinaryMessage(long payloadPtr, int payloadLen) { + if (info != null) { + // Server sent a second binary frame before the client consumed + // the first. Nothing legitimate does this; flag it so the + // caller gets a clear diagnostic instead of a silent ignore. + if (unexpectedFrame == null) { + unexpectedFrame = "second binary frame before consumer advanced"; + } + return; + } + if (decodeError != null) { + return; + } + try { + info = QwpServerInfoDecoder.decode(payloadPtr, payloadLen); + } catch (QwpDecodeException e) { + decodeError = e.getMessage(); + } + } + + @Override + public void onClose(int code, String reason) { + closeCode = code; + closeReason = reason; + } + + @Override + public void onTextMessage(long payloadPtr, int payloadLen) { + // A text frame before the SERVER_INFO binary frame is non-standard + // for this protocol; flag it so the connect() error is specific + // instead of a bland timeout. + if (unexpectedFrame == null) { + unexpectedFrame = "text frame"; + } + } + } + + private static final class TerminalFailure { + final String message; + final byte status; + + TerminalFailure(byte status, String message) { + this.status = status; + this.message = message; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java new file mode 100644 index 00000000..3fadb6e0 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java @@ -0,0 +1,990 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpGorillaDecoder; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Zstd; + +import java.nio.charset.StandardCharsets; + +/** + * Zero-alloc (after warmup) decoder for inbound QWP egress {@code RESULT_BATCH} frames. + *

+ * The decoder parses the payload in-place -- no values are copied out of the + * WebSocket receive buffer. It maintains pooled {@link QwpColumnLayout} slots, + * per-column {@code int[]} index arrays, and a native SYMBOL dictionary + * (UTF-8 heap + packed offset/length entries) that is reused across batches. + * After the connection has seen its peak schema width and row count, decoding + * a batch allocates nothing on the JVM heap. + *

+ * The produced {@link QwpColumnBatch} is valid only during the surrounding + * {@code onBatch} callback because its pointers refer into the caller's native + * payload buffer. + */ +public class QwpResultBatchDecoder implements QuietCloseable { + + private static final int CONN_DICT_INITIAL_BYTES = 4096; + private static final int CONN_DICT_INITIAL_ENTRIES = 512; + /** + * Hard cap on per-row ARRAY element count. 8 bytes per element x this ~ 256 MB max payload, + * which fits in {@code int} once {@code rowEnd - p} is computed. A malicious or buggy + * server cannot push a negative or wrap-around length past this guard. + */ + private static final long MAX_ARRAY_ELEMENTS = (Integer.MAX_VALUE - 1024) / 8L; + /** + * Hard cap on the connection-scoped SYMBOL dict's UTF-8 heap size in bytes. + * Well below {@link Integer#MAX_VALUE} to prevent {@code connDictHeapPos + len} + * from wrapping negative (which would bypass {@link #ensureConnDictHeapCapacity} + * and let {@code copyMemory} write past the allocated heap). Servers that + * approach this cap are expected to emit {@code CACHE_RESET}; crossing it + * means either a broken server or a hostile frame, so we fail hard. + */ + private static final int MAX_CONN_DICT_HEAP_BYTES = 256 * 1024 * 1024; + /** + * Hard cap on the connection-scoped SYMBOL dict entry count. Matches the + * rationale for {@link #MAX_CONN_DICT_HEAP_BYTES}: a hostile server that + * never emits {@code CACHE_RESET} cannot drive {@code connDictSize} to + * {@link Integer#MAX_VALUE} without being rejected first. + */ + private static final int MAX_CONN_DICT_SIZE = 8_388_608; + /** + * Hard cap on {@code row_count} per batch. Matches the server's MAX_ROWS_PER_BATCH. + * A hostile server could otherwise encode row_count = Integer.MAX_VALUE; ensureIntArray + * would then try to allocate an {@code int[Integer.MAX_VALUE]} (~8 GB) before any + * wire-length bounds check fires. Cap two orders of magnitude above the batch size + * to leave head-room for future server-side batch enlargement without breaking clients. + */ + private static final int MAX_ROWS_PER_BATCH = 1_048_576; + /** + * Hard cap on registered schema ids per connection. Matches + * {@code QwpConstants.DEFAULT_MAX_SCHEMAS_PER_CONNECTION} on the server side. + * Capping protects the client from a hostile or buggy server that could + * otherwise force unbounded {@code schemaRegistry} growth (or AIOOBE on a + * negative schema id) by encoding {@code schemaId = Integer.MAX_VALUE} (or + * a negative varint that long-to-int casts negative). + */ + private static final int MAX_SCHEMAS_PER_CONNECTION = 65_535; + // Reusable scratch for decoding column names during SCHEMA_MODE_FULL parsing. + // Sized to the wire-level cap so {@link #readColumnName} can copy bytes + // without allocating a fresh {@code byte[]} per column. The returned + // {@code String} is still a fresh allocation (immutable + retained + // long-term on {@link QwpEgressColumnInfo#name}), but dropping the + // throwaway byte[] is measurable on schema-heavy workloads and post + // CACHE_RESET re-registration. + private final byte[] colNameScratch = new byte[QwpConstants.MAX_COLUMN_NAME_LENGTH]; + // Reused for Gorilla-encoded TIMESTAMP / TIMESTAMP_NANOS / DATE columns. + // Stateful per-column -- {@link #parseTimestampColumn} calls {@code reset} + // before decoding each column so residue from a previous column never bleeds + // in. + private final QwpGorillaDecoder gorillaDecoder = new QwpGorillaDecoder(); + // Connection-scoped state (safe to share across buffers -- reused across batches + // of the same query and across queries on the same connection). + // Connection-scoped SYMBOL dictionary. Populated by {@link #parseDeltaSymbolDict} + // from the per-message delta section. Two native buffers: + // - {@link #connDictHeapAddr} holds the concatenated UTF-8 bytes of every entry. + // - {@link #connDictEntriesAddr} holds one packed 8-byte pair per entry: + // {@code (offset:i32 | length:i32<<32)} relative to the heap base. + // Storing offsets (rather than absolute addresses) means heap reallocs don't + // invalidate entries; the hot-path accessor does one 64-bit load + base add, + // no object dereferences. Grows but never shrinks; freed on {@link #close}. + // Registry indexed by schemaId. null = not registered. Schema ids are server-assigned + // and small (monotonic from 0). + private final ObjList> schemaRegistry = new ObjList<>(); + private long connDictEntriesAddr; + private int connDictEntriesCapacity; + // Monotonic generation counter for the connection-scoped dict. Incremented + // whenever {@link #applyCacheReset} wipes the dict, so per-layout SYMBOL + // String caches (which key by dict id) can detect a reset and wipe lazily + // instead of paying an eager clear on every batch. + private long connDictGeneration; + private long connDictHeapAddr; + private int connDictHeapCapacity; + private int connDictHeapPos; + private int connDictSize; + // Native ZSTD_DCtx pointer, lazy-allocated on the first {@code FLAG_ZSTD} + // batch. One per decoder instance (which in turn is one per IoThread), reused + // for every subsequent compressed batch on the same connection. + private long dctx; + // Growable scratch holding the decompressed body between the zstd call and + // the downstream parse. Starts small and doubles on demand when the decoder + // reports a destination-too-small error. + private long decompressScratchAddr; + private int decompressScratchCapacity; + // True when the current message carries {@code FLAG_DELTA_SYMBOL_DICT}. Read by + // {@link #parseSymbolColumn} to decide whether to consume a per-column dict. + private boolean deltaMode; + // True when the current message carries {@code FLAG_GORILLA}. When set, + // TIMESTAMP / TIMESTAMP_NANOS / DATE columns are prefixed by a 1-byte + // encoding discriminator (0x00 raw, 0x01 Gorilla). + private boolean gorillaMode; + // Reusable varint decode state: value in varintValue, new position in varintPos. + // Instance-level so no {@code long[2]} scratch is allocated per call. + private long varintPos; + private long varintValue; + + /** + * Clears the caches indicated by {@code resetMask} (bitwise OR of + * {@link QwpEgressMsgKind#RESET_MASK_DICT} and + * {@link QwpEgressMsgKind#RESET_MASK_SCHEMAS}). Drives the client-side + * state machine when the server emits a {@code CACHE_RESET} frame after + * hitting a connection-level soft cap: discards the SYMBOL dict and / or + * schema-fingerprint cache so the next batch's {@code deltaStart} and + * schema-reference ids line up with the server's fresh counter. + *

+ * The native dict buffers are reused (positions reset to 0, capacity + * retained) so a workload that churns just above the cap does not reallocate + * on every reset. + */ + public void applyCacheReset(byte resetMask) { + if ((resetMask & QwpEgressMsgKind.RESET_MASK_DICT) != 0) { + connDictSize = 0; + connDictHeapPos = 0; + // Bump generation so any per-layout SYMBOL String cache populated + // against the pre-reset dict detects the change on its next lookup + // and wipes before reading. + connDictGeneration++; + } + if ((resetMask & QwpEgressMsgKind.RESET_MASK_SCHEMAS) != 0) { + schemaRegistry.clear(); + } + } + + @Override + public void close() { + if (connDictHeapAddr != 0) { + Unsafe.free(connDictHeapAddr, connDictHeapCapacity, MemoryTag.NATIVE_DEFAULT); + connDictHeapAddr = 0; + connDictHeapCapacity = 0; + connDictHeapPos = 0; + } + if (connDictEntriesAddr != 0) { + Unsafe.free(connDictEntriesAddr, connDictEntriesCapacity, MemoryTag.NATIVE_DEFAULT); + connDictEntriesAddr = 0; + connDictEntriesCapacity = 0; + } + if (decompressScratchAddr != 0) { + Unsafe.free(decompressScratchAddr, decompressScratchCapacity, MemoryTag.NATIVE_DEFAULT); + decompressScratchAddr = 0; + decompressScratchCapacity = 0; + } + if (dctx != 0) { + Zstd.freeDCtx(dctx); + dctx = 0; + } + connDictSize = 0; + } + + /** + * Decodes the RESULT_BATCH frame whose payload has been copied into {@code buffer}. + * Populates {@code buffer.batch} and {@code buffer.layoutPool}. The resulting + * batch view stays valid as long as the buffer is not reused. + */ + public void decode(QwpBatchBuffer buffer) throws QwpDecodeException { + decodePayload(buffer, buffer.getScratchAddr(), buffer.getPayloadLen()); + } + + /** + * In-place decode: parses the frame whose bytes live at {@code payloadPtr} (e.g. the + * WebSocket recv buffer) without copying into {@code buffer}'s native scratch. + * {@code buffer} contributes only its reusable layout pool and batch view; all + * column pointers produced reference {@code payloadPtr}, so the caller must keep + * those bytes stable until it's done reading the {@link QwpColumnBatch}. + */ + public void decode(QwpBatchBuffer buffer, long payloadPtr, int payloadLen) throws QwpDecodeException { + decodePayload(buffer, payloadPtr, payloadLen); + } + + // Pool helpers + + private static long advanceFixed(QwpColumnLayout layout, long p, long limit, int sizeBytes) throws QwpDecodeException { + layout.valuesAddr = p; + long total = (long) sizeBytes * layout.nonNullCount; + if (p + total > limit) throw new QwpDecodeException("truncated fixed-width column"); + return p + total; + } + + private static QwpColumnLayout borrowLayout(ObjList layoutPool, int colIdx) { + while (layoutPool.size() <= colIdx) { + layoutPool.add(new QwpColumnLayout()); + } + return layoutPool.getQuick(colIdx); + } + + private static int[] ensureIntArray(int[] current, int size) { + if (current != null && current.length >= size) return current; + return new int[Math.max(size, current == null ? 16 : current.length * 2)]; + } + + private static long[] ensureLongArray(long[] current, int size) { + if (current != null && current.length >= size) return current; + return new long[Math.max(size, current == null ? 16 : current.length * 2)]; + } + + /** + * STRING / VARCHAR: the offsets array is (nonNullCount+1) x uint32 starting at {@code p}, + * followed by the concatenated UTF-8 bytes. + */ + private static long parseStringColumn(QwpColumnLayout layout, long p, long limit) throws QwpDecodeException { + int nonNull = layout.nonNullCount; + long offsetsSize = 4L * (nonNull + 1); + if (p + offsetsSize > limit) throw new QwpDecodeException("truncated string offsets"); + layout.valuesAddr = p; + layout.stringBytesAddr = p + offsetsSize; + int totalBytes = nonNull == 0 ? 0 : Unsafe.getUnsafe().getInt(p + 4L * nonNull); + // totalBytes is signed int32 read from the wire. A negative value passes the + // "addr + totalBytes > limit" check (the sum stays below limit) and would + // return a position before stringBytesAddr -- subsequent column parsing would + // then read native memory backwards. Reject it explicitly. + if (totalBytes < 0 || layout.stringBytesAddr + totalBytes > limit) { + throw new QwpDecodeException("invalid string column total bytes: " + totalBytes); + } + // Validate per-row offsets: each must be in [0, totalBytes] and + // monotonically non-decreasing. Without this loop only offset[nonNull] + // is checked; a hostile server can supply a negative or out-of-order + // intermediate offset that QwpColumnBatch.lookupBinaryBytes / + // lookupStringBytes will then turn into a negative-length view (which + // surfaces as NegativeArraySizeException in getBinary) or a native + // address before stringBytesAddr (out-of-bounds read of unrelated + // native memory). Cost is one int load per non-null row over bytes + // already in cache. + int prev = 0; + for (int i = 0; i < nonNull; i++) { + int off = Unsafe.getUnsafe().getInt(p + 4L * i); + if (off < prev || off > totalBytes) { + throw new QwpDecodeException("invalid string column offset[" + i + "]=" + + off + " (prev=" + prev + ", total=" + totalBytes + ")"); + } + prev = off; + } + return layout.stringBytesAddr + totalBytes; + } + + private void decodePayload(QwpBatchBuffer buffer, long payload, int payloadLen) throws QwpDecodeException { + if (payloadLen < QwpConstants.HEADER_SIZE + 10) { + throw new QwpDecodeException("RESULT_BATCH payload too short: " + payloadLen); + } + // Message header + int magic = Unsafe.getUnsafe().getInt(payload); + if (magic != QwpConstants.MAGIC_MESSAGE) { + throw new QwpDecodeException("bad magic 0x" + Integer.toHexString(magic)); + } + byte version = Unsafe.getUnsafe().getByte(payload + 4); + if (version < QwpConstants.VERSION_1 || version > QwpConstants.MAX_SUPPORTED_VERSION) { + throw new QwpDecodeException("unsupported version " + (version & 0xFF)); + } + byte flags = Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS); + deltaMode = (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; + gorillaMode = (flags & QwpConstants.FLAG_GORILLA) != 0; + long p = payload + QwpConstants.HEADER_SIZE; + long limit = payload + payloadLen; + + byte msgKind = Unsafe.getUnsafe().getByte(p++); + if (msgKind != (byte) 0x11) { + throw new QwpDecodeException("expected RESULT_BATCH (0x11), got 0x" + Integer.toHexString(msgKind & 0xFF)); + } + if (p + 8 > limit) throw new QwpDecodeException("truncated request_id"); + long requestId = Unsafe.getUnsafe().getLong(p); + p += 8; + decodeVarint(p, limit); + long batchSeq = varintValue; + p = varintPos; + + // Zstd-compressed body: the region from here to {@code limit} is a + // single zstd frame covering the delta section + table block. Decode + // into the decoder-owned scratch buffer and rebind {@code p} / + // {@code limit} so downstream parsers see the plain bytes without any + // structural awareness of compression. {@code payloadAddr} exposed + // on the batch view also follows {@code p} here so size-measuring + // callers ({@code batch.payloadLimit() - batch.payloadAddr()}) report + // the uncompressed-equivalent body size rather than an arbitrary + // pointer delta between two native allocations. + long batchViewAddr = payload; + if ((flags & QwpConstants.FLAG_ZSTD) != 0) { + long srcLen = limit - p; + if (dctx == 0) { + try { + dctx = Zstd.createDCtx(); + } catch (UnsatisfiedLinkError e) { + // The early probe in QwpQueryClient.connect() should have + // caught this already. If we end up here, something went + // around that probe (custom client wiring, direct use of + // QwpResultBatchDecoder, etc.) -- surface a message that + // names the actual cause instead of letting the JNI symbol + // name leak into the user callback. + throw new QwpDecodeException("server sent a zstd-compressed batch but this " + + "client build does not include zstd -- libquestdb was built without " + + "the zstd submodule. Rebuild the native library with the submodule " + + "initialised, or set compression=raw on the connection string " + + "[cause=" + e.getMessage() + "]"); + } + if (dctx == 0) { + throw new QwpDecodeException("failed to allocate zstd decompression context"); + } + } + // Read the declared content size from the frame header BEFORE any + // allocation. This both lets us size the scratch in a single malloc + // (no decompress-fail-grow-retry loop) and -- crucially -- rejects + // corrupt or truncated frames upfront. Without the up-front check, + // a single hostile or malformed frame would have driven scratch + // doubling all the way to MAX_SCRATCH (~127 MiB of malloc/free + // churn per bad frame, plus 64 MiB pinned for the connection's + // lifetime). + final int MAX_SCRATCH = 64 * 1024 * 1024; + final int MIN_SCRATCH = 1024 * 1024; + long expectedSize = Zstd.getFrameContentSize(p, srcLen); + if (expectedSize == -2) { + throw new QwpDecodeException("invalid zstd frame header (truncated, bad magic, or content size > Long.MAX_VALUE)"); + } + if (expectedSize == -1) { + // The server's encoder leaves ZSTD_c_contentSizeFlag at its + // default (on), so every frame it produces declares content + // size. An "unknown" reading is therefore a protocol violation + // -- refuse rather than guess a buffer size and reopen the + // amplification window. + throw new QwpDecodeException("zstd frame missing content size (protocol violation)"); + } + if (expectedSize > MAX_SCRATCH) { + throw new QwpDecodeException("zstd frame content size " + expectedSize + + " exceeds client cap " + MAX_SCRATCH); + } + if (decompressScratchCapacity < expectedSize) { + long newCap = Math.max((long) decompressScratchCapacity * 2L, expectedSize); + if (newCap < MIN_SCRATCH) newCap = MIN_SCRATCH; + if (newCap > MAX_SCRATCH) newCap = MAX_SCRATCH; + // Reset to 0 before free + malloc so a throwing malloc cannot + // leave a dangling address + non-zero capacity behind; the next + // decode would otherwise skip the first-alloc branch and + // use-after-free. + if (decompressScratchAddr != 0) { + Unsafe.free(decompressScratchAddr, decompressScratchCapacity, MemoryTag.NATIVE_DEFAULT); + decompressScratchAddr = 0; + decompressScratchCapacity = 0; + } + decompressScratchAddr = Unsafe.malloc(newCap, MemoryTag.NATIVE_DEFAULT); + decompressScratchCapacity = (int) newCap; + } + long decLen = Zstd.decompress(dctx, p, srcLen, decompressScratchAddr, decompressScratchCapacity); + if (decLen < 0) { + throw new QwpDecodeException("zstd decompression failed [code=" + (-decLen) + "]"); + } + if (decLen != expectedSize) { + throw new QwpDecodeException("zstd decompressed size " + decLen + + " does not match frame content size " + expectedSize); + } + p = decompressScratchAddr; + limit = decompressScratchAddr + decLen; + batchViewAddr = decompressScratchAddr; + } + + // Delta section (if enabled) sits right after the prelude and before the + // table block. We consume it first so that SYMBOL columns inside the + // table block resolve indices against the freshly-updated connection dict. + if (deltaMode) { + p = parseDeltaSymbolDict(p, limit); + } + + // Table block: name_length, name, row_count, column_count, schema, columns + decodeVarint(p, limit); + long nameLen = varintValue; + p = varintPos; + // Negative nameLen would make p + nameLen wrap below the bound check + // and let the next decode rewind into already-consumed bytes. + if (nameLen < 0 || nameLen > QwpConstants.MAX_TABLE_NAME_LENGTH) { + throw new QwpDecodeException("table name length out of range: " + nameLen); + } + if (p + nameLen > limit) throw new QwpDecodeException("truncated table name"); + p += nameLen; + + decodeVarint(p, limit); + // Reject row counts that would force multi-GB allocations in ensureIntArray/ensureLongArray + // before the per-column bounds checks fire. A hostile varint with the high bit set also + // casts negative, which would silently flip bitmapBytes = (rowCount + 7) >>> 3 into a huge + // positive int via unsigned shift. + if (varintValue < 0 || varintValue > MAX_ROWS_PER_BATCH) { + throw new QwpDecodeException("row_count out of range: " + varintValue); + } + int rowCount = (int) varintValue; + p = varintPos; + decodeVarint(p, limit); + if (varintValue < 0 || varintValue > QwpConstants.MAX_COLUMNS_PER_TABLE) { + throw new QwpDecodeException("column_count out of range: " + varintValue); + } + int columnCount = (int) varintValue; + p = varintPos; + + // Schema section + if (p >= limit) throw new QwpDecodeException("truncated schema mode"); + byte schemaMode = Unsafe.getUnsafe().getByte(p++); + decodeVarint(p, limit); + // Reject schema ids that wouldn't fit in our registry (or that cast negative + // from a hostile high varint). Without this guard, ensureSchemaSlot would + // either OOM appending billions of nulls or AIOOBE on a negative index. + if (varintValue < 0 || varintValue >= MAX_SCHEMAS_PER_CONNECTION) { + throw new QwpDecodeException("schema_id out of range: " + varintValue); + } + int schemaId = (int) varintValue; + p = varintPos; + + ObjList columns; + if (schemaMode == QwpConstants.SCHEMA_MODE_FULL) { + columns = ensureSchemaSlot(schemaId, columnCount); + for (int i = 0; i < columnCount; i++) { + decodeVarint(p, limit); + // Same negative / out-of-range guard as nameLen above; without + // it, a hostile colNameLen wraps p + colNameLen below limit + // and lets the decoder rewind into already-consumed bytes. + if (varintValue < 0 || varintValue > QwpConstants.MAX_COLUMN_NAME_LENGTH) { + throw new QwpDecodeException("column name length out of range: " + varintValue); + } + int colNameLen = (int) varintValue; + p = varintPos; + if (p + colNameLen + 1 > limit) throw new QwpDecodeException("truncated column def"); + String colName = readColumnName(p, colNameLen); + p += colNameLen; + byte wireType = Unsafe.getUnsafe().getByte(p++); + columns.getQuick(i).of(colName, wireType); + } + } else if (schemaMode == QwpConstants.SCHEMA_MODE_REFERENCE) { + if (schemaId >= schemaRegistry.size() || schemaRegistry.getQuick(schemaId) == null) { + throw new QwpDecodeException("schema id " + schemaId + " not registered on this connection"); + } + columns = schemaRegistry.getQuick(schemaId); + if (columns.size() != columnCount) { + throw new QwpDecodeException("schema id " + schemaId + " column count mismatch"); + } + } else { + throw new QwpDecodeException("unknown schema mode 0x" + Integer.toHexString(schemaMode & 0xFF)); + } + + // Reset batch view and parse columns into per-column layouts owned by the buffer. + resetBatch(buffer, requestId, batchSeq, rowCount, columnCount, columns, batchViewAddr, limit); + for (int ci = 0; ci < columnCount; ci++) { + QwpColumnLayout layout = borrowLayout(buffer.layoutPool, ci); + layout.clear(); + layout.info = columns.getQuick(ci); + p = parseColumn(layout, rowCount, p, limit); + } + } + + /** + * Decodes a varint starting at {@code p}. Stores the decoded value in + * {@link #varintValue} and the position just past the varint in + * {@link #varintPos}. Caller reads both before issuing the next varint call. + *

+ * The encoding is the standard 7-bit varint: each byte contributes 7 data + * bits and signals continuation via the MSB. With ten data-carrying bytes + * the last byte's high data bits would spill past bit 63 of the result -- + * in Java the {@code <<} operator silently masks the shift count to 6 bits, + * which would wrap the high bits back into the low end and produce a value + * the caller couldn't tell from a legitimate small one. Reject those. + */ + private void decodeVarint(long p, long limit) throws QwpDecodeException { + long value = 0; + int shift = 0; + long cur = p; + while (true) { + if (cur >= limit) throw new QwpDecodeException("truncated varint"); + byte b = Unsafe.getUnsafe().getByte(cur++); + // On byte 10 (shift == 63) only bit 0 of the data nibble can fit + // in the result without overflowing bit 63 (the sign bit of long). + // Any other data bit on byte 10 means the encoded value would not + // round-trip; refuse to claim it decoded successfully. The server + // has the same guard in QwpVarint.decodeMultiByte. + if (shift == 63 && (b & 0x7E) != 0) { + throw new QwpDecodeException("varint overflow"); + } + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) break; + shift += 7; + if (shift > 63) throw new QwpDecodeException("varint overflow"); + } + varintValue = value; + varintPos = cur; + } + + // Per-column parse: advances through wire bytes, populates layout pointers, + // precomputes nonNullIdx for O(1) per-row access. + + private void ensureConnDictEntriesCapacity(int requiredEntries) { + int requiredBytes = requiredEntries * 8; + if (connDictEntriesCapacity >= requiredBytes) return; + int newCap = Math.max(connDictEntriesCapacity * 2, Math.max(CONN_DICT_INITIAL_ENTRIES * 8, requiredBytes)); + connDictEntriesAddr = Unsafe.realloc(connDictEntriesAddr, connDictEntriesCapacity, newCap, MemoryTag.NATIVE_DEFAULT); + connDictEntriesCapacity = newCap; + } + + private void ensureConnDictHeapCapacity(int required) { + if (connDictHeapCapacity >= required) return; + int newCap = Math.max(connDictHeapCapacity * 2, Math.max(CONN_DICT_INITIAL_BYTES, required)); + connDictHeapAddr = Unsafe.realloc(connDictHeapAddr, connDictHeapCapacity, newCap, MemoryTag.NATIVE_DEFAULT); + connDictHeapCapacity = newCap; + // No re-pointing needed: entries store offsets into the heap, not absolute + // addresses, so the hot-path accessor resolves against the current heap base. + } + + private ObjList ensureSchemaSlot(int schemaId, int columnCount) { + while (schemaRegistry.size() <= schemaId) { + schemaRegistry.add(null); + } + ObjList slot = schemaRegistry.getQuick(schemaId); + if (slot == null) { + slot = new ObjList<>(); + schemaRegistry.setQuick(schemaId, slot); + } + int currentPos = slot.size(); + if (columnCount > currentPos) { + slot.setPos(columnCount); + for (int i = currentPos; i < columnCount; i++) { + if (slot.getQuick(i) == null) { + slot.setQuick(i, new QwpEgressColumnInfo()); + } + } + } else { + slot.setPos(columnCount); + } + return slot; + } + + private long parseArrayColumn(QwpColumnLayout layout, int rowCount, long p, long limit) throws QwpDecodeException { + layout.arrayRowAddr = ensureLongArray(layout.arrayRowAddr, rowCount); + layout.arrayRowLen = ensureIntArray(layout.arrayRowLen, rowCount); + layout.valuesAddr = p; + // Hoist the no-nulls discriminator out of the per-row loop -- when the + // column has no nulls in this batch, every row is non-null and the + // null-skip branch is dead. + boolean noNulls = layout.nullBitmapAddr == 0; + int[] nonNullIdx = layout.nonNullIdx; + for (int i = 0; i < rowCount; i++) { + if (!noNulls && nonNullIdx[i] < 0) { + layout.arrayRowAddr[i] = 0; + layout.arrayRowLen[i] = 0; + continue; + } + if (p + 1 > limit) throw new QwpDecodeException("truncated ARRAY header"); + int nDims = Unsafe.getUnsafe().getByte(p) & 0xFF; + if (nDims < 1 || nDims > ColumnType.ARRAY_NDIMS_LIMIT) { + throw new QwpDecodeException("invalid array dimensions: " + nDims + + " (must be 1-" + ColumnType.ARRAY_NDIMS_LIMIT + ")"); + } + long headerEnd = p + 1 + 4L * nDims; + if (headerEnd > limit) throw new QwpDecodeException("truncated ARRAY dims"); + long elements = 1; + for (int d = 0; d < nDims; d++) { + int dl = Unsafe.getUnsafe().getInt(p + 1 + 4L * d); + // Require dl >= 1 in every dimension. A dl of 0 in any + // position would zero out {@code elements} and short-circuit + // the {@code MAX_ARRAY_ELEMENTS} cap for the rest of the + // loop, letting subsequent dimensions hold arbitrary values + // unchecked. The encoder side never emits dl == 0 (see + // {@code ColumnType} which asserts nDims >= 1 and treats + // every dimension symmetrically), so reject the + // wire-format inconsistency outright. + if (dl < 1) { + throw new QwpDecodeException("ARRAY dim " + d + " must be >= 1: " + dl); + } + elements *= dl; + if (elements > MAX_ARRAY_ELEMENTS) { + throw new QwpDecodeException("ARRAY element count exceeds limit (" + + elements + " > " + MAX_ARRAY_ELEMENTS + ")"); + } + } + long rowEnd = headerEnd + 8L * elements; + if (rowEnd > limit) throw new QwpDecodeException("truncated ARRAY payload"); + layout.arrayRowAddr[i] = p; + layout.arrayRowLen[i] = (int) (rowEnd - p); + p = rowEnd; + } + return p; + } + + private long parseColumn(QwpColumnLayout layout, int rowCount, long p, long limit) throws QwpDecodeException { + p = parseNullSection(layout, rowCount, p, limit); + byte wt = layout.info.wireType; + if (wt == QwpConstants.TYPE_BOOLEAN) { + layout.valuesAddr = p; + int bytes = (layout.nonNullCount + 7) >>> 3; + if (p + bytes > limit) throw new QwpDecodeException("truncated BOOLEAN"); + return p + bytes; + } + if (wt == QwpConstants.TYPE_BYTE) return advanceFixed(layout, p, limit, 1); + if (wt == QwpConstants.TYPE_SHORT || wt == QwpConstants.TYPE_CHAR) return advanceFixed(layout, p, limit, 2); + if (wt == QwpConstants.TYPE_INT || wt == QwpConstants.TYPE_FLOAT + || wt == QwpConstants.TYPE_IPv4) return advanceFixed(layout, p, limit, 4); + if (wt == QwpConstants.TYPE_LONG || wt == QwpConstants.TYPE_DOUBLE) { + return advanceFixed(layout, p, limit, 8); + } + if (wt == QwpConstants.TYPE_DATE + || wt == QwpConstants.TYPE_TIMESTAMP + || wt == QwpConstants.TYPE_TIMESTAMP_NANOS) { + return parseTimestampColumn(layout, p, limit); + } + if (wt == QwpConstants.TYPE_DECIMAL64) { + if (p >= limit) throw new QwpDecodeException("truncated DECIMAL64 scale"); + layout.info.scale = Unsafe.getUnsafe().getByte(p++) & 0xFF; + return advanceFixed(layout, p, limit, 8); + } + if (wt == QwpConstants.TYPE_UUID) return advanceFixed(layout, p, limit, 16); + if (wt == QwpConstants.TYPE_DECIMAL128) { + if (p >= limit) throw new QwpDecodeException("truncated DECIMAL128 scale"); + layout.info.scale = Unsafe.getUnsafe().getByte(p++) & 0xFF; + return advanceFixed(layout, p, limit, 16); + } + if (wt == QwpConstants.TYPE_LONG256) return advanceFixed(layout, p, limit, 32); + if (wt == QwpConstants.TYPE_DECIMAL256) { + if (p >= limit) throw new QwpDecodeException("truncated DECIMAL256 scale"); + layout.info.scale = Unsafe.getUnsafe().getByte(p++) & 0xFF; + return advanceFixed(layout, p, limit, 32); + } + if (wt == QwpConstants.TYPE_VARCHAR || wt == QwpConstants.TYPE_BINARY) { + // VARCHAR/BINARY share the (N+1) x uint32 offsets + concatenated bytes layout. + // BINARY differs only in that the bytes are opaque (no UTF-8 contract). + return parseStringColumn(layout, p, limit); + } + if (wt == QwpConstants.TYPE_SYMBOL) { + return parseSymbolColumn(layout, rowCount, p, limit); + } + if (wt == QwpConstants.TYPE_GEOHASH) { + decodeVarint(p, limit); + // The server enforces [1, 60] on GEOLONG precision; mirror the + // check here so a hostile varint that decodes out of range (or + // negative once cast to int) fails fast rather than driving a + // nonsense bytesPerValue into the length calculation below. + if (varintValue < 1 || varintValue > 60) { + throw new QwpDecodeException("GEOHASH precision bits out of range (1..60): " + varintValue); + } + layout.info.precisionBits = (int) varintValue; + p = varintPos; + int bytesPerValue = (layout.info.precisionBits + 7) >>> 3; + layout.valuesAddr = p; + long total = (long) bytesPerValue * layout.nonNullCount; + if (p + total > limit) throw new QwpDecodeException("truncated GEOHASH"); + return p + total; + } + if (wt == QwpConstants.TYPE_DOUBLE_ARRAY || wt == QwpConstants.TYPE_LONG_ARRAY) { + return parseArrayColumn(layout, rowCount, p, limit); + } + throw new QwpDecodeException("unsupported wire type 0x" + Integer.toHexString(wt & 0xFF)); + } + + /** + * Parses the message-level delta symbol dictionary section present when + * {@code FLAG_DELTA_SYMBOL_DICT} is set. Copies newly-seen symbol bytes into + * the decoder's connection-scoped native heap and appends packed + * {@code (offset:i32 | length:i32<<32)} entries to {@link #connDictEntriesAddr}. + * 

+     *   [deltaStartId: varint]
+     *   [deltaCount:   varint]
+     *   for each new entry: [length: varint][UTF-8 bytes]
+     *
+ * The server is required to emit {@code deltaStartId == connDictSize} + * (otherwise the two ends are out of sync and we bail rather than silently + * corrupt the dict). Returns the wire position just past the section. + */ + private long parseDeltaSymbolDict(long p, long limit) throws QwpDecodeException { + decodeVarint(p, limit); + long deltaStart = varintValue; + p = varintPos; + decodeVarint(p, limit); + long deltaCount = varintValue; + p = varintPos; + // Validate operands separately rather than via the sum: a hostile + // (deltaStart, deltaCount) pair where deltaStart matches connDictSize + // and deltaCount is large enough to make deltaStart + deltaCount wrap + // negative would otherwise bypass the cap check, then the (int) + // deltaCount cast in newSize below collapses to a small/negative int + // and the for-loop iterates while writing past connDictEntriesAddr. + // After the deltaStart upper-bound, MAX_CONN_DICT_SIZE - deltaStart is + // non-negative and the deltaCount comparison is well-defined. + if (deltaStart < 0 || deltaCount < 0 + || deltaStart > MAX_CONN_DICT_SIZE + || deltaCount > MAX_CONN_DICT_SIZE - deltaStart) { + throw new QwpDecodeException("delta symbol section out of range: start=" + + deltaStart + ", count=" + deltaCount); + } + if (deltaStart != connDictSize) { + throw new QwpDecodeException("delta symbol dict out of sync: expected start=" + + connDictSize + ", got=" + deltaStart); + } + int newSize = connDictSize + (int) deltaCount; + ensureConnDictEntriesCapacity(newSize); + for (long i = 0; i < deltaCount; i++) { + decodeVarint(p, limit); + long entryLen = varintValue; + p = varintPos; + if (entryLen < 0 || entryLen > Integer.MAX_VALUE || p + entryLen > limit) { + throw new QwpDecodeException("truncated delta symbol entry"); + } + int len = (int) entryLen; + // Long arithmetic so a hostile (connDictHeapPos + len) that would + // wrap int-negative gets caught by the cap check instead of + // silently skipping ensureConnDictHeapCapacity (which would let + // copyMemory write past the heap). + long newHeapPos = (long) connDictHeapPos + (long) len; + if (newHeapPos > MAX_CONN_DICT_HEAP_BYTES) { + throw new QwpDecodeException("connection SYMBOL dict heap exceeds cap (" + + MAX_CONN_DICT_HEAP_BYTES + " bytes); server must emit CACHE_RESET"); + } + ensureConnDictHeapCapacity((int) newHeapPos); + int offset = connDictHeapPos; + Unsafe.getUnsafe().copyMemory(p, connDictHeapAddr + offset, len); + // Pack (offset, length) into one 8-byte entry. Low 32 bits = offset, + // high 32 bits = length. Single 64-bit load + two int extractions in + // the hot-path accessor. + long packed = (offset & 0xFFFFFFFFL) | ((long) len << 32); + Unsafe.getUnsafe().putLong(connDictEntriesAddr + 8L * connDictSize, packed); + connDictSize++; + connDictHeapPos = (int) newHeapPos; + p += len; + } + return p; + } + + /** + * Reads the null flag and bitmap, populates {@code layout.nullBitmapAddr} and + * {@code layout.nonNullCount}, and fills {@code layout.nonNullIdx[0..rowCount)} + * with dense indices (or -1 for NULL rows). Returns the position just past + * the null section. + */ + private long parseNullSection(QwpColumnLayout layout, int rowCount, long p, long limit) throws QwpDecodeException { + if (p >= limit) throw new QwpDecodeException("truncated null flag"); + byte flag = Unsafe.getUnsafe().getByte(p++); + if (flag == 0) { + // No nulls in this column -- skip the per-row "nonNullIdx[i] = i" + // array fill entirely. Accessors detect the no-nulls case via + // {@code nullBitmapAddr == 0} and treat dense-index == row directly + // (see {@link QwpColumnLayout#denseIndex}), so the array is unread + // on this path. For a 16K-row x 100-column wide result this saves + // 1.6M trivial assignments per batch. nonNullIdx is nulled so a + // raw-API caller of {@code QwpColumnBatch.nonNullIndex(col)} can + // distinguish "needs identity-fill" from "fully populated by a + // prior with-nulls batch"; that path lazy-materialises on demand. + layout.nullBitmapAddr = 0; + layout.nonNullIdx = null; + layout.nonNullCount = rowCount; + return p; + } + int bitmapBytes = (rowCount + 7) >>> 3; + if (p + bitmapBytes > limit) throw new QwpDecodeException("truncated null bitmap"); + layout.nullBitmapAddr = p; + layout.nonNullIdx = ensureIntArray(layout.nonNullIdx, rowCount); + int denseIdx = 0; + for (int i = 0; i < rowCount; i++) { + int bi = i >>> 3; + int bit = i & 7; + byte bm = Unsafe.getUnsafe().getByte(p + bi); + if ((bm & (1 << bit)) != 0) { + layout.nonNullIdx[i] = -1; + } else { + layout.nonNullIdx[i] = denseIdx++; + } + } + layout.nonNullCount = denseIdx; + return p + bitmapBytes; + } + + /** + * SYMBOL: in delta mode (always, with the current server) there's no per-column + * dictionary -- indices reference the connection-scoped dict built up by + * {@link #parseDeltaSymbolDict}. In non-delta mode the column carries its own + * dict: (dict_size varint, then len+bytes per entry), followed by per-non-null-row + * varint indices. + */ + private long parseSymbolColumn(QwpColumnLayout layout, int rowCount, long p, long limit) throws QwpDecodeException { + final int dictSize; + if (deltaMode) { + // Point the column's dict at the connection-scoped native arrays; size + // is whatever the dict has grown to across all batches on this connection. + layout.symbolDictHeapAddr = connDictHeapAddr; + layout.symbolDictEntriesAddr = connDictEntriesAddr; + dictSize = connDictSize; + // Stamp the cache version with an even-tagged generation so the + // String cache populated on a previous delta-mode batch stays + // valid until CACHE_RESET bumps the generation. Low bit 0 marks + // delta-mode tokens; non-delta uses bit 0 = 1 (see else branch). + layout.symbolDictVersion = connDictGeneration << 1; + } else { + decodeVarint(p, limit); + // A column can have at most rowCount distinct symbols, and a + // hostile varint can decode negative once cast to int. Reject both + // before the int-multiply below, which would otherwise overflow + // silently and let {@code ensureOwnedEntriesAddr} return without + // growing the buffer. + if (varintValue < 0 || varintValue > rowCount) { + throw new QwpDecodeException("SYMBOL dict size out of range: " + varintValue + + " (rowCount=" + rowCount + ")"); + } + dictSize = (int) varintValue; + p = varintPos; + // Non-delta: dict entries point directly into the payload buffer. Build + // a per-layout packed-entries array keyed by offset-from-the-dict-base. + long dictBase = p; + long entriesAddr = layout.ensureOwnedEntriesAddr(dictSize * 8); + for (int e = 0; e < dictSize; e++) { + decodeVarint(p, limit); + long entryLen = varintValue; + p = varintPos; + // entryLen must be non-negative and fit in int. Without this, + // a hostile varint can drive `p + entryLen` negative (int-cast + // sign-extend on the long addend wraps), slipping the + // `> limit` check; the subsequent `p += entryLen` then + // advances backwards through previously-consumed bytes. + if (entryLen < 0 || entryLen > Integer.MAX_VALUE || p + entryLen > limit) { + throw new QwpDecodeException("truncated symbol entry"); + } + int lenBytes = (int) entryLen; + int offset = (int) (p - dictBase); + long packed = (offset & 0xFFFFFFFFL) | ((long) lenBytes << 32); + Unsafe.getUnsafe().putLong(entriesAddr + 8L * e, packed); + p += lenBytes; + } + layout.symbolDictHeapAddr = dictBase; + layout.symbolDictEntriesAddr = entriesAddr; + // Non-delta: the dict lives inside the payload and changes every + // batch, so the cache must invalidate on every parse. Using + // (dictBase | 1) picks up the per-batch address change and also + // sets the low bit so a non-delta token can never collide with a + // delta generation token. + layout.symbolDictVersion = dictBase | 1L; + } + layout.symbolDictSize = dictSize; + // Materialise per-row IDs into int[rowCount] so random access is O(1). + layout.symbolRowIds = ensureIntArray(layout.symbolRowIds, rowCount); + // Hoist the no-nulls discriminator out of the per-row loop -- when the + // column has no nulls in this batch, every row carries an id and the + // null-skip branch is dead. + boolean noNulls = layout.nullBitmapAddr == 0; + int[] nonNullIdx = layout.nonNullIdx; + for (int i = 0; i < rowCount; i++) { + if (!noNulls && nonNullIdx[i] < 0) continue; // NULL row; leave slot stale + decodeVarint(p, limit); + p = varintPos; + int id = (int) varintValue; + if (id < 0 || id >= dictSize) { + throw new QwpDecodeException("symbol index out of range: " + id); + } + layout.symbolRowIds[i] = id; + } + layout.valuesAddr = 0; // Not applicable; accessors use symbolRowIds + symbolDictEntriesAddr. + return p; + } + + /** + * TIMESTAMP / TIMESTAMP_NANOS / DATE: with {@code FLAG_GORILLA} set on the + * message, the column is prefixed with a 1-byte encoding discriminator + * ({@code 0x00} uncompressed, {@code 0x01} Gorilla bitstream). Without the + * flag the column is plain 8-byte fixed-width -- the pre-Gorilla format. + *

+ * Uncompressed: {@link QwpColumnLayout#valuesAddr} points directly at the + * wire bytes. Gorilla: we decode into a per-layout native i64 buffer and + * point {@code valuesAddr} at it, so the caller's {@code getLong(col, row)} + * path is identical in both cases. + */ + private long parseTimestampColumn(QwpColumnLayout layout, long p, long limit) throws QwpDecodeException { + int nonNull = layout.nonNullCount; + if (!gorillaMode) { + return advanceFixed(layout, p, limit, 8); + } + if (p >= limit) throw new QwpDecodeException("truncated TIMESTAMP encoding byte"); + byte encoding = Unsafe.getUnsafe().getByte(p++); + if (encoding == 0x00) { + // Uncompressed: bytes sit inline just like advanceFixed. + long total = (long) nonNull * 8L; + if (p + total > limit) throw new QwpDecodeException("truncated TIMESTAMP raw values"); + layout.valuesAddr = p; + return p + total; + } + if (encoding != 0x01) { + throw new QwpDecodeException("unknown TIMESTAMP encoding 0x" + Integer.toHexString(encoding & 0xFF)); + } + // Gorilla: first two timestamps are uncompressed (16 bytes), the rest is + // a bitstream. Server shortcuts nonNull<3 to the uncompressed branch, so + // we always have at least 3 values here. + if (nonNull < 3) throw new QwpDecodeException("Gorilla-encoded column with nonNull<3: " + nonNull); + if (p + 16L > limit) throw new QwpDecodeException("truncated Gorilla prefix"); + long firstTs = Unsafe.getUnsafe().getLong(p); + long secondTs = Unsafe.getUnsafe().getLong(p + 8L); + long bitstreamStart = p + 16L; + long decodeAddr = layout.ensureTimestampDecodeAddr(nonNull * 8); + Unsafe.getUnsafe().putLong(decodeAddr, firstTs); + Unsafe.getUnsafe().putLong(decodeAddr + 8L, secondTs); + gorillaDecoder.reset(firstTs, secondTs, bitstreamStart, limit - bitstreamStart); + for (int i = 2; i < nonNull; i++) { + Unsafe.getUnsafe().putLong(decodeAddr + (long) i * 8L, gorillaDecoder.decodeNext()); + } + layout.valuesAddr = decodeAddr; + long bitPos = gorillaDecoder.getBitPosition(); + long bitstreamBytes = (bitPos + 7L) >>> 3; + long end = bitstreamStart + bitstreamBytes; + if (end > limit) throw new QwpDecodeException("truncated Gorilla bitstream"); + return end; + } + + /** + * Copies {@code len} bytes from native address {@code p} into the reusable + * {@link #colNameScratch} buffer and materialises them as a UTF-8 {@code + * String}. Callers must hold {@code len <= MAX_COLUMN_NAME_LENGTH}; the + * varint-length parse ({@link QwpConstants#MAX_COLUMN_NAME_LENGTH} guard) + * upstream in {@link #decodePayload} already ensures that. + */ + private String readColumnName(long p, int len) { + for (int i = 0; i < len; i++) { + colNameScratch[i] = Unsafe.getUnsafe().getByte(p + i); + } + return new String(colNameScratch, 0, len, StandardCharsets.UTF_8); + } + + private void resetBatch( + QwpBatchBuffer buffer, + long requestId, + long batchSeq, + int rowCount, + int columnCount, + ObjList columns, + long payloadAddr, + long payloadLimit + ) { + QwpColumnBatch batch = buffer.batch; + batch.requestId = requestId; + batch.batchSeq = batchSeq; + batch.rowCount = rowCount; + batch.columnCount = columnCount; + batch.columns = columns; + batch.payloadAddr = payloadAddr; + batch.payloadLimit = payloadLimit; + // Surface the buffer-owned layouts to the batch view + while (batch.columnLayouts.size() < columnCount) { + batch.columnLayouts.add(null); + } + for (int i = 0; i < columnCount; i++) { + batch.columnLayouts.setQuick(i, borrowLayout(buffer.layoutPool, i)); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpRoleMismatchException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpRoleMismatchException.java new file mode 100644 index 00000000..00209bed --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpRoleMismatchException.java @@ -0,0 +1,60 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; + +/** + * Thrown by {@link QwpQueryClient#connect()} when every configured endpoint + * reports a role that does not satisfy the client's {@code target=} filter. + * Callers can distinguish this from a plain {@link HttpClientException} (all + * endpoints unreachable) to tailor retry behaviour: "no primary available" is + * worth waiting and retrying after a failover window, "all endpoints down" is + * a harder failure. + */ +public class QwpRoleMismatchException extends HttpClientException { + + private final QwpServerInfo lastObserved; + private final String targetRole; + + public QwpRoleMismatchException(String targetRole, QwpServerInfo lastObserved, String message) { + super(message); + this.targetRole = targetRole; + this.lastObserved = lastObserved; + } + + /** + * The {@link QwpServerInfo} from the last endpoint the client tried before + * giving up. Useful for diagnostics; may be null if no endpoint even + * responded with a {@code SERVER_INFO} frame. + */ + public QwpServerInfo getLastObserved() { + return lastObserved; + } + + public String getTargetRole() { + return targetRole; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfo.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfo.java new file mode 100644 index 00000000..951fb9f2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfo.java @@ -0,0 +1,112 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Decoded {@code SERVER_INFO} frame delivered by a v2 QWP egress server as the + * first WebSocket frame after the upgrade handshake. Immutable; safe to publish + * across threads without synchronisation once assigned. + */ +public final class QwpServerInfo { + + private final int capabilities; + private final String clusterId; + private final long epoch; + private final String nodeId; + private final byte role; + private final long serverWallNs; + + public QwpServerInfo( + byte role, + long epoch, + int capabilities, + long serverWallNs, + String clusterId, + String nodeId + ) { + this.role = role; + this.epoch = epoch; + this.capabilities = capabilities; + this.serverWallNs = serverWallNs; + this.clusterId = clusterId; + this.nodeId = nodeId; + } + + /** + * Returns the human-readable name for the given role byte. Useful when + * constructing error messages from {@link QwpQueryClient#getServerRole()} + * or logging role-mismatch diagnostics. Unknown values (outside the + * published enum) are returned as {@code "UNKNOWN(n)"}. + */ + public static String roleName(byte role) { + switch (role) { + case QwpEgressMsgKind.ROLE_STANDALONE: + return "STANDALONE"; + case QwpEgressMsgKind.ROLE_PRIMARY: + return "PRIMARY"; + case QwpEgressMsgKind.ROLE_REPLICA: + return "REPLICA"; + case QwpEgressMsgKind.ROLE_PRIMARY_CATCHUP: + return "PRIMARY_CATCHUP"; + default: + return "UNKNOWN(" + (role & 0xFF) + ")"; + } + } + + public int getCapabilities() { + return capabilities; + } + + public String getClusterId() { + return clusterId; + } + + public long getEpoch() { + return epoch; + } + + public String getNodeId() { + return nodeId; + } + + public byte getRole() { + return role; + } + + public long getServerWallNs() { + return serverWallNs; + } + + @Override + public String toString() { + return "QwpServerInfo{role=" + roleName(role) + + ", epoch=" + epoch + + ", clusterId='" + clusterId + '\'' + + ", nodeId='" + nodeId + '\'' + + ", capabilities=0x" + Integer.toHexString(capabilities) + + ", serverWallNs=" + serverWallNs + + '}'; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfoDecoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfoDecoder.java new file mode 100644 index 00000000..7b9d382e --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfoDecoder.java @@ -0,0 +1,113 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.Unsafe; + +import java.nio.charset.StandardCharsets; + +/** + * Parses a {@code SERVER_INFO} frame payload off the wire into an immutable + * {@link QwpServerInfo}. The payload pointer points at the start of the QWP + * message (12-byte QWP header followed by the body), matching what the + * WebSocket frame handler receives for every egress binary frame. + *

+ * Bounds checks on every read are required because a hostile or buggy server + * can encode a {@code u16} length that overshoots the frame remainder. The + * decoder surfaces the mismatch as a {@link QwpDecodeException} rather than + * reading past the end of the native buffer. + */ +public final class QwpServerInfoDecoder { + + private QwpServerInfoDecoder() { + } + + /** + * Decodes a {@code SERVER_INFO} frame in place from {@code payload} for + * {@code payloadLen} bytes. + * + * @throws QwpDecodeException if the frame is truncated, the msg_kind is not + * {@link QwpEgressMsgKind#SERVER_INFO}, or either + * length prefix exceeds the remainder. + */ + public static QwpServerInfo decode(long payload, int payloadLen) throws QwpDecodeException { + final int fixedBytes = QwpConstants.HEADER_SIZE + 1 + 1 + 8 + 4 + 8 + 2; + if (payloadLen < fixedBytes) { + throw new QwpDecodeException("SERVER_INFO frame truncated [payloadLen=" + payloadLen + + ", minRequired=" + fixedBytes + ']'); + } + long p = payload + QwpConstants.HEADER_SIZE; + byte msgKind = Unsafe.getUnsafe().getByte(p); + if (msgKind != QwpEgressMsgKind.SERVER_INFO) { + throw new QwpDecodeException("expected SERVER_INFO msg_kind 0x" + + Integer.toHexString(QwpEgressMsgKind.SERVER_INFO & 0xFF) + + " got 0x" + Integer.toHexString(msgKind & 0xFF)); + } + p += 1; + byte role = Unsafe.getUnsafe().getByte(p); + p += 1; + long epoch = Unsafe.getUnsafe().getLong(p); + p += 8; + int capabilities = Unsafe.getUnsafe().getInt(p); + p += 4; + long serverWallNs = Unsafe.getUnsafe().getLong(p); + p += 8; + long payloadEnd = payload + payloadLen; + String clusterId = readUtf8U16(p, payloadEnd, "cluster_id"); + int clusterLen = Unsafe.getUnsafe().getShort(p) & 0xFFFF; + p += 2 + clusterLen; + if (p + 2 > payloadEnd) { + throw new QwpDecodeException("SERVER_INFO truncated before node_id length"); + } + String nodeId = readUtf8U16(p, payloadEnd, "node_id"); + return new QwpServerInfo(role, epoch, capabilities, serverWallNs, clusterId, nodeId); + } + + /** + * Reads a u16-length-prefixed UTF-8 string at {@code p}. Validates that the + * length fits within the payload before copying, so a hostile {@code u16} + * length can't drag bytes out of unrelated native memory. + */ + private static String readUtf8U16(long p, long payloadEnd, String fieldName) throws QwpDecodeException { + if (p + 2 > payloadEnd) { + throw new QwpDecodeException("SERVER_INFO truncated before " + fieldName + " length"); + } + int len = Unsafe.getUnsafe().getShort(p) & 0xFFFF; + long bytesStart = p + 2; + if (bytesStart + len > payloadEnd) { + throw new QwpDecodeException("SERVER_INFO " + fieldName + " length " + len + + " exceeds frame remainder " + (payloadEnd - bytesStart)); + } + if (len == 0) { + return ""; + } + byte[] bytes = new byte[len]; + for (int i = 0; i < len; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(bytesStart + i); + } + return new String(bytes, StandardCharsets.UTF_8); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSpscQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSpscQueue.java new file mode 100644 index 00000000..ccc6990b --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSpscQueue.java @@ -0,0 +1,148 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import java.util.concurrent.locks.LockSupport; + +/** + * Bounded single-producer single-consumer queue with spin-then-park blocking. + *

+ * Purpose-built for the QwpEgressIoThread hand-off hot paths where + * {@code java.util.concurrent.ArrayBlockingQueue} costs ~2 microseconds per + * offer/take cycle under an uncontended lock + park pattern. This queue is + * lock-free on the fast path and parks only after a short spin window (default + * ~50 microseconds at current CPU speeds), so queries whose producer arrives + * within the budget skip park/unpark entirely. + *

+ * Contract: + *

    + *
  • Exactly one producer thread may call {@link #offer}.
    • + *
  • Exactly one consumer thread may call {@link #poll} or {@link #take}.
    • + *
  • Capacity is rounded up to the next power of two.
    • + *
+ * Behaviour outside these assumptions is undefined. + */ +public final class QwpSpscQueue { + + // Tuned for latency-sensitive localhost workloads: long enough to catch a + // typical round-trip inside the spin window (~30 microseconds on loopback), + // short enough not to dominate CPU on idle queues. The spin loop exits + // early on every iteration that observes a non-empty ring, so this is an + // upper bound, not a fixed cost. + private static final int SPIN_ITERATIONS = 2048; + private final int mask; + // One-slot handshake so the producer knows whether to unpark the consumer. + // Null when the consumer is running (no unpark needed); non-null when + // parked (producer must unpark to release). Volatile write on set ensures + // the consumer's prior tail read is visible to the producer's queue check. + private final Object[] slots; + private volatile Thread consumerThread; + private volatile long head; + private volatile long tail; + + public QwpSpscQueue(int capacity) { + int pow2 = 1; + while (pow2 < capacity) pow2 <<= 1; + this.slots = new Object[pow2]; + this.mask = pow2 - 1; + } + + /** + * Publishes {@code value} to the consumer. Returns {@code false} when the + * ring is full (caller may retry or spin externally). Never blocks. + */ + public boolean offer(T value) { + final long h = head; + // Producer-only read of tail -- consumer's volatile write to tail + // happens-before this read via the queue discipline. + if (h - tail >= slots.length) { + return false; + } + slots[(int) (h & mask)] = value; + head = h + 1; // volatile write publishes the slot to the consumer + // Wake the consumer if it parked between its last poll and our offer. + // consumerThread is non-null only when the consumer is inside the park + // phase of take(); unpark while it is null is a no-op so we stay cheap + // on the fast path. + Thread consumer = consumerThread; + if (consumer != null) { + LockSupport.unpark(consumer); + } + return true; + } + + /** Non-blocking read. Returns {@code null} when the ring is empty. */ + @SuppressWarnings("unchecked") + public T poll() { + final long t = tail; + if (t == head) { + return null; + } + final int idx = (int) (t & mask); + T value = (T) slots[idx]; + slots[idx] = null; + tail = t + 1; + return value; + } + + /** + * Spin-then-park take. Returns the next value, or throws + * {@link InterruptedException} when the consumer thread was interrupted + * while waiting. + */ + @SuppressWarnings("unchecked") + public T take() throws InterruptedException { + // Fast path: the producer beat us here and the value is already visible. + T value = poll(); + if (value != null) { + return value; + } + // Spin phase: burn a bounded amount of CPU hoping the producer arrives + // inside the window. onSpinWait lets the CPU slow its pipeline while + // waiting -- meaningful on hyperthreaded cores. + for (int i = 0; i < SPIN_ITERATIONS; i++) { + Thread.onSpinWait(); + if ((value = poll()) != null) { + return value; + } + } + // Park phase: publish ourselves as the consumer so a subsequent offer + // sees a non-null reference and unparks us. Re-poll after publishing + // to close the race where the producer offered between our last poll + // and the consumerThread assignment. + consumerThread = Thread.currentThread(); + try { + while ((value = poll()) == null) { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + LockSupport.park(); + } + return value; + } finally { + consumerThread = null; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 93189006..16dfc14a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1368,7 +1368,7 @@ private void ensureConnected() { // Connect and upgrade to WebSocket try { - client.setQwpMaxVersion(QwpConstants.MAX_SUPPORTED_VERSION); + client.setQwpMaxVersion(QwpConstants.MAX_SUPPORTED_INGEST_VERSION); client.setQwpClientId(QwpConstants.CLIENT_ID); client.setQwpRequestDurableAck(requestDurableAck); client.connect(host, port); @@ -1441,7 +1441,7 @@ private void flushPendingRows() { return; } - // Invalidate cached column references — table buffers will be reset below + // Invalidate cached column references -- table buffers will be reset below cachedTimestampColumn = null; cachedTimestampNanosColumn = null; @@ -1548,7 +1548,7 @@ private void flushSync() { return; } - // Invalidate cached column references — table buffers will be reset below + // Invalidate cached column references -- table buffers will be reset below cachedTimestampColumn = null; cachedTimestampNanosColumn = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/RowCallback.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/RowCallback.java new file mode 100644 index 00000000..cfcb6192 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/RowCallback.java @@ -0,0 +1,46 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +/** + * Per-row callback consumed by {@link QwpColumnBatch#forEachRow(RowCallback)}. + * The {@link RowView} handed to {@link #onRow(RowView)} is a reusable flyweight + * pointing at the current row; do not retain it past the call. To capture a + * row's values, copy them out of the view. + *

+ * Throwing from {@link #onRow} aborts iteration and propagates the exception + * out of {@code forEachRow} on the caller's thread. + */ +@FunctionalInterface +public interface RowCallback { + + /** + * Invoked once for each row in the batch, in row-index order starting at 0. + * + * @param row reusable view bound to the current row; valid only for the + * duration of this call + */ + void onRow(RowView row); +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/RowView.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/RowView.java new file mode 100644 index 00000000..6405374b --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/RowView.java @@ -0,0 +1,289 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.std.Long256Sink; +import io.questdb.client.std.Uuid; +import io.questdb.client.std.bytes.DirectByteSequence; +import io.questdb.client.std.str.CharSink; +import io.questdb.client.std.str.DirectUtf8Sequence; + +/** + * Row-pinned view over a {@link QwpColumnBatch}. Exposes the per-cell accessors + * with the row index implicit, so consumers iterate rows in the conventional + * "current row + column index" style instead of passing both indices to every + * call: + * 

+ *   batch.forEachRow(row -> {
+ *       long ts    = row.getLongValue(0);
+ *       String sym = row.getString(1);
+ *       double px  = row.getDoubleValue(2);
+ *   });
+ *
+ *

+ * Lifetime. The instance is owned by the batch, reused across + * rows, and valid only for the duration of the {@code forEachRow} (or + * surrounding {@code onBatch}) call. Do not retain past the callback. + *

+ * Type contract / NULL handling. Each accessor delegates to + * the corresponding {@code QwpColumnBatch} primitive; the contracts in + * {@link QwpColumnBatch} apply unchanged. In particular, calling an accessor + * whose type doesn't match the column's wire type yields undefined results, + * and NULL rows return type-specific zero values. + *

+ * Column metadata. Per-column attributes (name, wire type, + * decimal scale, geohash precision, symbol dict size) live on the batch. Use + * {@link #batch()} to reach them without capturing a separate reference. + */ +public class RowView { + + private final QwpColumnBatch batch; + private int row; + + RowView(QwpColumnBatch batch) { + this.batch = batch; + } + + /** + * Returns the parent batch. Use it to read column-level metadata + * ({@link QwpColumnBatch#getColumnName}, {@link QwpColumnBatch#getColumnWireType}, + * etc.) or to reach for less common per-cell accessors not exposed on this view. + */ + public QwpColumnBatch batch() { + return batch; + } + + /** + * @see QwpColumnBatch#getArrayNDims(int, int) + */ + public int getArrayNDims(int col) { + return batch.getArrayNDims(col, row); + } + + /** + * @see QwpColumnBatch#getBinary(int, int) + */ + public byte[] getBinary(int col) { + return batch.getBinary(col, row); + } + + /** + * @see QwpColumnBatch#getBinaryA(int, int) + */ + public DirectByteSequence getBinaryA(int col) { + return batch.getBinaryA(col, row); + } + + /** + * @see QwpColumnBatch#getBinaryB(int, int) + */ + public DirectByteSequence getBinaryB(int col) { + return batch.getBinaryB(col, row); + } + + /** + * @see QwpColumnBatch#getBoolValue(int, int) + */ + public boolean getBoolValue(int col) { + return batch.getBoolValue(col, row); + } + + /** + * @see QwpColumnBatch#getByteValue(int, int) + */ + public byte getByteValue(int col) { + return batch.getByteValue(col, row); + } + + /** + * @see QwpColumnBatch#getCharValue(int, int) + */ + public char getCharValue(int col) { + return batch.getCharValue(col, row); + } + + /** + * @see QwpColumnBatch#getDecimal128High(int, int) + */ + public long getDecimal128High(int col) { + return batch.getDecimal128High(col, row); + } + + /** + * @see QwpColumnBatch#getDecimal128Low(int, int) + */ + public long getDecimal128Low(int col) { + return batch.getDecimal128Low(col, row); + } + + /** + * @see QwpColumnBatch#getDoubleArrayElements(int, int) + */ + public double[] getDoubleArrayElements(int col) { + return batch.getDoubleArrayElements(col, row); + } + + /** + * @see QwpColumnBatch#getDoubleValue(int, int) + */ + public double getDoubleValue(int col) { + return batch.getDoubleValue(col, row); + } + + /** + * @see QwpColumnBatch#getFloatValue(int, int) + */ + public float getFloatValue(int col) { + return batch.getFloatValue(col, row); + } + + /** + * @see QwpColumnBatch#getGeohashValue(int, int) + */ + public long getGeohashValue(int col) { + return batch.getGeohashValue(col, row); + } + + /** + * @see QwpColumnBatch#getIntValue(int, int) + */ + public int getIntValue(int col) { + return batch.getIntValue(col, row); + } + + /** + * @see QwpColumnBatch#getLong256(int, int, Long256Sink) + */ + public boolean getLong256(int col, Long256Sink sink) { + return batch.getLong256(col, row, sink); + } + + /** + * @see QwpColumnBatch#getLong256Word(int, int, int) + */ + public long getLong256Word(int col, int wordIndex) { + return batch.getLong256Word(col, row, wordIndex); + } + + /** + * @see QwpColumnBatch#getLongValue(int, int) + */ + public long getLongValue(int col) { + return batch.getLongValue(col, row); + } + + /** + * Index of the row this view is currently pinned to. + */ + public int getRowIndex() { + return row; + } + + /** + * @see QwpColumnBatch#getShortValue(int, int) + */ + public short getShortValue(int col) { + return batch.getShortValue(col, row); + } + + /** + * @see QwpColumnBatch#getStrA(int, int) + */ + public DirectUtf8Sequence getStrA(int col) { + return batch.getStrA(col, row); + } + + /** + * @see QwpColumnBatch#getStrB(int, int) + */ + public DirectUtf8Sequence getStrB(int col) { + return batch.getStrB(col, row); + } + + /** + * @see QwpColumnBatch#getString(int, int) + */ + public String getString(int col) { + return batch.getString(col, row); + } + + /** + * @see QwpColumnBatch#getString(int, int, CharSink) + */ + public boolean getString(int col, CharSink sink) { + return batch.getString(col, row, sink); + } + + /** + * @see QwpColumnBatch#getSymbol(int, int) + */ + public String getSymbol(int col) { + return batch.getSymbol(col, row); + } + + /** + * @see QwpColumnBatch#getSymbolId(int, int) + */ + public int getSymbolId(int col) { + return batch.getSymbolId(col, row); + } + + /** + * @see QwpColumnBatch#getUuid(int, int, Uuid) + */ + public boolean getUuid(int col, Uuid sink) { + return batch.getUuid(col, row, sink); + } + + /** + * @see QwpColumnBatch#getUuidHi(int, int) + */ + public long getUuidHi(int col) { + return batch.getUuidHi(col, row); + } + + /** + * @see QwpColumnBatch#getUuidLo(int, int) + */ + public long getUuidLo(int col) { + return batch.getUuidLo(col, row); + } + + /** + * @see QwpColumnBatch#isNull(int, int) + */ + public boolean isNull(int col) { + return batch.isNull(col, row); + } + + /** + * Re-points this flyweight at {@code row}. Returns {@code this} so callers + * can chain a single accessor: {@code batch.row().of(r).getLongValue(0)}. + */ + public RowView of(int row) { + this.row = row; + return this; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java new file mode 100644 index 00000000..6427131c --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpBitReader.java @@ -0,0 +1,157 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.client.QwpDecodeException; +import io.questdb.client.std.Unsafe; + +/** + * Client-side bit-level reader for QWP v1 Gorilla-compressed columns. Mirrors + * the server-side reader. Bits are read LSB-first; the buffer lazily pulls + * bytes from the underlying native address as needed. + *

+ * Overflow surfaces as {@link QwpDecodeException} so a malformed server frame + * is reported to the user handler via {@code onError}, not an uncaught error. + */ +public class QwpBitReader { + + // Buffer for reading bits + private long bitBuffer; + // Number of bits currently available in the buffer (0-64) + private int bitsInBuffer; + private long currentAddress; + private long endAddress; + // Total bits available for reading (from reset) + private long totalBitsAvailable; + // Total bits already consumed + private long totalBitsRead; + + public QwpBitReader() { + } + + public long getBitPosition() { + return totalBitsRead; + } + + /** + * Reads a single bit. + */ + public int readBit() throws QwpDecodeException { + if (totalBitsRead >= totalBitsAvailable) { + throw new QwpDecodeException("QwpBitReader: read past end"); + } + if (!ensureBits(1)) { + throw new QwpDecodeException("QwpBitReader: read past end"); + } + + int bit = (int) (bitBuffer & 1); + bitBuffer >>>= 1; + bitsInBuffer--; + totalBitsRead++; + return bit; + } + + /** + * Reads multiple bits and returns them as a long (unsigned, LSB-aligned). + */ + public long readBits(int numBits) throws QwpDecodeException { + if (numBits <= 0) { + return 0; + } + if (numBits > 64) { + throw new AssertionError("Asked to read more than 64 bits into a long"); + } + if (totalBitsRead + numBits > totalBitsAvailable) { + throw new QwpDecodeException("QwpBitReader: read past end"); + } + + long result = 0; + int bitsRemaining = numBits; + int resultShift = 0; + + while (bitsRemaining > 0) { + if (bitsInBuffer == 0) { + if (!ensureBits(Math.min(bitsRemaining, 64))) { + throw new QwpDecodeException("QwpBitReader: read past end"); + } + } + + int bitsToTake = Math.min(bitsRemaining, bitsInBuffer); + long mask = bitsToTake == 64 ? -1L : (1L << bitsToTake) - 1; + result |= (bitBuffer & mask) << resultShift; + + // Java masks the right operand of {@code >>>} by 0x3F for long, so + // {@code bitBuffer >>>= 64} is a no-op and would leave the just- + // consumed 64 bits in {@code bitBuffer}. The next ensureBits OR-fills + // a fresh byte at bit 0 of that stale buffer, silently corrupting + // every subsequent read. Special-case the full-width consume so + // callers using readBits(64) (or readSigned(64)) are safe. + if (bitsToTake == 64) { + bitBuffer = 0L; + } else { + bitBuffer >>>= bitsToTake; + } + bitsInBuffer -= bitsToTake; + bitsRemaining -= bitsToTake; + resultShift += bitsToTake; + } + + totalBitsRead += numBits; + return result; + } + + /** + * Reads multiple bits and interprets them as a signed value (two's complement). + */ + public long readSigned(int numBits) throws QwpDecodeException { + long unsigned = readBits(numBits); + if (numBits < 64 && (unsigned & (1L << (numBits - 1))) != 0) { + unsigned |= -1L << numBits; + } + return unsigned; + } + + /** + * Resets the reader to read from {@code length} bytes starting at {@code address}. + */ + public void reset(long address, long length) { + this.currentAddress = address; + this.endAddress = address + length; + this.bitBuffer = 0; + this.bitsInBuffer = 0; + this.totalBitsAvailable = length * 8L; + this.totalBitsRead = 0; + } + + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + private boolean ensureBits(int bitsNeeded) { + while (bitsInBuffer < bitsNeeded && bitsInBuffer <= 56 && currentAddress < endAddress) { + byte b = Unsafe.getUnsafe().getByte(currentAddress++); + bitBuffer |= (long) (b & 0xFF) << bitsInBuffer; + bitsInBuffer += 8; + } + return bitsInBuffer >= bitsNeeded; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index b3d948cf..c23b8045 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -45,6 +45,12 @@ public final class QwpConstants { * Flag bit: Gorilla timestamp encoding enabled. */ public static final byte FLAG_GORILLA = 0x04; + /** + * Flag bit: payload region after the prelude is zstd-compressed. Set only + * when the handshake negotiated zstd compression. Mirror of the server-side + * constant; see the server QwpConstants for the full description. + */ + public static final byte FLAG_ZSTD = 0x10; /** * Offset of flags byte in header. */ @@ -63,6 +69,17 @@ public final class QwpConstants { * the server and finding out. */ public static final int MAX_COLUMNS_PER_TABLE = 2048; + /** + * Maximum column name length in bytes. Mirrors the server's same-named + * constant so the decoder can reject hostile or malformed wire bytes that + * advertise an oversized name length. + */ + public static final int MAX_COLUMN_NAME_LENGTH = 127; + /** + * Maximum table name length in bytes. Mirrors the server's same-named + * constant; used by the decoder to reject malformed wire bytes. + */ + public static final int MAX_TABLE_NAME_LENGTH = 127; /** * Schema mode: Full schema included. */ @@ -71,6 +88,24 @@ public final class QwpConstants { * Schema mode: Schema reference (ID lookup). */ public static final byte SCHEMA_MODE_REFERENCE = 0x01; + /** + * Status byte on a {@code QUERY_ERROR} frame: the query was cancelled, + * either by a client {@code CANCEL} frame or by explicit server-side + * cancel. Egress extension of the ingress {@code STATUS_*} namespace + * (0x00-0x09). + */ + public static final byte STATUS_CANCELLED = 0x0A; + /** + * Status byte on a {@code QUERY_ERROR} frame: a server-side limit was hit + * (query timeout, memory cap, circuit breaker, OOM). Egress extension of + * the ingress {@code STATUS_*} namespace (0x00-0x09). + */ + public static final byte STATUS_LIMIT_EXCEEDED = 0x0B; + /** + * Column type: BINARY (length-prefixed opaque bytes). + * Wire format: identical to VARCHAR — (N+1) x uint32 offsets + concatenated bytes. + */ + public static final byte TYPE_BINARY = 0x17; /** * Column type: BOOLEAN (1 bit per value, packed). */ @@ -123,6 +158,11 @@ public final class QwpConstants { * Column type: INT (int32, little-endian). */ public static final byte TYPE_INT = 0x04; + /** + * Column type: IPv4 (32-bit address). Wire format: 4 bytes LE, identical to INT. + * NULL is signalled via the standard null bitmap. + */ + public static final byte TYPE_IPv4 = 0x18; /** * Column type: LONG (int64, little-endian). */ @@ -167,9 +207,25 @@ public final class QwpConstants { */ public static final byte VERSION_1 = 1; /** - * Maximum protocol version supported by this client. + * Protocol v2 adds an unsolicited {@code SERVER_INFO} control frame + * delivered as the first WebSocket frame after the 101 upgrade. Servers that + * advertise v2 send the frame automatically; v2 clients must consume it + * before submitting the first query and may use the role value it carries + * to route reads to primary vs replica. + */ + public static final byte VERSION_2 = 2; + /** + * Maximum protocol version the ingest path advertises / accepts. Pinned to + * v1 because the v2 bump only adds an egress control frame; bumping the + * ingest wire to v2 would silently round-trip a version byte that no v1 + * server can accept. + */ + public static final byte MAX_SUPPORTED_INGEST_VERSION = VERSION_1; + /** + * Maximum protocol version supported by this client on the egress path. + * Ingest pins to {@link #MAX_SUPPORTED_INGEST_VERSION}. */ - public static final byte MAX_SUPPORTED_VERSION = VERSION_1; + public static final byte MAX_SUPPORTED_VERSION = VERSION_2; private QwpConstants() { // utility class @@ -187,8 +243,7 @@ private QwpConstants() { * @return size in bytes, 0 for bit-packed (BOOLEAN), or -1 for variable-width types */ public static int getFixedTypeSize(byte typeCode) { - int code = typeCode; - switch (code) { + switch ((int) typeCode) { case TYPE_BOOLEAN: return 0; // Special: bit-packed case TYPE_BYTE: @@ -226,9 +281,8 @@ public static int getFixedTypeSize(byte typeCode) { * @return type name */ public static String getTypeName(byte typeCode) { - int code = typeCode; String name; - switch (code) { + switch ((int) typeCode) { case TYPE_BOOLEAN: name = "BOOLEAN"; break; @@ -293,7 +347,7 @@ public static String getTypeName(byte typeCode) { name = "DECIMAL256"; break; default: - name = "UNKNOWN(" + code + ")"; + name = "UNKNOWN(" + (int) typeCode + ")"; break; } return name; @@ -306,22 +360,21 @@ public static String getTypeName(byte typeCode) { * @return true if fixed-width */ public static boolean isFixedWidthType(byte typeCode) { - int code = typeCode; - return code == TYPE_BOOLEAN || - code == TYPE_BYTE || - code == TYPE_SHORT || - code == TYPE_CHAR || - code == TYPE_INT || - code == TYPE_LONG || - code == TYPE_FLOAT || - code == TYPE_DOUBLE || - code == TYPE_TIMESTAMP || - code == TYPE_TIMESTAMP_NANOS || - code == TYPE_DATE || - code == TYPE_UUID || - code == TYPE_LONG256 || - code == TYPE_DECIMAL64 || - code == TYPE_DECIMAL128 || - code == TYPE_DECIMAL256; + return (int) typeCode == TYPE_BOOLEAN || + (int) typeCode == TYPE_BYTE || + (int) typeCode == TYPE_SHORT || + (int) typeCode == TYPE_CHAR || + (int) typeCode == TYPE_INT || + (int) typeCode == TYPE_LONG || + (int) typeCode == TYPE_FLOAT || + (int) typeCode == TYPE_DOUBLE || + (int) typeCode == TYPE_TIMESTAMP || + (int) typeCode == TYPE_TIMESTAMP_NANOS || + (int) typeCode == TYPE_DATE || + (int) typeCode == TYPE_UUID || + (int) typeCode == TYPE_LONG256 || + (int) typeCode == TYPE_DECIMAL64 || + (int) typeCode == TYPE_DECIMAL128 || + (int) typeCode == TYPE_DECIMAL256; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaDecoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaDecoder.java new file mode 100644 index 00000000..70c3d10f --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpGorillaDecoder.java @@ -0,0 +1,102 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.client.QwpDecodeException; + +/** + * Client-side Gorilla delta-of-delta decoder for timestamp columns in QWP + * egress {@code RESULT_BATCH} frames. Mirrors the server-side decoder and + * reads the bitstream produced by {@code QwpGorillaEncoder}. + *

+ * Encoding buckets: + * 

+ *   '0'                     -> DoD = 0                 (1 bit)
+ *   '10' + 7-bit signed     -> DoD in [-64, 63]        (9 bits)
+ *   '110' + 9-bit signed    -> DoD in [-256, 255]     (12 bits)
+ *   '1110' + 12-bit signed  -> DoD in [-2048, 2047]   (16 bits)
+ *   '1111' + 32-bit signed  -> any other DoD          (36 bits)
+ *
+ */ +public class QwpGorillaDecoder { + + private final QwpBitReader bitReader = new QwpBitReader(); + private long prevDelta; + private long prevTimestamp; + + public QwpGorillaDecoder() { + } + + /** + * Decodes the next timestamp from the bit stream. + */ + public long decodeNext() throws QwpDecodeException { + long deltaOfDelta = decodeDoD(); + long delta = prevDelta + deltaOfDelta; + long timestamp = prevTimestamp + delta; + + prevDelta = delta; + prevTimestamp = timestamp; + return timestamp; + } + + /** + * Returns the current bit position (bits read since reset). + */ + public long getBitPosition() { + return bitReader.getBitPosition(); + } + + /** + * Resets the decoder. First two timestamps are always shipped uncompressed + * at the head of the column's wire bytes; the address + length here point + * at the bitstream that follows them. + */ + public void reset(long firstTimestamp, long secondTimestamp, long address, long length) { + this.prevTimestamp = secondTimestamp; + this.prevDelta = secondTimestamp - firstTimestamp; + bitReader.reset(address, length); + } + + private long decodeDoD() throws QwpDecodeException { + int bit = bitReader.readBit(); + if (bit == 0) { + return 0; + } + bit = bitReader.readBit(); + if (bit == 0) { + return bitReader.readSigned(7); + } + bit = bitReader.readBit(); + if (bit == 0) { + return bitReader.readSigned(9); + } + bit = bitReader.readBit(); + if (bit == 0) { + return bitReader.readSigned(12); + } + return bitReader.readSigned(32); + } +} diff --git a/core/src/main/java/io/questdb/client/std/Long256.java b/core/src/main/java/io/questdb/client/std/Long256.java index 1a62b96b..e49ea1fe 100644 --- a/core/src/main/java/io/questdb/client/std/Long256.java +++ b/core/src/main/java/io/questdb/client/std/Long256.java @@ -25,9 +25,18 @@ package io.questdb.client.std; /** - * A 256-bit hash with string representation up to 64 hex digits following a prefix '0x'. - * (e.g. 0xaba86bf575ba7fde98b6673bb7d85bf489fd71a619cddaecba5de0378e3d22ed) + * A 256-bit value split into four 64-bit words, least-significant first. + * Also used for QuestDB's DECIMAL256 unscaled value (same wire layout). + * Pair with {@link Long256Impl} as a reusable sink. */ public interface Long256 { int BYTES = 32; -} \ No newline at end of file + + long getLong0(); + + long getLong1(); + + long getLong2(); + + long getLong3(); +} diff --git a/core/src/main/java/io/questdb/client/std/Long256Impl.java b/core/src/main/java/io/questdb/client/std/Long256Impl.java new file mode 100644 index 00000000..cc39be79 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/Long256Impl.java @@ -0,0 +1,65 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +/** + * Mutable concrete {@link Long256} backed by four {@code long} fields. Intended + * as a reusable sink: hand the same instance to a series of producers (e.g. + * QWP batch accessors) to avoid per-row allocation. + */ +public class Long256Impl implements Long256, Long256Sink { + private long l0; + private long l1; + private long l2; + private long l3; + + @Override + public long getLong0() { + return l0; + } + + @Override + public long getLong1() { + return l1; + } + + @Override + public long getLong2() { + return l2; + } + + @Override + public long getLong3() { + return l3; + } + + @Override + public void setAll(long l0, long l1, long l2, long l3) { + this.l0 = l0; + this.l1 = l1; + this.l2 = l2; + this.l3 = l3; + } +} diff --git a/core/src/main/java/io/questdb/client/std/Long256Sink.java b/core/src/main/java/io/questdb/client/std/Long256Sink.java new file mode 100644 index 00000000..f1d012b6 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/Long256Sink.java @@ -0,0 +1,53 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +/** + * Write-side of a 256-bit value split into four 64-bit words. Implementations + * accept a full value in a single call, so producers can ship all four words + * with one virtual dispatch instead of one per word. + */ +public interface Long256Sink { + + /** + * Copies four little-endian 64-bit words starting at {@code address} into + * this sink. Default implementation issues four native 64-bit loads and + * delegates to {@link #setAll}. + */ + default void fromAddress(long address) { + setAll( + Unsafe.getUnsafe().getLong(address), + Unsafe.getUnsafe().getLong(address + 8L), + Unsafe.getUnsafe().getLong(address + 16L), + Unsafe.getUnsafe().getLong(address + 24L) + ); + } + + /** + * Sets all four 64-bit words of this value. {@code l0} is the least + * significant word; {@code l3} the most significant. + */ + void setAll(long l0, long l1, long l2, long l3); +} diff --git a/core/src/main/java/io/questdb/client/std/Uuid.java b/core/src/main/java/io/questdb/client/std/Uuid.java new file mode 100644 index 00000000..43b1c8ea --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/Uuid.java @@ -0,0 +1,64 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +/** + * Mutable 128-bit UUID value split into two 64-bit words. Intended as a + * reusable sink: hand the same instance to a series of producers (e.g. QWP + * batch accessors) and read {@link #getLo} / {@link #getHi} after each + * {@link #setAll} call. + */ +public class Uuid { + public static final int BYTES = 16; + private long hi; + private long lo; + + /** + * Loads the two 64-bit words from {@code address} (little-endian, low word first). + */ + public void fromAddress(long address) { + setAll( + Unsafe.getUnsafe().getLong(address), + Unsafe.getUnsafe().getLong(address + 8L) + ); + } + + public long getHi() { + return hi; + } + + public long getLo() { + return lo; + } + + /** + * Sets both 64-bit words. {@code lo} is the least significant; {@code hi} + * the most significant. + */ + public void setAll(long lo, long hi) { + this.lo = lo; + this.hi = hi; + } +} diff --git a/core/src/main/java/io/questdb/client/std/Zstd.java b/core/src/main/java/io/questdb/client/std/Zstd.java new file mode 100644 index 00000000..a4ea4426 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/Zstd.java @@ -0,0 +1,64 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +/** + * Client-side JNI wrapper over libzstd's decompression API. QWP's RESULT_BATCH + * frames can carry zstd-compressed bodies when the handshake negotiated + * {@code X-QWP-Accept-Encoding: zstd}; this class is the decoder-side entry + * point. The native implementation lives in + * {@code share/zstd/zstd_jni.c} (bundled libzstd, decompress only). + */ +public final class Zstd { + + private Zstd() { + } + + public static native long createDCtx(); + + /** + * Decompresses {@code srcLen} bytes at {@code srcAddr} into the buffer at + * {@code dstAddr} (capacity {@code dstCap}). Returns the number of bytes + * written on success; a negative value encodes a libzstd error code. + */ + public static native long decompress(long ctx, long srcAddr, long srcLen, long dstAddr, long dstCap); + + public static native void freeDCtx(long ptr); + + /** + * Returns the decompressed size declared in the zstd frame header at + * {@code srcAddr} (length {@code srcLen}). Allows the caller to size the + * destination buffer in a single allocation instead of retrying on + * dst-too-small. + *
    + *
  • positive: declared content size in bytes
    • + *
  • {@code -1}: frame valid but content size not stored + * (ZSTD_CONTENTSIZE_UNKNOWN; the encoder disabled the size flag)
    • + *
  • {@code -2}: invalid frame, truncated header, or size exceeds + * {@link Long#MAX_VALUE}
    • + *
+ */ + public static native long getFrameContentSize(long srcAddr, long srcLen); +} diff --git a/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib b/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib index 38f76b86..6157114f 100644 Binary files a/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib and b/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib differ diff --git a/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib b/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib index 72d228a1..daef5dce 100644 Binary files a/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib and b/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib differ diff --git a/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so b/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so index 76158e16..16ae826d 100644 Binary files a/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so and b/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so differ diff --git a/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so b/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so index 426586bf..f9513ef2 100644 Binary files a/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so and b/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so differ diff --git a/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll b/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll index 7aeb62f9..2e6bbb72 100755 Binary files a/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll and b/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll differ diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java new file mode 100644 index 00000000..9867ee95 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QueryEventTest.java @@ -0,0 +1,166 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QueryEvent; +import io.questdb.client.cutlass.qwp.client.QwpBatchBuffer; +import org.junit.Assert; +import org.junit.Test; + +/** + * Coverage for the {@link QueryEvent} pooled bean: each {@code asX()} builder + * sets the matching {@code KIND_*} discriminator and its payload fields, and + * {@link QueryEvent#reset()} clears every reference / scalar so a recycled + * event from the I/O thread's pool starts in a clean state. + */ +public class QueryEventTest { + + @Test + public void testFreshEventDefaults() { + // A bare new QueryEvent has kind=0 (KIND_BATCH default) and zeroed + // primitives. The I/O thread always calls one of the asX() builders + // before publishing, so the defaults are only relevant before first use. + QueryEvent ev = new QueryEvent(); + Assert.assertEquals(0, ev.kind); + Assert.assertNull(ev.buffer); + Assert.assertNull(ev.errorMessage); + Assert.assertEquals(0, ev.errorStatus); + Assert.assertEquals(0, ev.opType); + Assert.assertEquals(0L, ev.rowsAffected); + Assert.assertEquals(0L, ev.totalRows); + } + + @Test + public void testAsBatchSetsBufferAndKind() { + QueryEvent ev = new QueryEvent(); + try (QwpBatchBuffer buf = new QwpBatchBuffer(64)) { + QueryEvent returned = ev.asBatch(buf); + Assert.assertSame("builder must return this for chaining", ev, returned); + Assert.assertEquals(QueryEvent.KIND_BATCH, ev.kind); + Assert.assertSame(buf, ev.buffer); + } + } + + @Test + public void testAsEndSetsTotalRowsAndClearsBuffer() { + QueryEvent ev = new QueryEvent(); + // pre-populate to verify asEnd clears the buffer field. + try (QwpBatchBuffer stale = new QwpBatchBuffer(64)) { + ev.asBatch(stale); + QueryEvent returned = ev.asEnd(123_456L); + Assert.assertSame(ev, returned); + Assert.assertEquals(QueryEvent.KIND_END, ev.kind); + Assert.assertEquals(123_456L, ev.totalRows); + Assert.assertNull("asEnd must drop the stale buffer reference", ev.buffer); + } + } + + @Test + public void testAsErrorSetsStatusAndMessage() { + QueryEvent ev = new QueryEvent(); + QueryEvent returned = ev.asError((byte) 0x05, "parse error: unexpected token"); + Assert.assertSame(ev, returned); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals((byte) 0x05, ev.errorStatus); + Assert.assertEquals("parse error: unexpected token", ev.errorMessage); + Assert.assertNull("asError must clear buffer", ev.buffer); + } + + @Test + public void testAsExecDoneSetsOpTypeAndRowsAffected() { + QueryEvent ev = new QueryEvent(); + QueryEvent returned = ev.asExecDone((short) 7, 999L); + Assert.assertSame(ev, returned); + Assert.assertEquals(QueryEvent.KIND_EXEC_DONE, ev.kind); + Assert.assertEquals((short) 7, ev.opType); + Assert.assertEquals(999L, ev.rowsAffected); + Assert.assertNull(ev.buffer); + } + + @Test + public void testAsTransportErrorSetsStatusAndMessage() { + QueryEvent ev = new QueryEvent(); + QueryEvent returned = ev.asTransportError((byte) 0x7F, "socket reset"); + Assert.assertSame(ev, returned); + Assert.assertEquals(QueryEvent.KIND_TRANSPORT_ERROR, ev.kind); + Assert.assertEquals((byte) 0x7F, ev.errorStatus); + Assert.assertEquals("socket reset", ev.errorMessage); + Assert.assertNull(ev.buffer); + } + + @Test + public void testKindConstantsAreDistinct() { + // Sanity: the KIND_* discriminators must be five distinct ints, otherwise + // a switch over event.kind would silently merge two cases. Putting them + // in a set asserts uniqueness without hard-coding the actual byte values. + int[] kinds = { + QueryEvent.KIND_BATCH, + QueryEvent.KIND_END, + QueryEvent.KIND_ERROR, + QueryEvent.KIND_EXEC_DONE, + QueryEvent.KIND_TRANSPORT_ERROR + }; + for (int i = 0; i < kinds.length; i++) { + for (int j = i + 1; j < kinds.length; j++) { + Assert.assertNotEquals("KIND constants must be distinct", kinds[i], kinds[j]); + } + } + } + + @Test + public void testResetClearsEverythingFromError() { + QueryEvent ev = new QueryEvent(); + ev.asError((byte) 0x05, "boom"); + ev.reset(); + Assert.assertEquals(-1, ev.kind); + Assert.assertNull(ev.buffer); + Assert.assertNull(ev.errorMessage); + Assert.assertEquals(0, ev.errorStatus); + Assert.assertEquals(0, ev.opType); + Assert.assertEquals(0L, ev.rowsAffected); + Assert.assertEquals(0L, ev.totalRows); + } + + @Test + public void testResetClearsEverythingFromBatch() { + QueryEvent ev = new QueryEvent(); + try (QwpBatchBuffer buf = new QwpBatchBuffer(64)) { + ev.asBatch(buf); + ev.reset(); + Assert.assertEquals(-1, ev.kind); + Assert.assertNull("reset must drop the buffer reference for prompt GC", ev.buffer); + } + } + + @Test + public void testResetClearsEverythingFromExecDone() { + QueryEvent ev = new QueryEvent(); + ev.asExecDone((short) 9, 1234L); + ev.reset(); + Assert.assertEquals(-1, ev.kind); + Assert.assertEquals(0, ev.opType); + Assert.assertEquals(0L, ev.rowsAffected); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpBatchBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpBatchBufferTest.java new file mode 100644 index 00000000..734fbcda --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpBatchBufferTest.java @@ -0,0 +1,113 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpBatchBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +/** + * Hardening tests for {@link QwpBatchBuffer#copyFromPayload}'s capacity growth + * path. The previous {@code ensureCapacity} implementation spun forever when + * {@code scratchCapacity == 0} (doubling never advances past zero) and + * silently wrapped negative when {@code required > Integer.MAX_VALUE / 2}. + */ +public class QwpBatchBufferTest { + + /** + * Regression: a buffer constructed with {@code initialCapacity == 0} must + * still grow on the first non-zero payload. Previously the doubling loop + * {@code newCap *= 2} with {@code newCap == 0} spun forever, hanging the + * decoder. The fix starts the doubling at {@code max(current, 1)}. + */ + @Test + public void testEnsureCapacityGrowsFromZero() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpBatchBuffer buffer = new QwpBatchBuffer(0); + long src = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 64; i++) { + Unsafe.getUnsafe().putByte(src + i, (byte) (i + 1)); + } + buffer.copyFromPayload(src, 64); + Assert.assertEquals(64, buffer.getPayloadLen()); + // Spot-check a couple of bytes to confirm the copy actually + // landed in the grown scratch. + Assert.assertEquals((byte) 1, Unsafe.getUnsafe().getByte(buffer.getScratchAddr())); + Assert.assertEquals((byte) 64, Unsafe.getUnsafe().getByte(buffer.getScratchAddr() + 63)); + } finally { + Unsafe.free(src, 64, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + } + }); + } + + /** + * Regression: a negative required size used to flow through the doubling + * loop as an always-true comparison, and subsequent realloc produced + * undefined native behaviour. The fix rejects it explicitly. + */ + @Test(expected = IllegalArgumentException.class) + public void testEnsureCapacityRejectsNegativeRequired() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpBatchBuffer buffer = new QwpBatchBuffer(64); + try { + // copyFromPayload forwards the negative length to ensureCapacity. + buffer.copyFromPayload(0L, -1); + } finally { + buffer.close(); + } + }); + } + + /** + * Regression: the doubling loop used to perform {@code newCap *= 2} in int + * space, which wraps negative once {@code newCap} exceeds 2^30 and would + * then spin forever without ever reaching {@code required}. The fix + * performs the doubling in long space and clamps at + * {@link Integer#MAX_VALUE}. We drive the growth path with a request + * comfortably under the halfway mark but well above the initial + * capacity, which would have had to double several times under the old + * code path; completing in bounded time means the fix held. + */ + @Test(timeout = 5_000L) + public void testEnsureCapacityGrowsInBoundedTime() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpBatchBuffer buffer = new QwpBatchBuffer(4); + int required = 16 * 1024 * 1024; // 16 MiB -- several doublings above 4 bytes. + long src = Unsafe.malloc(required, MemoryTag.NATIVE_DEFAULT); + try { + buffer.copyFromPayload(src, required); + Assert.assertEquals(required, buffer.getPayloadLen()); + } finally { + Unsafe.free(src, required, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpBindEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpBindEncoderTest.java new file mode 100644 index 00000000..19ddd2fc --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpBindEncoderTest.java @@ -0,0 +1,987 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpBindValues; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimals; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * Wire-format unit tests for {@link QwpBindValues}. Each test encodes one or + * more binds and asserts the exact byte layout against a hand-built expected + * payload. No server or connection required. + */ +public class QwpBindEncoderTest { + + // Bind header bytes. + private static final byte NON_NULL = 0x00; + private static final byte NULL_BITMAP = 0x01; + private static final byte NULL_FLAG = 0x01; + + @Test + public void testBindsWithinSingleReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.reset(); + binds.setLong(0, 42L); + binds.setInt(1, 7); + Assert.assertEquals(2, binds.count()); + byte[] first = readBuffer(binds); + + binds.reset(); + Assert.assertEquals(0, binds.count()); + Assert.assertEquals(0, binds.bufferLen()); + + binds.setLong(0, 42L); + binds.setInt(1, 7); + byte[] second = readBuffer(binds); + Assert.assertArrayEquals(first, second); + } + }); + } + + @Test + public void testEncodeBoolean() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setBoolean(0, true); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_BOOLEAN); + writer.write(NON_NULL); + writer.write(1); + })); + } + try (QwpBindValues binds = new QwpBindValues()) { + binds.setBoolean(0, false); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_BOOLEAN); + writer.write(NON_NULL); + writer.write(0); + })); + } + }); + } + + @Test + public void testEncodeByte() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setByte(0, (byte) -128); + binds.setByte(1, (byte) 0); + binds.setByte(2, (byte) 127); + byte[] want = expected(writer -> { + writer.write(QwpConstants.TYPE_BYTE); + writer.write(NON_NULL); + writer.write((byte) -128); + writer.write(QwpConstants.TYPE_BYTE); + writer.write(NON_NULL); + writer.write(0); + writer.write(QwpConstants.TYPE_BYTE); + writer.write(NON_NULL); + writer.write(127); + }); + assertEncoded(binds, 3, want); + } + }); + } + + @Test + public void testEncodeChar() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setChar(0, 'Z'); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_CHAR); + writer.write(NON_NULL); + writeShortLe(writer, (short) 'Z'); + })); + } + }); + } + + @Test + public void testEncodeDate() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setDate(0, 1_700_000_000_000L); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DATE); + writer.write(NON_NULL); + writeLongLe(writer, 1_700_000_000_000L); + })); + } + }); + } + + @Test + public void testEncodeDecimal128() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setDecimal128(0, 6, 0x0123456789ABCDEFL, 0x7766554433221100L); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DECIMAL128); + writer.write(NON_NULL); + writer.write(6); + writeLongLe(writer, 0x0123456789ABCDEFL); + writeLongLe(writer, 0x7766554433221100L); + })); + } + }); + } + + @Test + public void testEncodeDecimal128ConvenienceNullSentinel() throws Exception { + // The server reads the DECIMAL128 scale byte regardless of the null + // flag (it becomes part of the bound variable's type), so the client + // must emit it on the NULL path too -- and preserve the scale from + // the supplied Decimal128 sentinel rather than defaulting to 0. + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + Decimal128 nullValue = new Decimal128(Decimals.DECIMAL128_HI_NULL, Decimals.DECIMAL128_LO_NULL, 4); + binds.setDecimal128(0, nullValue); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DECIMAL128); + writer.write(NULL_FLAG); + writer.write(NULL_BITMAP); + writer.write(4); + })); + } + }); + } + + @Test + public void testEncodeDecimal128RejectsNegativeScale() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + try { + binds.setDecimal128(0, -1, 1L, 0L); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("scale")); + } + } + }); + } + + /** + * DECIMAL128 tops out at 38 digits of precision, so scale above 38 is + * mathematically invalid. Reject per-width rather than against the + * DECIMAL256 ceiling (76) the shared cap used to fall back to. + */ + @Test + public void testEncodeDecimal128RejectsScaleAbove38() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + try { + binds.setDecimal128(0, 39, 1L, 0L); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue("message must mention DECIMAL128: " + expected.getMessage(), + expected.getMessage().contains("DECIMAL128")); + } + } + }); + } + + @Test + public void testEncodeDecimal128RejectsScaleAboveMax() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + try { + binds.setDecimal128(0, Decimals.MAX_SCALE + 1, 1L, 0L); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("scale")); + } + } + }); + } + + @Test + public void testEncodeDecimal256() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setDecimal256(0, 10, 0x1111111111111111L, 0x2222222222222222L, 0x3333333333333333L, 0x4444444444444444L); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DECIMAL256); + writer.write(NON_NULL); + writer.write(10); + writeLongLe(writer, 0x1111111111111111L); + writeLongLe(writer, 0x2222222222222222L); + writeLongLe(writer, 0x3333333333333333L); + writeLongLe(writer, 0x4444444444444444L); + })); + } + }); + } + + @Test + public void testEncodeDecimal256ConvenienceNullSentinel() throws Exception { + // Parallel to the DECIMAL128 case: the scale from the null sentinel + // is propagated to the wire so it survives round-trip as part of the + // bound variable's type. + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + Decimal256 nullValue = new Decimal256( + Decimals.DECIMAL256_HH_NULL, + Decimals.DECIMAL256_HL_NULL, + Decimals.DECIMAL256_LH_NULL, + Decimals.DECIMAL256_LL_NULL, + 3); + binds.setDecimal256(0, nullValue); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DECIMAL256); + writer.write(NULL_FLAG); + writer.write(NULL_BITMAP); + writer.write(3); + })); + } + }); + } + + @Test + public void testEncodeDecimal256AllowsScale76() throws Exception { + // Top of the DECIMAL256 range: scale 76 is the documented ceiling. + // Verify the encoder accepts it (used to pass under the shared-cap + // check; the per-width split must not accidentally lower the ceiling). + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setDecimal256(0, 76, 1L, 0L, 0L, 0L); + Assert.assertEquals(1, binds.count()); + } + }); + } + + @Test + public void testEncodeDecimal64() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setDecimal64(0, 2, 12345L); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DECIMAL64); + writer.write(NON_NULL); + writer.write(2); + writeLongLe(writer, 12345L); + })); + } + }); + } + + /** + * DECIMAL64 holds 18 digits of precision, so scale above 18 is + * mathematically invalid. Under the shared-cap regime a scale up to 76 + * was silently accepted; the per-width check rejects at the correct + * boundary. + */ + @Test + public void testEncodeDecimal64RejectsScaleAbove18() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + try { + binds.setDecimal64(0, 19, 1L); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue("message must mention DECIMAL64: " + expected.getMessage(), + expected.getMessage().contains("DECIMAL64")); + } + } + }); + } + + @Test + public void testEncodeDouble() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setDouble(0, 2.718281828); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DOUBLE); + writer.write(NON_NULL); + writeLongLe(writer, Double.doubleToRawLongBits(2.718281828)); + })); + } + try (QwpBindValues binds = new QwpBindValues()) { + binds.setDouble(0, Double.NaN); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DOUBLE); + writer.write(NON_NULL); + writeLongLe(writer, Double.doubleToRawLongBits(Double.NaN)); + })); + } + }); + } + + @Test + public void testEncodeFloat() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setFloat(0, 3.14f); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_FLOAT); + writer.write(NON_NULL); + writeIntLe(writer, Float.floatToRawIntBits(3.14f)); + })); + } + }); + } + + @Test + public void testEncodeGeohashMaxPrecision() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + long value = 0x0FFF_FFFF_FFFF_FFFFL; + binds.setGeohash(0, 60, value); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_GEOHASH); + writer.write(NON_NULL); + writeVarint(writer, 60); + for (int i = 0; i < 8; i++) { + writer.write((byte) (value >>> (i * 8))); + } + })); + } + }); + } + + @Test + public void testEncodeGeohashMinPrecision() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setGeohash(0, 1, 1L); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_GEOHASH); + writer.write(NON_NULL); + writeVarint(writer, 1); + writer.write((byte) 0x01); + })); + } + }); + } + + @Test + public void testEncodeGeohashRejectsPrecisionAboveMax() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + try { + binds.setGeohash(0, 61, 1L); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("precision")); + } + } + }); + } + + /** + * Regression: before the fix, {@code setGeohash} did not mask {@code value} + * to {@code precisionBits}. For a precision that is not a multiple of 8 + * (e.g. 5 bits, 1 byte on the wire) the top byte would leak whatever bits + * the caller happened to have set above bit 4, silently changing the geohash + * the server stored. The fix masks the high bits off before encoding. + */ + @Test + public void testEncodeGeohashMasksHighBitsForSubByteprecision() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + // precisionBits=5 -> 1 byte on the wire, 5 low bits significant. + // Bits above 4 must be masked away so 0xFF on the wire becomes 0x1F. + binds.setGeohash(0, 5, 0xFFL); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_GEOHASH); + writer.write(NON_NULL); + writeVarint(writer, 5); + writer.write((byte) 0x1F); + })); + } + }); + } + + /** + * Same regression at the 60-bit ceiling: the top nibble (bits 60-63) must + * never reach the wire, regardless of the caller-supplied {@code value}. + */ + @Test + public void testEncodeGeohashMasksHighBitsAtMaxPrecision() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + // All 64 bits set -- the encoder must zero bits 60..63. + binds.setGeohash(0, 60, -1L); + long expectedValue = (1L << 60) - 1L; + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_GEOHASH); + writer.write(NON_NULL); + writeVarint(writer, 60); + for (int i = 0; i < 8; i++) { + writer.write((byte) (expectedValue >>> (i * 8))); + } + })); + } + }); + } + + @Test + public void testEncodeGeohashRejectsPrecisionBelowMin() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + try { + binds.setGeohash(0, 0, 0L); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("precision")); + } + } + }); + } + + @Test + public void testEncodeInt() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setInt(0, Integer.MIN_VALUE); + binds.setInt(1, 0); + binds.setInt(2, Integer.MAX_VALUE); + byte[] want = expected(writer -> { + writer.write(QwpConstants.TYPE_INT); + writer.write(NON_NULL); + writeIntLe(writer, Integer.MIN_VALUE); + writer.write(QwpConstants.TYPE_INT); + writer.write(NON_NULL); + writeIntLe(writer, 0); + writer.write(QwpConstants.TYPE_INT); + writer.write(NON_NULL); + writeIntLe(writer, Integer.MAX_VALUE); + }); + assertEncoded(binds, 3, want); + } + }); + } + + @Test + public void testEncodeLong() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setLong(0, 42L); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_LONG); + writer.write(NON_NULL); + writeLongLe(writer, 42L); + })); + } + }); + } + + @Test + public void testEncodeLong256() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setLong256(0, 0x1111111111111111L, 0x2222222222222222L, 0x3333333333333333L, 0x4444444444444444L); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_LONG256); + writer.write(NON_NULL); + writeLongLe(writer, 0x1111111111111111L); + writeLongLe(writer, 0x2222222222222222L); + writeLongLe(writer, 0x3333333333333333L); + writeLongLe(writer, 0x4444444444444444L); + })); + } + }); + } + + @Test + public void testEncodeMultiBindMixedTypes() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setLong(0, 1234567890L); + binds.setVarchar(1, "hello"); + binds.setBoolean(2, true); + binds.setDouble(3, 1.5); + byte[] want = expected(writer -> { + writer.write(QwpConstants.TYPE_LONG); + writer.write(NON_NULL); + writeLongLe(writer, 1234567890L); + + writer.write(QwpConstants.TYPE_VARCHAR); + writer.write(NON_NULL); + writeIntLe(writer, 0); + writeIntLe(writer, 5); + byte[] bytes = "hello".getBytes(StandardCharsets.UTF_8); + for (byte b : bytes) { + writer.write(b); + } + + writer.write(QwpConstants.TYPE_BOOLEAN); + writer.write(NON_NULL); + writer.write(1); + + writer.write(QwpConstants.TYPE_DOUBLE); + writer.write(NON_NULL); + writeLongLe(writer, Double.doubleToRawLongBits(1.5)); + }); + assertEncoded(binds, 4, want); + } + }); + } + + @Test + public void testEncodeNullScalar() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setNull(0, QwpConstants.TYPE_LONG); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_LONG); + writer.write(NULL_FLAG); + writer.write(NULL_BITMAP); + })); + } + }); + } + + /** + * {@link QwpBindValues#setNullDecimal128(int, int)} explicitly names the + * scale, which survives to the wire even for a NULL value (the server + * treats scale as part of the bound variable's type). Pins the framing: + * type + null_flag + null_bitmap + scale. + */ + @Test + public void testEncodeNullDecimal128WithExplicitScale() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setNullDecimal128(0, 12); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DECIMAL128); + writer.write(NULL_FLAG); + writer.write(NULL_BITMAP); + writer.write(12); + })); + } + }); + } + + @Test + public void testEncodeNullDecimal256WithExplicitScale() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setNullDecimal256(0, 50); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DECIMAL256); + writer.write(NULL_FLAG); + writer.write(NULL_BITMAP); + writer.write(50); + })); + } + }); + } + + @Test + public void testEncodeNullDecimal64WithExplicitScale() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setNullDecimal64(0, 3); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_DECIMAL64); + writer.write(NULL_FLAG); + writer.write(NULL_BITMAP); + writer.write(3); + })); + } + }); + } + + /** + * {@link QwpBindValues#setNullGeohash(int, int)} pins the precision_bits + * even for NULL, since the server reads the varint unconditionally before + * inspecting the null flag. + */ + @Test + public void testEncodeNullGeohashWithExplicitPrecision() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setNullGeohash(0, 40); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_GEOHASH); + writer.write(NULL_FLAG); + writer.write(NULL_BITMAP); + writeVarint(writer, 40); + })); + } + }); + } + + @Test + public void testEncodeNullTypesExhaustive() throws Exception { + // DECIMAL64/128/256 and GEOHASH NULLs carry a trailing scale byte + // (or precision_bits varint) because the server reads those fields + // unconditionally. All other types emit only type + null flag + + // null bitmap. The exhaustive walk pins both shapes. + byte[] allTypes = { + QwpConstants.TYPE_BOOLEAN, + QwpConstants.TYPE_BYTE, + QwpConstants.TYPE_SHORT, + QwpConstants.TYPE_CHAR, + QwpConstants.TYPE_INT, + QwpConstants.TYPE_LONG, + QwpConstants.TYPE_FLOAT, + QwpConstants.TYPE_DOUBLE, + QwpConstants.TYPE_DATE, + QwpConstants.TYPE_TIMESTAMP, + QwpConstants.TYPE_TIMESTAMP_NANOS, + QwpConstants.TYPE_UUID, + QwpConstants.TYPE_LONG256, + QwpConstants.TYPE_GEOHASH, + QwpConstants.TYPE_VARCHAR, + QwpConstants.TYPE_DECIMAL64, + QwpConstants.TYPE_DECIMAL128, + QwpConstants.TYPE_DECIMAL256, + }; + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + for (int i = 0; i < allTypes.length; i++) { + binds.setNull(i, allTypes[i]); + } + Assert.assertEquals(allTypes.length, binds.count()); + byte[] got = readBuffer(binds); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (byte t : allTypes) { + out.write(t); + out.write(NULL_FLAG); + out.write(NULL_BITMAP); + if (t == QwpConstants.TYPE_DECIMAL64 + || t == QwpConstants.TYPE_DECIMAL128 + || t == QwpConstants.TYPE_DECIMAL256) { + // setNull defaults to scale 0 for decimals. + out.write(0); + } else if (t == QwpConstants.TYPE_GEOHASH) { + // setNull defaults to the minimum valid precision (1 bit), + // encoded as a single-byte varint. + out.write(1); + } + } + Assert.assertArrayEquals(out.toByteArray(), got); + } + }); + } + + @Test + public void testEncodeShort() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setShort(0, Short.MIN_VALUE); + binds.setShort(1, (short) 0); + binds.setShort(2, Short.MAX_VALUE); + byte[] want = expected(writer -> { + writer.write(QwpConstants.TYPE_SHORT); + writer.write(NON_NULL); + writeShortLe(writer, Short.MIN_VALUE); + writer.write(QwpConstants.TYPE_SHORT); + writer.write(NON_NULL); + writeShortLe(writer, (short) 0); + writer.write(QwpConstants.TYPE_SHORT); + writer.write(NON_NULL); + writeShortLe(writer, Short.MAX_VALUE); + }); + assertEncoded(binds, 3, want); + } + }); + } + + @Test + public void testEncodeTimestampMicros() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setTimestampMicros(0, 1_700_000_000_000_000L); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_TIMESTAMP); + writer.write(NON_NULL); + writeLongLe(writer, 1_700_000_000_000_000L); + })); + } + }); + } + + @Test + public void testEncodeTimestampNanos() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setTimestampNanos(0, 1_700_000_000_000_000_000L); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_TIMESTAMP_NANOS); + writer.write(NON_NULL); + writeLongLe(writer, 1_700_000_000_000_000_000L); + })); + } + }); + } + + @Test + public void testEncodeUuidConvenienceFromJavaUuid() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + UUID uuid = UUID.fromString("123e4567-e89b-12d3-a456-426614174000"); + binds.setUuid(0, uuid); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_UUID); + writer.write(NON_NULL); + writeLongLe(writer, uuid.getLeastSignificantBits()); + writeLongLe(writer, uuid.getMostSignificantBits()); + })); + } + }); + } + + @Test + public void testEncodeUuidExplicit() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setUuid(0, 0xFEEDFACECAFEBEEFL, 0x0BADF00DDEADBEEFL); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_UUID); + writer.write(NON_NULL); + writeLongLe(writer, 0xFEEDFACECAFEBEEFL); + writeLongLe(writer, 0x0BADF00DDEADBEEFL); + })); + } + }); + } + + @Test + public void testEncodeUuidNullJavaUuid() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setUuid(0, null); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_UUID); + writer.write(NULL_FLAG); + writer.write(NULL_BITMAP); + })); + } + }); + } + + @Test + public void testEncodeVarcharAscii() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setVarchar(0, "hello"); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_VARCHAR); + writer.write(NON_NULL); + writeIntLe(writer, 0); + writeIntLe(writer, 5); + for (byte b : "hello".getBytes(StandardCharsets.UTF_8)) { + writer.write(b); + } + })); + } + }); + } + + @Test + public void testEncodeVarcharEmpty() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setVarchar(0, ""); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_VARCHAR); + writer.write(NON_NULL); + writeIntLe(writer, 0); + writeIntLe(writer, 0); + })); + } + }); + } + + @Test + public void testEncodeVarcharNullShortcut() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setVarchar(0, null); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_VARCHAR); + writer.write(NULL_FLAG); + writer.write(NULL_BITMAP); + })); + } + }); + } + + @Test + public void testEncodeVarcharUnicode() throws Exception { + assertMemoryLeak(() -> { + String value = "café"; + byte[] utf8 = value.getBytes(StandardCharsets.UTF_8); + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setVarchar(0, value); + assertEncoded(binds, 1, expected(writer -> { + writer.write(QwpConstants.TYPE_VARCHAR); + writer.write(NON_NULL); + writeIntLe(writer, 0); + writeIntLe(writer, utf8.length); + for (byte b : utf8) { + writer.write(b); + } + })); + } + }); + }); + } + + @Test + public void testEncoderGrowsBufferBeyondDefault() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + String big = "x".repeat(20_000); + binds.setVarchar(0, big); + Assert.assertEquals(1, binds.count()); + // type(1) + flag(1) + offset0(4) + len(4) + 20000 bytes = 20010 + Assert.assertEquals(1 + 1 + 4 + 4 + 20_000, binds.bufferLen()); + } + }); + } + + @Test + public void testRejectsDuplicateIndex() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setLong(0, 1L); + try { + binds.setLong(0, 2L); + Assert.fail("expected IllegalStateException"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("index")); + } + } + }); + } + + @Test + public void testRejectsOutOfOrderIndex() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + binds.setLong(0, 1L); + try { + binds.setLong(2, 3L); + Assert.fail("expected IllegalStateException"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("index")); + } + } + }); + } + + @Test + public void testRejectsUnsupportedNullTypeCode() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + try { + binds.setNull(0, QwpConstants.TYPE_BINARY); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("bind type")); + } + } + }); + } + + @Test + public void testTooManyBinds() throws Exception { + assertMemoryLeak(() -> { + try (QwpBindValues binds = new QwpBindValues()) { + int max = QwpConstants.MAX_COLUMNS_PER_TABLE; + for (int i = 0; i < max; i++) { + binds.setInt(i, i); + } + try { + binds.setInt(max, max); + Assert.fail("expected IllegalStateException"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("too many")); + } + } + }); + } + + private static void assertEncoded(QwpBindValues binds, int expectedCount, byte[] expected) { + Assert.assertEquals(expectedCount, binds.count()); + Assert.assertArrayEquals(expected, readBuffer(binds)); + } + + private static byte[] expected(ExpectedWriter body) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + body.write(out); + return out.toByteArray(); + } + + private static byte[] readBuffer(QwpBindValues binds) { + long len = binds.bufferLen(); + byte[] out = new byte[(int) len]; + long ptr = binds.bufferPtr(); + for (int i = 0; i < out.length; i++) { + out[i] = Unsafe.getUnsafe().getByte(ptr + i); + } + return out; + } + + private static void writeIntLe(ByteArrayOutputStream out, int value) { + out.write(value & 0xFF); + out.write((value >>> 8) & 0xFF); + out.write((value >>> 16) & 0xFF); + out.write((value >>> 24) & 0xFF); + } + + private static void writeLongLe(ByteArrayOutputStream out, long value) { + for (int i = 0; i < 8; i++) { + out.write((int) ((value >>> (i * 8)) & 0xFF)); + } + } + + private static void writeShortLe(ByteArrayOutputStream out, short value) { + out.write(value & 0xFF); + out.write((value >>> 8) & 0xFF); + } + + private static void writeVarint(ByteArrayOutputStream out, long value) { + while (value > 0x7F) { + out.write((int) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + out.write((int) (value & 0x7F)); + } + + @FunctionalInterface + private interface ExpectedWriter { + void write(ByteArrayOutputStream out); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java new file mode 100644 index 00000000..153f5d84 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchHandlerTest.java @@ -0,0 +1,72 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import org.junit.Test; + +/** + * The two default methods on {@link QwpColumnBatchHandler} + * ({@link QwpColumnBatchHandler#onFailoverReset(QwpServerInfo) onFailoverReset} + * and {@link QwpColumnBatchHandler#onExecDone(short, long) onExecDone}) are + * documented as no-ops that handlers without explicit overrides can rely on. + * The QwpQueryClient calls them unconditionally; if the defaults ever started + * throwing we would silently break SELECT-only handlers and any handler that + * does not opt into failover-aware behaviour. + */ +public class QwpColumnBatchHandlerTest { + + @Test + public void testDefaultMethodsAreNoOpAndDoNotThrow() { + QwpColumnBatchHandler handler = new MinimalHandler(); + // Both callable with arbitrary args including null without raising: + handler.onFailoverReset(null); + handler.onFailoverReset(new QwpServerInfo((byte) 0, 0L, 0, 0L, "", "")); + handler.onExecDone((short) 0, 0L); + handler.onExecDone((short) 1, 12345L); + handler.onExecDone(Short.MAX_VALUE, Long.MAX_VALUE); + } + + /** + * Implements only the abstract methods. If onFailoverReset / onExecDone + * lost their {@code default} modifier this class would no longer compile, + * so this test doubles as a compile-time guard on the interface contract. + */ + private static final class MinimalHandler implements QwpColumnBatchHandler { + @Override + public void onBatch(QwpColumnBatch batch) { + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchViewsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchViewsTest.java new file mode 100644 index 00000000..697f3350 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnBatchViewsTest.java @@ -0,0 +1,1632 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.ColumnView; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.RowView; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.Long256Impl; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Uuid; +import io.questdb.client.std.bytes.DirectByteSequence; +import io.questdb.client.std.str.DirectUtf8Sequence; +import io.questdb.client.std.str.StringSink; +import io.questdb.client.std.str.Utf8s; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.List; + +/** + * White-box unit tests for {@link RowView} and {@link ColumnView}. + *

+ * Each test allocates native memory directly, populates a + * {@code QwpColumnLayout} via reflection (the layout fields are package-private + * on the production type; the production module is declared {@code open}, so + * reflective access works from the test module), registers it on a fresh + * {@link QwpColumnBatch}, and asserts the view's behaviour. All allocations + * are tracked in {@link #allocations} and freed in {@link #tearDown()}. + *

+ * The full wire-format round-trip is covered by + * {@code QwpEgressTypesExhaustiveTest} in the parent module; these tests pin + * the view-layer behaviour in isolation so a regression there points + * unambiguously at the views, not the decoder. + */ +public class QwpColumnBatchViewsTest { + + private static final String COLUMN_INFO = "io.questdb.client.cutlass.qwp.client.QwpEgressColumnInfo"; + private static final String COLUMN_LAYOUT = "io.questdb.client.cutlass.qwp.client.QwpColumnLayout"; + private final List allocations = new ArrayList<>(); + + @Before + public void setUp() { + allocations.clear(); + } + + @After + public void tearDown() { + for (long[] alloc : allocations) { + Unsafe.free(alloc[0], alloc[1], MemoryTag.NATIVE_DEFAULT); + } + allocations.clear(); + } + + @Test + public void testColumnViewArrayRowAddr() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + Object l = setupArrayColumnLayout(batch, + new boolean[]{false, true, false}, + new double[][]{{1.0, 2.0}, null, {3.0, 4.0}}); + ColumnView col = batch.column(0); + Assert.assertNotEquals(0L, col.arrayRowAddr(0)); + Assert.assertEquals(0L, col.arrayRowAddr(1)); // NULL row + Assert.assertNotEquals(0L, col.arrayRowAddr(2)); + Assert.assertEquals(1, col.getArrayNDims(0)); + Assert.assertEquals(0, col.getArrayNDims(1)); // NULL row -> 0 + long[] perRow = (long[]) readField(l, "arrayRowAddr"); + Assert.assertEquals(perRow[2], col.arrayRowAddr(2)); + }); + } + + @Test + public void testColumnViewBatchAccessorReturnsParent() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 1); + setupLongColumnLayout(batch, 0, "x", new long[]{42L}, new boolean[]{false}); + Assert.assertSame(batch, batch.column(0).batch()); + }); + } + + @Test + public void testColumnViewBinaryAccessors() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupBinaryColumnLayout(batch, + new byte[][]{{0x00, 0x7F, (byte) 0xFF}, null, {0x01}}, + new boolean[]{false, true, false}); + ColumnView col = batch.column(0); + // heap-allocating variant + byte[] copy = col.getBinary(0); + Assert.assertArrayEquals(new byte[]{0x00, 0x7F, (byte) 0xFF}, copy); + Assert.assertNull(col.getBinary(1)); + // zero-alloc A view + DirectByteSequence a = col.getBinaryA(0); + Assert.assertNotNull(a); + Assert.assertEquals(3, a.size()); + Assert.assertEquals((byte) 0xFF, a.byteAt(2)); + // B view reads independently of A (dual slots) + DirectByteSequence b = col.getBinaryB(2); + Assert.assertNotNull(b); + Assert.assertEquals(1, b.size()); + Assert.assertEquals((byte) 0x01, b.byteAt(0)); + Assert.assertNull(col.getBinaryB(1)); + }); + } + + @Test + public void testColumnViewBoolValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 5); + setupBooleanColumnLayout(batch, 0, + new boolean[]{true, false, true, true, false}, + new boolean[]{false, false, false, false, false}); + ColumnView col = batch.column(0); + Assert.assertTrue(col.getBoolValue(0)); + Assert.assertFalse(col.getBoolValue(1)); + Assert.assertTrue(col.getBoolValue(2)); + Assert.assertTrue(col.getBoolValue(3)); + Assert.assertFalse(col.getBoolValue(4)); + Assert.assertEquals(0, col.bytesPerValue()); + }); + } + + @Test + public void testColumnViewByteValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 4); + setupByteColumnLayout(batch, 0, + new byte[]{Byte.MIN_VALUE, -1, 0, Byte.MAX_VALUE}, + new boolean[]{false, false, false, false}); + ColumnView col = batch.column(0); + Assert.assertEquals(Byte.MIN_VALUE, col.getByteValue(0)); + Assert.assertEquals(-1, col.getByteValue(1)); + Assert.assertEquals(0, col.getByteValue(2)); + Assert.assertEquals(Byte.MAX_VALUE, col.getByteValue(3)); + }); + } + + @Test + public void testColumnViewBytesPerValuePerType() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(8, 1); + setupLongColumnLayout(batch, 0, "l", new long[]{0}, new boolean[]{false}); + setupIntColumnLayout(batch, 1, new int[]{0}, new boolean[]{false}); + setupDoubleColumnLayout(batch, 2, new double[]{0.0}, new boolean[]{false}); + setupFloatColumnLayout(batch, 3, new float[]{0.0f}, new boolean[]{false}); + setupBooleanColumnLayout(batch, 4, new boolean[]{false}, new boolean[]{false}); + setupByteColumnLayout(batch, 5, new byte[]{0}, new boolean[]{false}); + setupShortColumnLayout(batch, 6, new short[]{0}, new boolean[]{false}); + setupVarcharColumnLayout(batch, 7, "v", new String[]{""}, new boolean[]{false}); + + Assert.assertEquals(8, batch.column(0).bytesPerValue()); + Assert.assertEquals(4, batch.column(1).bytesPerValue()); + Assert.assertEquals(8, batch.column(2).bytesPerValue()); + Assert.assertEquals(4, batch.column(3).bytesPerValue()); + Assert.assertEquals(0, batch.column(4).bytesPerValue()); + Assert.assertEquals(1, batch.column(5).bytesPerValue()); + Assert.assertEquals(2, batch.column(6).bytesPerValue()); + Assert.assertEquals(-1, batch.column(7).bytesPerValue()); + }); + } + + @Test + public void testColumnViewCachedPerColumnIndex() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(2, 1); + setupLongColumnLayout(batch, 0, "a", new long[]{1L}, new boolean[]{false}); + setupLongColumnLayout(batch, 1, "b", new long[]{2L}, new boolean[]{false}); + + ColumnView a1 = batch.column(0); + ColumnView b1 = batch.column(1); + ColumnView a2 = batch.column(0); + ColumnView b2 = batch.column(1); + + Assert.assertNotSame(a1, b1); + Assert.assertSame(a1, a2); + Assert.assertSame(b1, b2); + Assert.assertEquals(0, a1.getColumnIndex()); + Assert.assertEquals(1, b1.getColumnIndex()); + Assert.assertEquals(1L, a1.getLongValue(0)); + Assert.assertEquals(2L, b1.getLongValue(0)); + }); + } + + @Test + public void testColumnViewCharValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupCharColumnLayout(batch, 0, + new char[]{'A', 'z', '0'}, + new boolean[]{false, false, false}); + ColumnView col = batch.column(0); + Assert.assertEquals('A', col.getCharValue(0)); + Assert.assertEquals('z', col.getCharValue(1)); + Assert.assertEquals('0', col.getCharValue(2)); + }); + } + + @Test + public void testColumnViewDecimal128Accessors() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + long[] lo = {0xFFEE_DDCC_BBAA_9988L, 0L, 0x1L}; + long[] hi = {0x1122_3344_5566_7788L, 0L, 0x2L}; + setupDecimal128ColumnLayout(batch, lo, hi, new boolean[]{false, true, false}, 6); + ColumnView col = batch.column(0); + Assert.assertEquals(0xFFEE_DDCC_BBAA_9988L, col.getDecimal128Low(0)); + Assert.assertEquals(0x1122_3344_5566_7788L, col.getDecimal128High(0)); + Assert.assertEquals(0L, col.getDecimal128Low(1)); // NULL -> 0 + Assert.assertEquals(0L, col.getDecimal128High(1)); + Assert.assertEquals(0x1L, col.getDecimal128Low(2)); + Assert.assertEquals(0x2L, col.getDecimal128High(2)); + }); + } + + @Test + public void testColumnViewDelegatesAgreeWithBatchPrimitives() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(5, 4); + setupLongColumnLayout(batch, 0, "l", new long[]{1L, 2L, 0L, 4L}, new boolean[]{false, false, true, false}); + setupIntColumnLayout(batch, 1, new int[]{10, 20, 0, 40}, new boolean[]{false, false, true, false}); + setupDoubleColumnLayout(batch, 2, new double[]{1.5, 2.5, 0.0, 4.5}, new boolean[]{false, false, true, false}); + setupBooleanColumnLayout(batch, 3, new boolean[]{true, false, false, true}, new boolean[]{false, false, true, false}); + setupVarcharColumnLayout(batch, 4, "s", new String[]{"a", "bb", null, "dddd"}, new boolean[]{false, false, true, false}); + + for (int c = 0; c < 5; c++) { + ColumnView col = batch.column(c); + for (int r = 0; r < 4; r++) { + Assert.assertEquals("isNull col=" + c + " row=" + r, batch.isNull(c, r), col.isNull(r)); + } + } + for (int r = 0; r < 4; r++) { + Assert.assertEquals(batch.getLongValue(0, r), batch.column(0).getLongValue(r)); + Assert.assertEquals(batch.getIntValue(1, r), batch.column(1).getIntValue(r)); + Assert.assertEquals(batch.getDoubleValue(2, r), batch.column(2).getDoubleValue(r), 0.0); + Assert.assertEquals(batch.getBoolValue(3, r), batch.column(3).getBoolValue(r)); + DirectUtf8Sequence batchStr = batch.getStrA(4, r); + DirectUtf8Sequence colStr = batch.column(4).getStrA(r); + if (batchStr == null) { + Assert.assertNull("row " + r + ": column-view returned non-null for NULL row", colStr); + } else { + Assert.assertNotNull("row " + r + ": column-view returned null for non-NULL row", colStr); + Assert.assertEquals(Utf8s.stringFromUtf8Bytes(batchStr), Utf8s.stringFromUtf8Bytes(colStr)); + } + } + }); + } + + @Test + public void testColumnViewDoubleArrayElements() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupArrayColumnLayout(batch, + new boolean[]{false, true, false}, + new double[][]{{1.0, 2.0, 3.0}, null, {4.5, 5.5}}); + ColumnView col = batch.column(0); + Assert.assertArrayEquals(new double[]{1.0, 2.0, 3.0}, col.getDoubleArrayElements(0), 0.0); + Assert.assertNull(col.getDoubleArrayElements(1)); + Assert.assertArrayEquals(new double[]{4.5, 5.5}, col.getDoubleArrayElements(2), 0.0); + }); + } + + @Test + public void testColumnViewDoubleValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 4); + setupDoubleColumnLayout(batch, 0, + new double[]{1.5, -1.5, 0.0, Double.MAX_VALUE}, + new boolean[]{false, false, false, false}); + ColumnView col = batch.column(0); + Assert.assertEquals(1.5, col.getDoubleValue(0), 0.0); + Assert.assertEquals(-1.5, col.getDoubleValue(1), 0.0); + Assert.assertEquals(0.0, col.getDoubleValue(2), 0.0); + Assert.assertEquals(Double.MAX_VALUE, col.getDoubleValue(3), 0.0); + }); + } + + @Test + public void testColumnViewFloatValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupFloatColumnLayout(batch, 0, + new float[]{1.5f, -1.5f, 0.0f}, + new boolean[]{false, false, false}); + ColumnView col = batch.column(0); + Assert.assertEquals(1.5f, col.getFloatValue(0), 0.0f); + Assert.assertEquals(-1.5f, col.getFloatValue(1), 0.0f); + Assert.assertEquals(0.0f, col.getFloatValue(2), 0.0f); + }); + } + + @Test + public void testColumnViewGeohashValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(3, 2); + setupGeohashColumnLayout(batch, 0, "g20", new long[]{0xABCDEL, 0L}, new boolean[]{false, true}, 20); + setupGeohashColumnLayout(batch, 1, "g40", new long[]{0x12345_6789AL, 0L}, new boolean[]{false, true}, 40); + setupGeohashColumnLayout(batch, 2, "g60", new long[]{0x0FFFF_FFFF_FFFF_FFFL, 0L}, new boolean[]{false, true}, 60); + + Assert.assertEquals(0xABCDEL, batch.column(0).getGeohashValue(0)); + Assert.assertEquals(0x12345_6789AL, batch.column(1).getGeohashValue(0)); + Assert.assertEquals(0x0FFFF_FFFF_FFFF_FFFL, batch.column(2).getGeohashValue(0)); + // NULL rows return 0. + Assert.assertEquals(0L, batch.column(0).getGeohashValue(1)); + Assert.assertEquals(0L, batch.column(1).getGeohashValue(1)); + Assert.assertEquals(0L, batch.column(2).getGeohashValue(1)); + // Stride mirrors (precisionBits + 7) / 8. + Assert.assertEquals(3, batch.column(0).bytesPerValue()); // 20 bits -> 3 bytes + Assert.assertEquals(5, batch.column(1).bytesPerValue()); // 40 bits -> 5 bytes + Assert.assertEquals(8, batch.column(2).bytesPerValue()); // 60 bits -> 8 bytes + }); + } + + @Test + public void testColumnViewGetColumnIndex() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(3, 1); + setupLongColumnLayout(batch, 0, "a", new long[]{0}, new boolean[]{false}); + setupLongColumnLayout(batch, 1, "b", new long[]{0}, new boolean[]{false}); + setupLongColumnLayout(batch, 2, "c", new long[]{0}, new boolean[]{false}); + Assert.assertEquals(0, batch.column(0).getColumnIndex()); + Assert.assertEquals(1, batch.column(1).getColumnIndex()); + Assert.assertEquals(2, batch.column(2).getColumnIndex()); + }); + } + + @Test + public void testColumnViewGetColumnWireType() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(2, 1); + setupLongColumnLayout(batch, 0, "l", new long[]{0}, new boolean[]{false}); + setupVarcharColumnLayout(batch, 1, "s", new String[]{""}, new boolean[]{false}); + Assert.assertEquals(QwpConstants.TYPE_LONG, batch.column(0).getColumnWireType()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, batch.column(1).getColumnWireType()); + }); + } + + @Test + public void testColumnViewIntValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 4); + setupIntColumnLayout(batch, 0, + new int[]{Integer.MIN_VALUE + 1, -1, 0, Integer.MAX_VALUE}, + new boolean[]{false, false, false, false}); + ColumnView col = batch.column(0); + Assert.assertEquals(Integer.MIN_VALUE + 1, col.getIntValue(0)); + Assert.assertEquals(-1, col.getIntValue(1)); + Assert.assertEquals(0, col.getIntValue(2)); + Assert.assertEquals(Integer.MAX_VALUE, col.getIntValue(3)); + }); + } + + @Test + public void testColumnViewLong256AndLong256Word() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + long[][] words = {{0xAAAAL, 0xBBBBL, 0xCCCCL, 0xDDDDL}, {0L, 0L, 0L, 0L}}; + setupLong256ColumnLayout(batch, words, new boolean[]{false, true}); + ColumnView col = batch.column(0); + + // Sink-based bulk read + Long256Impl sink = new Long256Impl(); + Assert.assertTrue(col.getLong256(0, sink)); + Assert.assertEquals(0xAAAAL, sink.getLong0()); + Assert.assertEquals(0xBBBBL, sink.getLong1()); + Assert.assertEquals(0xCCCCL, sink.getLong2()); + Assert.assertEquals(0xDDDDL, sink.getLong3()); + Assert.assertFalse("NULL returns false", col.getLong256(1, sink)); + + // Per-word read + Assert.assertEquals(0xAAAAL, col.getLong256Word(0, 0)); + Assert.assertEquals(0xBBBBL, col.getLong256Word(0, 1)); + Assert.assertEquals(0xCCCCL, col.getLong256Word(0, 2)); + Assert.assertEquals(0xDDDDL, col.getLong256Word(0, 3)); + Assert.assertEquals(0L, col.getLong256Word(1, 0)); // NULL -> 0 + }); + } + + @Test + public void testColumnViewLongValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 4); + setupLongColumnLayout(batch, 0, "l", + new long[]{Long.MIN_VALUE + 1, -1L, 0L, Long.MAX_VALUE}, + new boolean[]{false, false, false, false}); + ColumnView col = batch.column(0); + Assert.assertEquals(Long.MIN_VALUE + 1, col.getLongValue(0)); + Assert.assertEquals(-1L, col.getLongValue(1)); + Assert.assertEquals(0L, col.getLongValue(2)); + Assert.assertEquals(Long.MAX_VALUE, col.getLongValue(3)); + }); + } + + @Test + public void testColumnViewNonNullCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 5); + setupLongColumnLayout(batch, 0, "l", + new long[]{1L, 2L, 0L, 4L, 0L}, + new boolean[]{false, false, true, false, true}); + Assert.assertEquals(3, batch.column(0).nonNullCount()); + }); + } + + @Test + public void testColumnViewNonNullIndex() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 5); + // Rows 1 and 3 are NULL; dense indices for non-null rows are 0, 1, 2. + setupLongColumnLayout(batch, 0, "l", + new long[]{10L, 0L, 20L, 0L, 30L}, + new boolean[]{false, true, false, true, false}); + ColumnView col = batch.column(0); + Assert.assertEquals(0, col.nonNullIndex(0)); + Assert.assertEquals(1, col.nonNullIndex(2)); + Assert.assertEquals(2, col.nonNullIndex(4)); + }); + } + + @Test + public void testColumnViewNonNullIndexNoNulls() throws Exception { + // When there are no nulls, dense index equals row index (layout skips the + // nonNullIdx fill; the method just returns the row back). + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupLongColumnLayout(batch, 0, "l", + new long[]{1L, 2L, 3L}, + new boolean[]{false, false, false}); + ColumnView col = batch.column(0); + Assert.assertEquals(0, col.nonNullIndex(0)); + Assert.assertEquals(1, col.nonNullIndex(1)); + Assert.assertEquals(2, col.nonNullIndex(2)); + }); + } + + @Test + public void testColumnViewNullBitmapAddrNoNulls() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupLongColumnLayout(batch, 0, "l", + new long[]{1L, 2L, 3L}, + new boolean[]{false, false, false}); + Assert.assertEquals(0L, batch.column(0).nullBitmapAddr()); + }); + } + + @Test + public void testColumnViewNullBitmapAddrWithNulls() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 5); + setupLongColumnLayout(batch, 0, "l", + new long[]{1L, 0L, 3L, 0L, 5L}, + new boolean[]{false, true, false, true, false}); + ColumnView col = batch.column(0); + long bm = col.nullBitmapAddr(); + Assert.assertNotEquals(0L, bm); + byte b = Unsafe.getUnsafe().getByte(bm); + Assert.assertEquals(0, b & 1); + Assert.assertEquals(2, b & 2); + Assert.assertEquals(0, b & 4); + Assert.assertEquals(8, b & 8); + Assert.assertEquals(0, b & 16); + }); + } + + @Test + public void testColumnViewNullValuesReturnTypeSentinels() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(6, 1); + setupLongColumnLayout(batch, 0, "l", new long[]{0L}, new boolean[]{true}); + setupIntColumnLayout(batch, 1, new int[]{0}, new boolean[]{true}); + setupDoubleColumnLayout(batch, 2, new double[]{0.0}, new boolean[]{true}); + setupFloatColumnLayout(batch, 3, new float[]{0.0f}, new boolean[]{true}); + setupBooleanColumnLayout(batch, 4, new boolean[]{false}, new boolean[]{true}); + setupVarcharColumnLayout(batch, 5, "s", new String[]{null}, new boolean[]{true}); + + Assert.assertEquals(0L, batch.column(0).getLongValue(0)); + Assert.assertEquals(0, batch.column(1).getIntValue(0)); + Assert.assertTrue(Double.isNaN(batch.column(2).getDoubleValue(0))); + Assert.assertTrue(Float.isNaN(batch.column(3).getFloatValue(0))); + Assert.assertFalse(batch.column(4).getBoolValue(0)); + Assert.assertNull(batch.column(5).getStrA(0)); + for (int c = 0; c < 6; c++) { + Assert.assertTrue("col " + c + " row 0 must be null", batch.column(c).isNull(0)); + } + }); + } + + @Test + public void testColumnViewOfReturnsThis() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(2, 1); + setupLongColumnLayout(batch, 0, "a", new long[]{1L}, new boolean[]{false}); + setupLongColumnLayout(batch, 1, "b", new long[]{2L}, new boolean[]{false}); + ColumnView v = batch.column(0); + Assert.assertSame(v, v.of(1)); + Assert.assertEquals(1, v.getColumnIndex()); + Assert.assertEquals(2L, v.getLongValue(0)); + }); + } + + @Test + public void testColumnViewRebindingPicksUpFreshLayout() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + setupLongColumnLayout(batch, 0, "l", new long[]{1L, 2L}, new boolean[]{false, false}); + ColumnView col = batch.column(0); + Assert.assertEquals(1L, col.getLongValue(0)); + + // Swap in a fresh layout, as the decoder would on the next batch. + Object freshLayout = newLayoutInstance(); + Object info = newColumnInfo("l", QwpConstants.TYPE_LONG); + writeField(freshLayout, "info", info); + long addr = allocate(16); + Unsafe.getUnsafe().putLong(addr, 99L); + Unsafe.getUnsafe().putLong(addr + 8, 100L); + writeField(freshLayout, "valuesAddr", addr); + writeField(freshLayout, "nonNullCount", 2); + writeField(freshLayout, "nullBitmapAddr", 0L); + writeField(freshLayout, "nonNullIdx", null); + setLayoutInBatch(batch, 0, freshLayout); + + ColumnView refreshed = batch.column(0); + Assert.assertSame(col, refreshed); + Assert.assertEquals(99L, refreshed.getLongValue(0)); + Assert.assertEquals(100L, refreshed.getLongValue(1)); + }); + } + + @Test + public void testColumnViewShortValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 4); + setupShortColumnLayout(batch, 0, + new short[]{Short.MIN_VALUE + 1, -1, 0, Short.MAX_VALUE}, + new boolean[]{false, false, false, false}); + ColumnView col = batch.column(0); + Assert.assertEquals(Short.MIN_VALUE + 1, col.getShortValue(0)); + Assert.assertEquals(-1, col.getShortValue(1)); + Assert.assertEquals(0, col.getShortValue(2)); + Assert.assertEquals(Short.MAX_VALUE, col.getShortValue(3)); + }); + } + + @Test + public void testColumnViewStrBDualHold() throws Exception { + // strA and strB are independent slots; a call to strB must not invalidate + // an already-obtained strA view, and vice-versa. + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupVarcharColumnLayout(batch, 0, "s", + new String[]{"alpha", "beta", null}, + new boolean[]{false, false, true}); + ColumnView col = batch.column(0); + DirectUtf8Sequence a = col.getStrA(0); + DirectUtf8Sequence b = col.getStrB(1); + Assert.assertNotNull(a); + Assert.assertNotNull(b); + // Both must read back their original bindings. + Assert.assertEquals("alpha", Utf8s.stringFromUtf8Bytes(a)); + Assert.assertEquals("beta", Utf8s.stringFromUtf8Bytes(b)); + Assert.assertNull(col.getStrB(2)); + }); + } + + @Test + public void testColumnViewStringHeapAllocated() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupVarcharColumnLayout(batch, 0, "s", + new String[]{"alpha", null, "gamma"}, + new boolean[]{false, true, false}); + ColumnView col = batch.column(0); + Assert.assertEquals("alpha", col.getString(0)); + Assert.assertNull(col.getString(1)); + Assert.assertEquals("gamma", col.getString(2)); + }); + } + + @Test + public void testColumnViewStringSink() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupVarcharColumnLayout(batch, 0, "s", + new String[]{"alpha", null, "gamma"}, + new boolean[]{false, true, false}); + ColumnView col = batch.column(0); + StringSink sink = new StringSink(); + + Assert.assertTrue(col.getString(0, sink)); + Assert.assertEquals("alpha", sink.toString()); + sink.clear(); + + Assert.assertFalse(col.getString(1, sink)); + Assert.assertEquals(0, sink.length()); + + Assert.assertTrue(col.getString(2, sink)); + Assert.assertEquals("gamma", sink.toString()); + }); + } + + @Test + public void testColumnViewSymbolAccessors() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 5); + String[] dict = {"AAPL", "MSFT", "GOOG"}; + int[] rowIds = {0, 1, 0, 2, -1}; + setupSymbolColumnLayout(batch, dict, rowIds); + + ColumnView col = batch.column(0); + Assert.assertEquals(3, col.symbolDictSize()); + Assert.assertNotEquals(0L, col.symbolDictHeapAddr()); + Assert.assertNotEquals(0L, col.symbolDictEntriesAddr()); + Assert.assertSame(rowIds, col.symbolRowIds()); + + Assert.assertEquals("AAPL", col.getSymbol(0)); + Assert.assertEquals("MSFT", col.getSymbol(1)); + Assert.assertEquals("AAPL", col.getSymbol(2)); + Assert.assertEquals("GOOG", col.getSymbol(3)); + Assert.assertNull(col.getSymbol(4)); + + Assert.assertEquals(0, col.getSymbolId(0)); + Assert.assertEquals(1, col.getSymbolId(1)); + Assert.assertEquals(0, col.getSymbolId(2)); + Assert.assertEquals(2, col.getSymbolId(3)); + Assert.assertEquals(-1, col.getSymbolId(4)); + + Assert.assertSame(col.getSymbol(0), col.getSymbol(2)); + }); + } + + @Test + public void testColumnViewUuidLoHi() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + long[] lo = {0xCAFE_BABEL, 0L}; + long[] hi = {0xDEAD_BEEFL, 0L}; + setupUuidColumnLayout(batch, lo, hi, new boolean[]{false, true}); + ColumnView col = batch.column(0); + Assert.assertEquals(0xCAFE_BABEL, col.getUuidLo(0)); + Assert.assertEquals(0xDEAD_BEEFL, col.getUuidHi(0)); + Assert.assertEquals(0L, col.getUuidLo(1)); // NULL + Assert.assertEquals(0L, col.getUuidHi(1)); + }); + } + + @Test + public void testColumnViewUuidWithSink() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + long[] lo = {0x1111_1111_1111_1111L, 0L}; + long[] hi = {0x2222_2222_2222_2222L, 0L}; + setupUuidColumnLayout(batch, lo, hi, new boolean[]{false, true}); + ColumnView col = batch.column(0); + Uuid sink = new Uuid(); + Assert.assertTrue(col.getUuid(0, sink)); + Assert.assertEquals(0x1111_1111_1111_1111L, sink.getLo()); + Assert.assertEquals(0x2222_2222_2222_2222L, sink.getHi()); + Assert.assertFalse(col.getUuid(1, sink)); + }); + } + + @Test + public void testColumnViewValuesAddrMatchesLayout() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(2, 1); + Object lLayout = setupLongColumnLayout(batch, 0, "l", new long[]{1L}, new boolean[]{false}); + Object dLayout = setupDoubleColumnLayout(batch, 1, new double[]{2.0}, new boolean[]{false}); + Assert.assertEquals((long) readField(lLayout, "valuesAddr"), batch.column(0).valuesAddr()); + Assert.assertEquals((long) readField(dLayout, "valuesAddr"), batch.column(1).valuesAddr()); + }); + } + + @Test + public void testColumnViewVarcharAndStringBytesAddr() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupVarcharColumnLayout(batch, 0, "v", + new String[]{"hello", "world", null}, + new boolean[]{false, false, true}); + ColumnView col = batch.column(0); + Assert.assertNotEquals(0L, col.valuesAddr()); + Assert.assertNotEquals(0L, col.stringBytesAddr()); + // getStrA re-points the same view -- read it before reading row 1. + Assert.assertEquals("hello", Utf8s.stringFromUtf8Bytes(col.getStrA(0))); + Assert.assertEquals("world", Utf8s.stringFromUtf8Bytes(col.getStrA(1))); + Assert.assertNull(col.getStrA(2)); + Assert.assertEquals(2, col.nonNullCount()); + }); + } + + @Test + public void testForEachRowEmptyBatch() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 0); + // Register a minimal layout so column() doesn't trip on null, though + // forEachRow never reaches into it. + Object l = newLayoutInstance(); + writeField(l, "info", newColumnInfo("l", QwpConstants.TYPE_LONG)); + writeField(l, "valuesAddr", 0L); + writeField(l, "nullBitmapAddr", 0L); + writeField(l, "nonNullCount", 0); + setLayoutInBatch(batch, 0, l); + + int[] callbackCount = {0}; + batch.forEachRow(row -> callbackCount[0]++); + Assert.assertEquals(0, callbackCount[0]); + }); + } + + @Test + public void testForEachRowExceptionPropagates() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 5); + setupLongColumnLayout(batch, 0, "l", + new long[]{1L, 2L, 3L, 4L, 5L}, + new boolean[]{false, false, false, false, false}); + int[] visited = {0}; + try { + batch.forEachRow(row -> { + visited[0]++; + if (row.getRowIndex() == 2) throw new RuntimeException("boom"); + }); + Assert.fail("expected RuntimeException to propagate"); + } catch (RuntimeException expected) { + Assert.assertEquals("boom", expected.getMessage()); + } + Assert.assertEquals(3, visited[0]); + }); + } + + @Test + public void testForEachRowReusesSameInstance() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 4); + setupLongColumnLayout(batch, 0, "l", + new long[]{1L, 2L, 3L, 4L}, + new boolean[]{false, false, false, false}); + IdentityHashMap seen = new IdentityHashMap<>(); + batch.forEachRow(row -> seen.put(row, Boolean.TRUE)); + Assert.assertEquals(1, seen.size()); + }); + } + + @Test + public void testForEachRowVisitsRowsInOrder() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 5); + setupLongColumnLayout(batch, 0, "l", + new long[]{10L, 20L, 30L, 40L, 50L}, + new boolean[]{false, false, false, false, false}); + List values = new ArrayList<>(); + int[] indices = new int[5]; + int[] cursor = {0}; + batch.forEachRow(row -> { + indices[cursor[0]++] = row.getRowIndex(); + values.add(row.getLongValue(0)); + }); + Assert.assertEquals(5, values.size()); + Assert.assertArrayEquals(new int[]{0, 1, 2, 3, 4}, indices); + Assert.assertEquals(Long.valueOf(10L), values.get(0)); + Assert.assertEquals(Long.valueOf(50L), values.get(4)); + }); + } + + @Test + public void testRowViewArrayAccessors() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupArrayColumnLayout(batch, + new boolean[]{false, true, false}, + new double[][]{{1.0, 2.0}, null, {3.0, 4.0, 5.0}}); + Assert.assertEquals(1, batch.row(0).getArrayNDims(0)); + Assert.assertEquals(0, batch.row(1).getArrayNDims(0)); // NULL + Assert.assertArrayEquals(new double[]{1.0, 2.0}, batch.row(0).getDoubleArrayElements(0), 0.0); + Assert.assertNull(batch.row(1).getDoubleArrayElements(0)); + Assert.assertArrayEquals(new double[]{3.0, 4.0, 5.0}, batch.row(2).getDoubleArrayElements(0), 0.0); + }); + } + + @Test + public void testRowViewBatchAccessor() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 1); + setupLongColumnLayout(batch, 0, "x", new long[]{42L}, new boolean[]{false}); + Assert.assertSame(batch, batch.row(0).batch()); + }); + } + + @Test + public void testRowViewBinaryAccessor() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + setupBinaryColumnLayout(batch, + new byte[][]{{0x00, 0x7F, (byte) 0xFF}, null}, + new boolean[]{false, true}); + RowView row = batch.row(0); + DirectByteSequence v0 = row.getBinaryA(0); + Assert.assertNotNull(v0); + Assert.assertEquals(3, v0.size()); + Assert.assertEquals((byte) 0x00, v0.byteAt(0)); + Assert.assertEquals((byte) 0x7F, v0.byteAt(1)); + Assert.assertEquals((byte) 0xFF, v0.byteAt(2)); + Assert.assertNull(batch.row(1).getBinaryA(0)); + byte[] copy = batch.row(0).getBinary(0); + Assert.assertArrayEquals(new byte[]{0x00, 0x7F, (byte) 0xFF}, copy); + }); + } + + @Test + public void testRowViewBinaryBDualHold() throws Exception { + // binaryA and binaryB are independent slots, parallel to strA/strB. + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + setupBinaryColumnLayout(batch, + new byte[][]{{0x01, 0x02}, {(byte) 0xFE, (byte) 0xFF}}, + new boolean[]{false, false}); + DirectByteSequence a = batch.row(0).getBinaryA(0); + DirectByteSequence b = batch.row(1).getBinaryB(0); + Assert.assertNotNull(a); + Assert.assertNotNull(b); + // Both stay valid concurrently. + Assert.assertEquals(2, a.size()); + Assert.assertEquals((byte) 0x01, a.byteAt(0)); + Assert.assertEquals(2, b.size()); + Assert.assertEquals((byte) 0xFF, b.byteAt(1)); + }); + } + + @Test + public void testRowViewByteAndShortAndCharAndFloat() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(4, 2); + setupByteColumnLayout(batch, 0, new byte[]{(byte) 127, 0}, new boolean[]{false, true}); + setupShortColumnLayout(batch, 1, new short[]{(short) -32000, 0}, new boolean[]{false, true}); + setupCharColumnLayout(batch, 2, new char[]{'Q', 0}, new boolean[]{false, true}); + setupFloatColumnLayout(batch, 3, new float[]{3.25f, 0f}, new boolean[]{false, true}); + + RowView row0 = batch.row(0); + Assert.assertEquals((byte) 127, row0.getByteValue(0)); + Assert.assertEquals((short) -32000, row0.getShortValue(1)); + Assert.assertEquals('Q', row0.getCharValue(2)); + Assert.assertEquals(3.25f, row0.getFloatValue(3), 0f); + + // NULL row: type-specific sentinels via the row facade. + RowView row1 = batch.row(1); + Assert.assertEquals(0, row1.getByteValue(0)); + Assert.assertEquals(0, row1.getShortValue(1)); + Assert.assertEquals(0, row1.getCharValue(2)); + Assert.assertTrue(Float.isNaN(row1.getFloatValue(3))); + }); + } + + @Test + public void testRowViewDecimal128() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + long[] lo = {0x1122_3344_5566_7788L, 0L}; + long[] hi = {0x99AA_BBCC_DDEE_FF00L, 0L}; + setupDecimal128ColumnLayout(batch, lo, hi, new boolean[]{false, true}, 4); + Assert.assertEquals(0x1122_3344_5566_7788L, batch.row(0).getDecimal128Low(0)); + Assert.assertEquals(0x99AA_BBCC_DDEE_FF00L, batch.row(0).getDecimal128High(0)); + Assert.assertEquals(0L, batch.row(1).getDecimal128Low(0)); + Assert.assertEquals(0L, batch.row(1).getDecimal128High(0)); + }); + } + + @Test + public void testRowViewDelegatesAgreeWithBatchPrimitives() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(5, 4); + setupLongColumnLayout(batch, 0, "l", new long[]{1L, 2L, 0L, 4L}, new boolean[]{false, false, true, false}); + setupIntColumnLayout(batch, 1, new int[]{10, 20, 0, 40}, new boolean[]{false, false, true, false}); + setupDoubleColumnLayout(batch, 2, new double[]{1.5, 2.5, 0.0, 4.5}, new boolean[]{false, false, true, false}); + setupBooleanColumnLayout(batch, 3, new boolean[]{true, false, false, true}, new boolean[]{false, false, true, false}); + setupVarcharColumnLayout(batch, 4, "s", new String[]{"a", "bb", null, "dddd"}, new boolean[]{false, false, true, false}); + + for (int r = 0; r < 4; r++) { + RowView row = batch.row(r); + for (int c = 0; c < 5; c++) { + Assert.assertEquals("isNull(c=" + c + ", r=" + r + ")", batch.isNull(c, r), row.isNull(c)); + } + Assert.assertEquals(batch.getLongValue(0, r), row.getLongValue(0)); + Assert.assertEquals(batch.getIntValue(1, r), row.getIntValue(1)); + Assert.assertEquals(batch.getDoubleValue(2, r), row.getDoubleValue(2), 0.0); + Assert.assertEquals(batch.getBoolValue(3, r), row.getBoolValue(3)); + } + }); + } + + @Test + public void testRowViewGeohashValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + setupGeohashColumnLayout(batch, 0, "g", new long[]{0xDEAD_BEEFL, 0L}, new boolean[]{false, true}, 32); + Assert.assertEquals(0xDEAD_BEEFL, batch.row(0).getGeohashValue(0)); + Assert.assertEquals(0L, batch.row(1).getGeohashValue(0)); + }); + } + + @Test + public void testRowViewGetRowIndex() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 5); + setupLongColumnLayout(batch, 0, "l", + new long[]{0L, 0L, 0L, 0L, 0L}, + new boolean[]{false, false, false, false, false}); + RowView row = batch.row(3); + Assert.assertEquals(3, row.getRowIndex()); + row.of(0); + Assert.assertEquals(0, row.getRowIndex()); + row.of(4); + Assert.assertEquals(4, row.getRowIndex()); + }); + } + + @Test + public void testRowViewLong256WithSink() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + long[][] words = {{0x1L, 0x2L, 0x3L, 0x4L}, {0L, 0L, 0L, 0L}}; + setupLong256ColumnLayout(batch, words, new boolean[]{false, true}); + Long256Impl sink = new Long256Impl(); + + Assert.assertTrue(batch.row(0).getLong256(0, sink)); + Assert.assertEquals(0x1L, sink.getLong0()); + Assert.assertEquals(0x2L, sink.getLong1()); + Assert.assertEquals(0x3L, sink.getLong2()); + Assert.assertEquals(0x4L, sink.getLong3()); + + Assert.assertFalse(batch.row(1).getLong256(0, sink)); + }); + } + + @Test + public void testRowViewLong256Word() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + long[][] words = {{0x11L, 0x22L, 0x33L, 0x44L}, {0L, 0L, 0L, 0L}}; + setupLong256ColumnLayout(batch, words, new boolean[]{false, true}); + RowView row = batch.row(0); + Assert.assertEquals(0x11L, row.getLong256Word(0, 0)); + Assert.assertEquals(0x22L, row.getLong256Word(0, 1)); + Assert.assertEquals(0x33L, row.getLong256Word(0, 2)); + Assert.assertEquals(0x44L, row.getLong256Word(0, 3)); + Assert.assertEquals(0L, batch.row(1).getLong256Word(0, 0)); // NULL -> 0 + }); + } + + @Test + public void testRowViewOfReturnsThis() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + setupLongColumnLayout(batch, 0, "l", new long[]{7L, 8L}, new boolean[]{false, false}); + RowView v = batch.row(0); + Assert.assertSame(v, v.of(1)); + Assert.assertEquals(8L, v.getLongValue(0)); + }); + } + + @Test + public void testRowViewSingleSharedInstance() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupLongColumnLayout(batch, 0, "l", new long[]{1L, 2L, 3L}, new boolean[]{false, false, false}); + RowView a = batch.row(0); + RowView b = batch.row(1); + RowView c = batch.row(2); + Assert.assertSame(a, b); + Assert.assertSame(b, c); + Assert.assertEquals(2, a.getRowIndex()); + Assert.assertEquals(3L, a.getLongValue(0)); + }); + } + + @Test + public void testRowViewStrAStrBDualHold() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + setupVarcharColumnLayout(batch, 0, "s", + new String[]{"alpha", "beta"}, + new boolean[]{false, false}); + RowView row = batch.row(0); + DirectUtf8Sequence a = row.getStrA(0); + DirectUtf8Sequence b = batch.row(1).getStrB(0); + Assert.assertNotNull(a); + Assert.assertNotNull(b); + Assert.assertEquals("alpha", Utf8s.stringFromUtf8Bytes(a)); + Assert.assertEquals("beta", Utf8s.stringFromUtf8Bytes(b)); + }); + } + + @Test + public void testRowViewStringAccessors() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 3); + setupVarcharColumnLayout(batch, 0, "s", + new String[]{"alpha", null, "gamma"}, + new boolean[]{false, true, false}); + // Heap-allocating variant + Assert.assertEquals("alpha", batch.row(0).getString(0)); + Assert.assertNull(batch.row(1).getString(0)); + Assert.assertEquals("gamma", batch.row(2).getString(0)); + // Sink variant + StringSink sink = new StringSink(); + Assert.assertTrue(batch.row(0).getString(0, sink)); + Assert.assertEquals("alpha", sink.toString()); + sink.clear(); + Assert.assertFalse(batch.row(1).getString(0, sink)); + Assert.assertEquals(0, sink.length()); + Assert.assertTrue(batch.row(2).getString(0, sink)); + Assert.assertEquals("gamma", sink.toString()); + }); + } + + @Test + public void testRowViewSymbolAccessors() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 4); + String[] dict = {"AAPL", "MSFT"}; + int[] rowIds = {0, 1, 0, -1}; + setupSymbolColumnLayout(batch, dict, rowIds); + Assert.assertEquals("AAPL", batch.row(0).getSymbol(0)); + Assert.assertEquals("MSFT", batch.row(1).getSymbol(0)); + Assert.assertEquals("AAPL", batch.row(2).getSymbol(0)); + Assert.assertNull(batch.row(3).getSymbol(0)); + Assert.assertEquals(0, batch.row(0).getSymbolId(0)); + Assert.assertEquals(1, batch.row(1).getSymbolId(0)); + Assert.assertEquals(0, batch.row(2).getSymbolId(0)); + Assert.assertEquals(-1, batch.row(3).getSymbolId(0)); + }); + } + + // ---- Reflection + setup helpers ---- + + @Test + public void testRowViewUuidLoHi() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + long[] lo = {0xCAFE_BABEL, 0L}; + long[] hi = {0xDEAD_BEEFL, 0L}; + setupUuidColumnLayout(batch, lo, hi, new boolean[]{false, true}); + Assert.assertEquals(0xCAFE_BABEL, batch.row(0).getUuidLo(0)); + Assert.assertEquals(0xDEAD_BEEFL, batch.row(0).getUuidHi(0)); + Assert.assertEquals(0L, batch.row(1).getUuidLo(0)); + Assert.assertEquals(0L, batch.row(1).getUuidHi(0)); + }); + } + + @Test + public void testRowViewUuidWithSink() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnBatch batch = newBatch(1, 2); + long[] lo = {0xAAAAL, 0L}; + long[] hi = {0xBBBBL, 0L}; + setupUuidColumnLayout(batch, lo, hi, new boolean[]{false, true}); + Uuid sink = new Uuid(); + Assert.assertTrue(batch.row(0).getUuid(0, sink)); + Assert.assertEquals(0xAAAAL, sink.getLo()); + Assert.assertEquals(0xBBBBL, sink.getHi()); + Assert.assertFalse(batch.row(1).getUuid(0, sink)); + }); + } + + private static int countNonNull(boolean[] nulls) { + int n = 0; + for (boolean b : nulls) if (!b) n++; + return n; + } + + private static Field findField(Class cls, String name) throws NoSuchFieldException { + Class c = cls; + while (c != null) { + try { + return c.getDeclaredField(name); + } catch (NoSuchFieldException ignore) { + c = c.getSuperclass(); + } + } + throw new NoSuchFieldException(name); + } + + private static Object newColumnInfo(String name, byte wireType) throws Exception { + Class cls = Class.forName(COLUMN_INFO); + java.lang.reflect.Constructor ctor = cls.getDeclaredConstructor(); + ctor.setAccessible(true); + Object info = ctor.newInstance(); + java.lang.reflect.Method of = cls.getDeclaredMethod("of", String.class, byte.class); + of.setAccessible(true); + of.invoke(info, name, wireType); + return info; + } + + private static Object newLayoutInstance() throws Exception { + java.lang.reflect.Constructor ctor = Class.forName(COLUMN_LAYOUT).getDeclaredConstructor(); + ctor.setAccessible(true); + return ctor.newInstance(); + } + + private static Object readField(Object target, String name) throws Exception { + Field f = findField(target.getClass(), name); + f.setAccessible(true); + return f.get(target); + } + + private static void writeField(Object target, String name, Object value) throws Exception { + Field f = findField(target.getClass(), name); + f.setAccessible(true); + f.set(target, value); + } + + private static void writeField(Object target, String name, long value) throws Exception { + Field f = findField(target.getClass(), name); + f.setAccessible(true); + f.setLong(target, value); + } + + private static void writeField(Object target, String name, int value) throws Exception { + Field f = findField(target.getClass(), name); + f.setAccessible(true); + f.setInt(target, value); + } + + private long allocate(int bytes) { + long addr = Unsafe.malloc(bytes, MemoryTag.NATIVE_DEFAULT); + allocations.add(new long[]{addr, bytes}); + return addr; + } + + private void applyNullBitmap(Object layout, int rowCount, boolean[] nulls) throws Exception { + boolean hasNulls = false; + for (boolean n : nulls) + if (n) { + hasNulls = true; + break; + } + if (!hasNulls) { + writeField(layout, "nullBitmapAddr", 0L); + writeField(layout, "nonNullIdx", null); + return; + } + int bitmapBytes = (rowCount + 7) >>> 3; + long bitmap = allocate(bitmapBytes); + for (int i = 0; i < bitmapBytes; i++) Unsafe.getUnsafe().putByte(bitmap + i, (byte) 0); + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) { + long addr = bitmap + (r >>> 3); + Unsafe.getUnsafe().putByte(addr, (byte) (Unsafe.getUnsafe().getByte(addr) | (1 << (r & 7)))); + } + } + int[] nonNullIdx = new int[rowCount]; + int dense = 0; + for (int r = 0; r < rowCount; r++) { + nonNullIdx[r] = nulls[r] ? -1 : dense++; + } + writeField(layout, "nullBitmapAddr", bitmap); + writeField(layout, "nonNullIdx", nonNullIdx); + } + + private QwpColumnBatch newBatch(int columnCount, int rowCount) throws Exception { + QwpColumnBatch batch = new QwpColumnBatch(); + writeField(batch, "columnCount", columnCount); + writeField(batch, "rowCount", rowCount); + + // Populate a matching ObjList so the batch's public + // getters (getColumnName, getColumnWireType) don't NPE. Each slot is + // filled by the per-column setup helper. + Class objListCls = Class.forName("io.questdb.client.std.ObjList"); + Object columnsList = objListCls.getDeclaredConstructor().newInstance(); + for (int i = 0; i < columnCount; i++) { + objListCls.getMethod("add", Object.class).invoke(columnsList, (Object) null); + } + writeField(batch, "columns", columnsList); + + Object layouts = readField(batch, "columnLayouts"); + objListCls.getMethod("setPos", int.class).invoke(layouts, columnCount); + return batch; + } + + private void setColumnInfoInBatch(QwpColumnBatch batch, int colIdx, Object info) throws Exception { + Object columns = readField(batch, "columns"); + Class objListCls = Class.forName("io.questdb.client.std.ObjList"); + objListCls.getMethod("setQuick", int.class, Object.class).invoke(columns, colIdx, info); + } + + private void setLayoutInBatch(QwpColumnBatch batch, int colIdx, Object layout) throws Exception { + Object layouts = readField(batch, "columnLayouts"); + Class objListCls = Class.forName("io.questdb.client.std.ObjList"); + objListCls.getMethod("setQuick", int.class, Object.class).invoke(layouts, colIdx, layout); + } + + private Object setupArrayColumnLayout(QwpColumnBatch batch, boolean[] nulls, double[][] arrays) throws Exception { + int rowCount = arrays.length; + int colIdx = 0; + Object l = newLayoutInstance(); + Object info = newColumnInfo("a", QwpConstants.TYPE_DOUBLE_ARRAY); + writeField(l, "info", info); + long[] perRowAddr = new long[rowCount]; + int[] perRowLen = new int[rowCount]; + for (int r = 0; r < rowCount; r++) { + if (nulls[r] || arrays[r] == null) { + perRowAddr[r] = -1L; + continue; + } + int len = 1 + 4 + 8 * arrays[r].length; + long addr = allocate(len); + // 1-D layout: 1 byte ndims (=1), 1 x int32 dim size, then flat doubles. + Unsafe.getUnsafe().putByte(addr, (byte) 1); + Unsafe.getUnsafe().putInt(addr + 1, arrays[r].length); + for (int i = 0; i < arrays[r].length; i++) { + Unsafe.getUnsafe().putDouble(addr + 5 + 8L * i, arrays[r][i]); + } + perRowAddr[r] = addr; + perRowLen[r] = len; + } + writeField(l, "arrayRowAddr", perRowAddr); + writeField(l, "arrayRowLen", perRowLen); + writeField(l, "valuesAddr", 0L); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", countNonNull(nulls)); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + return l; + } + + private void setupBinaryColumnLayout(QwpColumnBatch batch, byte[][] values, boolean[] nulls) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + int colIdx = 0; + Object l = newLayoutInstance(); + Object info = newColumnInfo("b", QwpConstants.TYPE_BINARY); + writeField(l, "info", info); + long offsetsAddr = allocate((nonNull + 1) * 4); + int totalBytes = 0; + for (int r = 0; r < rowCount; r++) { + if (!nulls[r] && values[r] != null) totalBytes += values[r].length; + } + long bytesAddr = allocate(Math.max(1, totalBytes)); + int dense = 0, byteOff = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r] || values[r] == null) continue; + Unsafe.getUnsafe().putInt(offsetsAddr + 4L * dense, byteOff); + for (int i = 0; i < values[r].length; i++) { + Unsafe.getUnsafe().putByte(bytesAddr + byteOff + i, values[r][i]); + } + byteOff += values[r].length; + dense++; + } + Unsafe.getUnsafe().putInt(offsetsAddr + 4L * dense, byteOff); + writeField(l, "valuesAddr", offsetsAddr); + writeField(l, "stringBytesAddr", bytesAddr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private void setupBooleanColumnLayout(QwpColumnBatch batch, int colIdx, + boolean[] values, boolean[] nulls) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + Object l = newLayoutInstance(); + Object info = newColumnInfo("b", QwpConstants.TYPE_BOOLEAN); + writeField(l, "info", info); + int bytes = Math.max(1, (nonNull + 7) >>> 3); + long addr = allocate(bytes); + for (int i = 0; i < bytes; i++) Unsafe.getUnsafe().putByte(addr + i, (byte) 0); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + if (values[r]) { + long byteAddr = addr + (dense >>> 3); + Unsafe.getUnsafe().putByte(byteAddr, + (byte) (Unsafe.getUnsafe().getByte(byteAddr) | (1 << (dense & 7)))); + } + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private void setupByteColumnLayout(QwpColumnBatch batch, int colIdx, + byte[] values, boolean[] nulls) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + Object l = newLayoutInstance(); + Object info = newColumnInfo("y", QwpConstants.TYPE_BYTE); + writeField(l, "info", info); + long addr = allocate(Math.max(1, nonNull)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + Unsafe.getUnsafe().putByte(addr + dense, values[r]); + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private void setupCharColumnLayout(QwpColumnBatch batch, int colIdx, + char[] values, boolean[] nulls) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + Object l = newLayoutInstance(); + Object info = newColumnInfo("c", QwpConstants.TYPE_CHAR); + writeField(l, "info", info); + long addr = allocate(Math.max(2, nonNull * 2)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + Unsafe.getUnsafe().putShort(addr + 2L * dense, (short) values[r]); + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private void setupDecimal128ColumnLayout(QwpColumnBatch batch, + long[] lo, long[] hi, boolean[] nulls, int scale) throws Exception { + int rowCount = lo.length; + int nonNull = countNonNull(nulls); + int colIdx = 0; + Object l = newLayoutInstance(); + Object info = newColumnInfo("d", QwpConstants.TYPE_DECIMAL128); + writeField(info, "scale", scale); + writeField(l, "info", info); + long addr = allocate(Math.max(16, nonNull * 16)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + // Matching the batch's layout: low 64 bits at offset 0, high 64 at offset +8. + Unsafe.getUnsafe().putLong(addr + 16L * dense, lo[r]); + Unsafe.getUnsafe().putLong(addr + 16L * dense + 8L, hi[r]); + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private Object setupDoubleColumnLayout(QwpColumnBatch batch, int colIdx, + double[] values, boolean[] nulls) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + Object l = newLayoutInstance(); + Object info = newColumnInfo("d", QwpConstants.TYPE_DOUBLE); + writeField(l, "info", info); + long addr = allocate(Math.max(8, nonNull * 8)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + Unsafe.getUnsafe().putDouble(addr + 8L * dense, values[r]); + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + return l; + } + + private void setupFloatColumnLayout(QwpColumnBatch batch, int colIdx, + float[] values, boolean[] nulls) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + Object l = newLayoutInstance(); + Object info = newColumnInfo("f", QwpConstants.TYPE_FLOAT); + writeField(l, "info", info); + long addr = allocate(Math.max(4, nonNull * 4)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + Unsafe.getUnsafe().putFloat(addr + 4L * dense, values[r]); + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private void setupGeohashColumnLayout(QwpColumnBatch batch, int colIdx, String name, + long[] values, boolean[] nulls, int precisionBits) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + int bytesPerValue = (precisionBits + 7) >>> 3; + Object l = newLayoutInstance(); + Object info = newColumnInfo(name, QwpConstants.TYPE_GEOHASH); + writeField(info, "precisionBits", precisionBits); + writeField(l, "info", info); + long addr = allocate(Math.max(1, nonNull * bytesPerValue)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + long v = values[r]; + // Little-endian LSB-first packing; matches QwpColumnBatch.getGeohashValue. + for (int b = 0; b < bytesPerValue; b++) { + Unsafe.getUnsafe().putByte(addr + (long) bytesPerValue * dense + b, (byte) ((v >>> (b * 8)) & 0xFF)); + } + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private void setupIntColumnLayout(QwpColumnBatch batch, int colIdx, + int[] values, boolean[] nulls) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + Object l = newLayoutInstance(); + Object info = newColumnInfo("i", QwpConstants.TYPE_INT); + writeField(l, "info", info); + long addr = allocate(Math.max(4, nonNull * 4)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + Unsafe.getUnsafe().putInt(addr + 4L * dense, values[r]); + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private void setupLong256ColumnLayout(QwpColumnBatch batch, long[][] words4, boolean[] nulls) throws Exception { + int rowCount = words4.length; + int nonNull = countNonNull(nulls); + int colIdx = 0; + Object l = newLayoutInstance(); + Object info = newColumnInfo("x", QwpConstants.TYPE_LONG256); + writeField(l, "info", info); + long addr = allocate(Math.max(32, nonNull * 32)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + for (int w = 0; w < 4; w++) { + Unsafe.getUnsafe().putLong(addr + 32L * dense + 8L * w, words4[r][w]); + } + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private Object setupLongColumnLayout(QwpColumnBatch batch, int colIdx, String name, + long[] values, boolean[] nulls) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + Object l = newLayoutInstance(); + Object info = newColumnInfo(name, QwpConstants.TYPE_LONG); + writeField(l, "info", info); + long addr = allocate(Math.max(8, nonNull * 8)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + Unsafe.getUnsafe().putLong(addr + 8L * dense, values[r]); + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + return l; + } + + private void setupShortColumnLayout(QwpColumnBatch batch, int colIdx, + short[] values, boolean[] nulls) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + Object l = newLayoutInstance(); + Object info = newColumnInfo("s", QwpConstants.TYPE_SHORT); + writeField(l, "info", info); + long addr = allocate(Math.max(2, nonNull * 2)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + Unsafe.getUnsafe().putShort(addr + 2L * dense, values[r]); + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private void setupSymbolColumnLayout(QwpColumnBatch batch, String[] dict, int[] rowIds) throws Exception { + int rowCount = rowIds.length; + int colIdx = 0; + Object l = newLayoutInstance(); + Object info = newColumnInfo("sym", QwpConstants.TYPE_SYMBOL); + writeField(l, "info", info); + int totalBytes = 0; + byte[][] encoded = new byte[dict.length][]; + for (int i = 0; i < dict.length; i++) { + encoded[i] = dict[i].getBytes(StandardCharsets.UTF_8); + totalBytes += encoded[i].length; + } + long heapAddr = allocate(Math.max(1, totalBytes)); + long entriesAddr = allocate(dict.length * 8); + int off = 0; + for (int i = 0; i < dict.length; i++) { + for (int b = 0; b < encoded[i].length; b++) { + Unsafe.getUnsafe().putByte(heapAddr + off + b, encoded[i][b]); + } + long packed = ((long) off & 0xFFFF_FFFFL) | ((long) encoded[i].length << 32); + Unsafe.getUnsafe().putLong(entriesAddr + 8L * i, packed); + off += encoded[i].length; + } + writeField(l, "symbolDictHeapAddr", heapAddr); + writeField(l, "symbolDictEntriesAddr", entriesAddr); + writeField(l, "symbolDictSize", dict.length); + writeField(l, "symbolRowIds", rowIds); + boolean[] nulls = new boolean[rowCount]; + for (int r = 0; r < rowCount; r++) nulls[r] = rowIds[r] < 0; + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", countNonNull(nulls)); + writeField(l, "valuesAddr", 0L); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private void setupUuidColumnLayout(QwpColumnBatch batch, long[] lo, long[] hi, boolean[] nulls) throws Exception { + int rowCount = lo.length; + int nonNull = countNonNull(nulls); + int colIdx = 0; + Object l = newLayoutInstance(); + Object info = newColumnInfo("u", QwpConstants.TYPE_UUID); + writeField(l, "info", info); + long addr = allocate(Math.max(16, nonNull * 16)); + int dense = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r]) continue; + Unsafe.getUnsafe().putLong(addr + 16L * dense, lo[r]); + Unsafe.getUnsafe().putLong(addr + 16L * dense + 8L, hi[r]); + dense++; + } + writeField(l, "valuesAddr", addr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } + + private void setupVarcharColumnLayout(QwpColumnBatch batch, int colIdx, String name, + String[] values, boolean[] nulls) throws Exception { + int rowCount = values.length; + int nonNull = countNonNull(nulls); + Object l = newLayoutInstance(); + Object info = newColumnInfo(name, QwpConstants.TYPE_VARCHAR); + writeField(l, "info", info); + long offsetsAddr = allocate((nonNull + 1) * 4); + int totalBytes = 0; + byte[][] encoded = new byte[rowCount][]; + for (int r = 0; r < rowCount; r++) { + if (nulls[r] || values[r] == null) continue; + encoded[r] = values[r].getBytes(StandardCharsets.UTF_8); + totalBytes += encoded[r].length; + } + long bytesAddr = allocate(Math.max(1, totalBytes)); + int dense = 0, byteOff = 0; + for (int r = 0; r < rowCount; r++) { + if (nulls[r] || values[r] == null) continue; + Unsafe.getUnsafe().putInt(offsetsAddr + 4L * dense, byteOff); + for (int i = 0; i < encoded[r].length; i++) { + Unsafe.getUnsafe().putByte(bytesAddr + byteOff + i, encoded[r][i]); + } + byteOff += encoded[r].length; + dense++; + } + Unsafe.getUnsafe().putInt(offsetsAddr + 4L * dense, byteOff); + writeField(l, "valuesAddr", offsetsAddr); + writeField(l, "stringBytesAddr", bytesAddr); + applyNullBitmap(l, rowCount, nulls); + writeField(l, "nonNullCount", nonNull); + setLayoutInBatch(batch, colIdx, l); + setColumnInfoInBatch(batch, colIdx, info); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnLayoutTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnLayoutTest.java new file mode 100644 index 00000000..303dcdd6 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpColumnLayoutTest.java @@ -0,0 +1,255 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpColumnLayout; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; + +/** + * Coverage for {@link QwpColumnLayout}: pooled per-column scratch state used + * by the result-batch decoder. The layout's fields are package-private and + * stamped by the decoder on the hot path; we use reflection to write the + * fields the decoder would set, then verify that {@code clear()} / + * {@code close()} clean up correctly and that the lazy-allocating + * {@code ensureOwnedEntriesAddr} / {@code ensureTimestampDecodeAddr} grow + * monotonically. All native allocations are accounted for via + * {@code TestUtils.assertMemoryLeak}. + */ +public class QwpColumnLayoutTest { + + @Test + public void testClearResetsScalarsButPreservesSymbolCacheReference() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpColumnLayout layout = new QwpColumnLayout()) { + writeLong(layout, "valuesAddr", 0xDEADBEEFL); + writeLong(layout, "nullBitmapAddr", 0xCAFEBABEL); + writeLong(layout, "stringBytesAddr", 0x1000L); + writeLong(layout, "symbolDictHeapAddr", 0x2000L); + writeLong(layout, "symbolDictEntriesAddr", 0x3000L); + writeLong(layout, "nextAddr", 0x4000L); + writeInt(layout, "nonNullCount", 99); + writeInt(layout, "symbolDictSize", 7); + + Object cacheBefore = readField(layout, "symbolStringCache"); + Assert.assertNotNull("symbolStringCache must be eagerly initialised", cacheBefore); + + layout.clear(); + + Assert.assertEquals(0L, readLong(layout, "valuesAddr")); + Assert.assertEquals(0L, readLong(layout, "nullBitmapAddr")); + Assert.assertEquals(0L, readLong(layout, "stringBytesAddr")); + Assert.assertEquals(0L, readLong(layout, "symbolDictHeapAddr")); + Assert.assertEquals(0L, readLong(layout, "symbolDictEntriesAddr")); + Assert.assertEquals(0L, readLong(layout, "nextAddr")); + Assert.assertEquals(0, readInt(layout, "nonNullCount")); + Assert.assertEquals(0, readInt(layout, "symbolDictSize")); + Assert.assertNull("clear() must drop the schema info reference", readField(layout, "info")); + + // The symbol-string cache is intentionally retained across batches; + // clear() must not wipe or replace it. Lazy invalidation keys off + // symbolDictVersion vs. symbolCacheVersion at lookup time. + Assert.assertSame("clear() must NOT replace the symbolStringCache instance", + cacheBefore, readField(layout, "symbolStringCache")); + } + }); + } + + @Test + public void testEnsureOwnedEntriesAddrLazyAllocAndGrowth() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpColumnLayout layout = new QwpColumnLayout()) { + Method ensure = QwpColumnLayout.class.getDeclaredMethod("ensureOwnedEntriesAddr", int.class); + ensure.setAccessible(true); + + // First call -- buffer was 0, so newCap = max(0, max(64, 16)) = 64. + long addr1 = (long) ensure.invoke(layout, 16); + Assert.assertNotEquals("alloc must produce a non-zero address", 0L, addr1); + Assert.assertEquals(64, readInt(layout, "ownedEntriesCapacity")); + + // Re-call within capacity -- address must be returned without realloc. + long addr2 = (long) ensure.invoke(layout, 32); + Assert.assertEquals("within-capacity request must reuse the existing buffer", + addr1, addr2); + Assert.assertEquals(64, readInt(layout, "ownedEntriesCapacity")); + + // Grow request -- newCap must be at least max(prev*2, requested). + long addr3 = (long) ensure.invoke(layout, 200); + Assert.assertNotEquals("grow must produce a non-zero address", 0L, addr3); + Assert.assertTrue("capacity must double-and-floor at requested", + readInt(layout, "ownedEntriesCapacity") >= 200); + + // Even larger request -- doubling beats the requested value. + long prevCap = readInt(layout, "ownedEntriesCapacity"); + ensure.invoke(layout, (int) prevCap + 1); + Assert.assertTrue("capacity must grow past the previous capacity", + readInt(layout, "ownedEntriesCapacity") > prevCap); + } + }); + } + + @Test + public void testEnsureTimestampDecodeAddrLazyAllocAndGrowth() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpColumnLayout layout = new QwpColumnLayout()) { + Method ensure = QwpColumnLayout.class.getDeclaredMethod("ensureTimestampDecodeAddr", int.class); + ensure.setAccessible(true); + + long addr1 = (long) ensure.invoke(layout, 8); + Assert.assertNotEquals(0L, addr1); + Assert.assertEquals("min capacity floors at 64 even when requested is smaller", + 64, readInt(layout, "timestampDecodeCapacity")); + + long addr2 = (long) ensure.invoke(layout, 64); + Assert.assertEquals("at-cap request must reuse buffer", addr1, addr2); + + long addr3 = (long) ensure.invoke(layout, 65); + Assert.assertNotEquals(0L, addr3); + Assert.assertTrue(readInt(layout, "timestampDecodeCapacity") >= 65); + } + }); + } + + @Test + public void testCloseFreesBothBuffers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnLayout layout = new QwpColumnLayout(); + Method ensureEntries = QwpColumnLayout.class.getDeclaredMethod("ensureOwnedEntriesAddr", int.class); + Method ensureTs = QwpColumnLayout.class.getDeclaredMethod("ensureTimestampDecodeAddr", int.class); + ensureEntries.setAccessible(true); + ensureTs.setAccessible(true); + + ensureEntries.invoke(layout, 128); + ensureTs.invoke(layout, 128); + Assert.assertNotEquals(0L, readLong(layout, "ownedEntriesAddr")); + Assert.assertNotEquals(0L, readLong(layout, "timestampDecodeAddr")); + + layout.close(); + + Assert.assertEquals(0L, readLong(layout, "ownedEntriesAddr")); + Assert.assertEquals(0L, readLong(layout, "timestampDecodeAddr")); + Assert.assertEquals(0, readInt(layout, "ownedEntriesCapacity")); + Assert.assertEquals(0, readInt(layout, "timestampDecodeCapacity")); + }); + } + + @Test + public void testCloseIsIdempotent() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnLayout layout = new QwpColumnLayout(); + Method ensureEntries = QwpColumnLayout.class.getDeclaredMethod("ensureOwnedEntriesAddr", int.class); + ensureEntries.setAccessible(true); + ensureEntries.invoke(layout, 64); + layout.close(); + // Second close must be a no-op (and must not double-free). + layout.close(); + layout.close(); + Assert.assertEquals(0L, readLong(layout, "ownedEntriesAddr")); + }); + } + + @Test + public void testCloseOnFreshLayoutIsNoOp() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpColumnLayout layout = new QwpColumnLayout(); + // Never allocated; close() must handle the all-zero case. + layout.close(); + Assert.assertEquals(0L, readLong(layout, "ownedEntriesAddr")); + Assert.assertEquals(0L, readLong(layout, "timestampDecodeAddr")); + }); + } + + @Test + public void testDenseIndexFastPathWhenNoNulls() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpColumnLayout layout = new QwpColumnLayout()) { + // nullBitmapAddr == 0 means "column has no nulls in this batch"; + // dense index equals row index. The decoder skips populating + // nonNullIdx in that case, so we must NOT touch it here. + writeLong(layout, "nullBitmapAddr", 0L); + Assert.assertEquals(0, layout.denseIndex(0)); + Assert.assertEquals(7, layout.denseIndex(7)); + Assert.assertEquals(99, layout.denseIndex(99)); + } + }); + } + + @Test + public void testDenseIndexSlowPathReadsNonNullIdx() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (QwpColumnLayout layout = new QwpColumnLayout()) { + // Non-zero nullBitmapAddr -- decoder has populated nonNullIdx. + // Row 1 is null (-1), rows 0/2/3 map to dense indices 0/1/2. + writeLong(layout, "nullBitmapAddr", 0xABCDL); + writeField(layout, "nonNullIdx", new int[]{0, -1, 1, 2}); + + Assert.assertEquals(0, layout.denseIndex(0)); + Assert.assertEquals(-1, layout.denseIndex(1)); + Assert.assertEquals(1, layout.denseIndex(2)); + Assert.assertEquals(2, layout.denseIndex(3)); + } + }); + } + + private static long readLong(Object target, String name) throws Exception { + Field f = QwpColumnLayout.class.getDeclaredField(name); + f.setAccessible(true); + return f.getLong(target); + } + + private static int readInt(Object target, String name) throws Exception { + Field f = QwpColumnLayout.class.getDeclaredField(name); + f.setAccessible(true); + return f.getInt(target); + } + + private static Object readField(Object target, String name) throws Exception { + Field f = QwpColumnLayout.class.getDeclaredField(name); + f.setAccessible(true); + return f.get(target); + } + + private static void writeLong(Object target, String name, long value) throws Exception { + Field f = QwpColumnLayout.class.getDeclaredField(name); + f.setAccessible(true); + f.setLong(target, value); + } + + private static void writeInt(Object target, String name, int value) throws Exception { + Field f = QwpColumnLayout.class.getDeclaredField(name); + f.setAccessible(true); + f.setInt(target, value); + } + + private static void writeField(Object target, String name, Object value) throws Exception { + Field f = QwpColumnLayout.class.getDeclaredField(name); + f.setAccessible(true); + f.set(target, value); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressIoThreadCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressIoThreadCloseRaceTest.java new file mode 100644 index 00000000..56703328 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpEgressIoThreadCloseRaceTest.java @@ -0,0 +1,100 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpBatchBuffer; +import io.questdb.client.cutlass.qwp.client.QwpEgressIoThread; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Races {@link QwpEgressIoThread#releaseBuffer} against + * {@link QwpEgressIoThread#closePool} to confirm the TOCTOU close-race guard + * holds: prior to the fix, a user thread could read {@code closed == false}, + * pause, let {@code closePool} drain and clear {@code freeBuffers}, then + * offer its buffer into the emptied-but-still-instantiated queue -- stranding + * the buffer's native scratch with no consumer to close it. + *

+ * {@link TestUtils#assertMemoryLeak} makes the leak assertion automatic: any + * {@link QwpBatchBuffer} not closed by either {@code releaseBuffer}'s + * in-place fallback or {@code closePool}'s drain shows up as a native-memory + * leak delta at the end of the test. + */ +public class QwpEgressIoThreadCloseRaceTest { + + @Test + public void testReleaseBufferRacesClosePoolSafely() throws Exception { + Method closePoolMethod = QwpEgressIoThread.class.getDeclaredMethod("closePool"); + closePoolMethod.setAccessible(true); + TestUtils.assertMemoryLeak(() -> { + int iterations = 200; + for (int iter = 0; iter < iterations; iter++) { + // A fresh IoThread per iteration -- each runs its own race + // and must leave freeBuffers empty + all buffers closed by + // the time both threads exit. + QwpEgressIoThread io = new QwpEgressIoThread(null, /*bufferPoolSize=*/ 2, + (status, message) -> { /* unused */ }); + QwpBatchBuffer b0 = new QwpBatchBuffer(64); + QwpBatchBuffer b1 = new QwpBatchBuffer(64); + + CountDownLatch start = new CountDownLatch(1); + AtomicInteger failures = new AtomicInteger(); + + Thread releaseThread = new Thread(() -> { + try { + start.await(); + io.releaseBuffer(b0); + io.releaseBuffer(b1); + } catch (Throwable t) { + failures.incrementAndGet(); + } + }, "release-" + iter); + Thread closeThread = new Thread(() -> { + try { + start.await(); + closePoolMethod.invoke(io); + } catch (Throwable t) { + failures.incrementAndGet(); + } + }, "close-" + iter); + + releaseThread.start(); + closeThread.start(); + start.countDown(); + releaseThread.join(TimeUnit.SECONDS.toMillis(5)); + closeThread.join(TimeUnit.SECONDS.toMillis(5)); + + if (failures.get() != 0) { + throw new AssertionError("releaseBuffer/closePool race threw on iteration " + iter); + } + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java new file mode 100644 index 00000000..a28b8300 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java @@ -0,0 +1,248 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QueryEvent; +import io.questdb.client.cutlass.qwp.client.QwpBatchBuffer; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpEgressIoThread; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.DirectUtf8Sequence; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Regression for issue #3: close()-during-onBatch must not corrupt the + * user-visible column bytes. + *

+ * In-place RESULT_BATCH decoding leaves the {@link QwpColumnBatch} view + * pointing into the same native bytes that the WebSocket recv buffer owns + * (see {@code QwpResultBatchDecoder.parseStringColumn}). The I/O thread + * parks on {@code pendingRelease.take()} after publishing the + * {@code KIND_BATCH} event, so the user thread can read those bytes + * safely. Before the fix, an {@code InterruptedException} on that take() + * (delivered by {@code close()}'s {@code Thread.interrupt}) returned + * early; control unwound through {@code WebSocketClient.tryParseFrame} + * which advanced {@code recvReadPos} and ran {@code compactRecvBuffer} + * (a {@code Vect.memmove}) over bytes the user thread was still reading + * via aliased column pointers. + *

+ * The fix makes the park uninterruptible: an interrupt sets a + * thread-local sticky flag and re-enters {@code take()}. The I/O thread + * does not return from {@code handleResultBatch} until the user thread + * actually releases the buffer. {@code close()} may still time out and + * abandon the I/O thread (the documented leak path), but the abandoned + * thread is parked on {@code take()}, not unwinding through the recv + * buffer. + *

+ * This test pins that contract: under interrupt, the I/O thread stays + * parked; only release lets it proceed. + */ +public class QwpInPlaceDecodeAliasingTest { + + @Test + public void testInterruptDuringOnBatchKeepsIoThreadParkedUntilRelease() throws Exception { + TestUtils.assertMemoryLeak(() -> { + int payloadCap = 256; + long staging = Unsafe.malloc(payloadCap, MemoryTag.NATIVE_DEFAULT); + QwpEgressIoThread io = new QwpEgressIoThread(null, /*bufferPoolSize=*/ 2, + (status, message) -> { + // Terminal failures during this test would mean the + // decode itself rejected our frame -- fail loudly. + throw new AssertionError("unexpected terminal failure during decode: " + message); + }); + CountDownLatch onBatchReturned = new CountDownLatch(1); + AtomicBoolean workerInterruptedFlagOnReturn = new AtomicBoolean(); + + try { + int len = writeSingleVarcharBatch(staging, "ALPHA"); + + // Worker plays the I/O thread role: it decodes the frame, + // publishes the batch event, and (with the fix) parks on + // pendingRelease.take() until the main thread releases the + // buffer. + Thread worker = new Thread(() -> { + try { + io.onBinaryMessage(staging, len); + } finally { + workerInterruptedFlagOnReturn.set(Thread.currentThread().isInterrupted()); + onBatchReturned.countDown(); + } + }, "qwp-egress-io-test"); + worker.setDaemon(true); + worker.start(); + + // Drain the KIND_BATCH event the worker published. Use a + // bounded retry loop so a regression that fails to publish + // surfaces as a test timeout rather than a hang. + QueryEvent batchEvent = null; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (batchEvent == null && System.nanoTime() < deadline) { + QueryEvent ev = io.takeEvent(); + Assert.assertEquals("expected KIND_BATCH from a single RESULT_BATCH frame", + QueryEvent.KIND_BATCH, ev.kind); + batchEvent = ev; + } + Assert.assertNotNull("worker must publish a KIND_BATCH event", batchEvent); + QwpBatchBuffer batchBuf = batchEvent.buffer; + Assert.assertNotNull(batchBuf); + + QwpColumnBatch batch = batchOf(batchBuf); + DirectUtf8Sequence cell = batch.getStrA(0, 0); + Assert.assertNotNull(cell); + Assert.assertEquals("ALPHA", cell.toString()); + + // Simulate close()'s interrupt. The fix MUST keep the worker + // parked on pendingRelease.take(); without the fix, the + // worker returns from handleResultBatch / onBinaryMessage + // and onBatchReturned counts down before we release. + worker.interrupt(); + Assert.assertFalse( + "worker thread must stay parked on pendingRelease.take() while " + + "the user is mid-onBatch -- otherwise tryParseFrame's " + + "compactRecvBuffer would run over the user's aliased " + + "column pointers.", + onBatchReturned.await(500, TimeUnit.MILLISECONDS)); + Assert.assertTrue("worker must still be alive (parked on take())", worker.isAlive()); + + // Re-read the cell after the interrupt. With the fix, no + // recv-buf compaction has run, so the column view still + // resolves to the original payload bytes. + DirectUtf8Sequence cellAfterInterrupt = batch.getStrA(0, 0); + Assert.assertEquals( + "user-visible column bytes must remain stable while the worker " + + "is parked under interrupt", + "ALPHA", cellAfterInterrupt.toString()); + + // Release: the worker can now finish handleResultBatch and + // onBinaryMessage returns. The interrupt status set by + // worker.interrupt() must be preserved on the way out so + // any caller higher up the stack can still observe it. + releaseBuffer(io, batchBuf); + Assert.assertTrue("worker must complete after release", + onBatchReturned.await(5, TimeUnit.SECONDS)); + worker.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("worker must terminate", worker.isAlive()); + Assert.assertTrue( + "interrupt status must be preserved across the uninterruptible park " + + "so the run loop's outer shutdown check still observes it", + workerInterruptedFlagOnReturn.get()); + } finally { + Unsafe.free(staging, payloadCap, MemoryTag.NATIVE_DEFAULT); + closePool(io); + } + }); + } + + private static QwpColumnBatch batchOf(QwpBatchBuffer buffer) { + try { + Field f = QwpBatchBuffer.class.getDeclaredField("batch"); + f.setAccessible(true); + return (QwpColumnBatch) f.get(buffer); + } catch (ReflectiveOperationException e) { + throw new AssertionError("could not access QwpBatchBuffer.batch via reflection", e); + } + } + + private static void closePool(QwpEgressIoThread io) { + try { + Method m = QwpEgressIoThread.class.getDeclaredMethod("closePool"); + m.setAccessible(true); + m.invoke(io); + } catch (ReflectiveOperationException e) { + throw new AssertionError("could not invoke QwpEgressIoThread.closePool via reflection", e); + } + } + + private static long putByte(long p, byte v) { + Unsafe.getUnsafe().putByte(p, v); + return p + 1; + } + + private static long putInt(long p, int v) { + Unsafe.getUnsafe().putInt(p, v); + return p + 4; + } + + private static long putLong(long p, long v) { + Unsafe.getUnsafe().putLong(p, v); + return p + 8; + } + + private static long putVarint(long p, long value) { + while ((value & ~0x7FL) != 0) { + Unsafe.getUnsafe().putByte(p++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(p++, (byte) (value & 0x7F)); + return p; + } + + private static void releaseBuffer(QwpEgressIoThread io, QwpBatchBuffer buffer) { + io.releaseBuffer(buffer); + } + + /** + * Crafts a RESULT_BATCH carrying one VARCHAR row whose value is + * {@code content}. String bytes sit at the tail of the frame so the + * cell view's {@code stringBytesAddr} points there directly. + */ + private static int writeSingleVarcharBatch(long buf, String content) { + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); // flags + p = putByte(p, (byte) 1); // table_count + p = putInt(p, 0); // payload_length placeholder + p = putByte(p, (byte) 0x11); // msg_kind = RESULT_BATCH + p = putLong(p, 7L); // request_id + p = putVarint(p, 0L); // batch_seq + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 1L); // row_count = 1 + p = putVarint(p, 1L); // column_count = 1 + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); // schema_id + p = putVarint(p, 1L); // column name length + p = putByte(p, (byte) 's'); + p = putByte(p, QwpConstants.TYPE_VARCHAR); + p = putByte(p, (byte) 0); // null_flag + p = putInt(p, 0); // offset[0] + p = putInt(p, bytes.length); // offset[1] = totalBytes + for (byte b : bytes) p = putByte(p, b); + return (int) (p - buf); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java new file mode 100644 index 00000000..5ce009f7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java @@ -0,0 +1,678 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import org.junit.Assert; +import org.junit.Test; + +/** + * Exhaustive unit coverage for {@link QwpQueryClient#fromConfig}: every parse + * failure path is exercised with its exact user-visible error message, and + * the happy-path branches that land distinct settings on the returned client + * are verified via introspection where the accessors exist, or through + * round-trip connection strings otherwise. The point of verifying error + * message *text* (not just the exception type) is so that a refactor that + * silently changes the wording does not break downstream UI / CLI tools + * relying on specific strings for diagnostics. + */ +public class QwpQueryClientFromConfigTest { + + @Test + public void testAddrAcceptsHostWithoutPort() { + // Host-only accepted; port defaults to the public DEFAULT_WS_PORT constant. + assertParses("ws::addr=db.internal;"); + } + + @Test + public void testAddrAcceptsMultipleEntries() { + // Three-endpoint config must parse without error. + assertParses("ws::addr=a.internal:9000,b.internal:9000,c.internal:9000;"); + } + + @Test + public void testAddrEmptyEntryAtEndRejected() { + assertReject("ws::addr=a:9000,;", "empty addr entry"); + } + + @Test + public void testAddrEmptyEntryAtStartRejected() { + assertReject("ws::addr=,b:9000;", "empty addr entry"); + } + + @Test + public void testAddrEmptyEntryInMiddleRejected() { + assertReject("ws::addr=a:9000,,b:9000;", "empty addr entry"); + } + + @Test + public void testAddrEmptyHostRejected() { + assertReject("ws::addr=:9000;", "empty host in addr entry: :9000"); + } + + @Test + public void testAddrInvalidPortMultiEntryRejected() { + assertReject("ws::addr=a:9000,b:notaport;", "invalid port in addr: b:notaport"); + } + + @Test + public void testAddrInvalidPortRejected() { + assertReject("ws::addr=host:abc;", "invalid port in addr: host:abc"); + } + + @Test + public void testAddrIpv6BareMultiColonTreatedAsHost() { + // Multi-colon, unbracketed: treated as bare IPv6 host with default + // port. Custom port on IPv6 requires brackets. + assertParses("ws::addr=fe80::1;"); + } + + @Test + public void testAddrIpv6BracketedEmptyHostRejected() { + assertReject("ws::addr=[]:9000;", "empty host in addr entry: []:9000"); + } + + @Test + public void testAddrIpv6BracketedMixedListAccepted() { + assertParses("ws::addr=[::1]:9000,[fe80::1],host.local:9001;"); + } + + @Test + public void testAddrIpv6BracketedRejectsTrailingGarbageBeforePort() { + // "]" must be followed immediately by ':' (or end of entry), not by + // trailing characters. Surfaces obvious typos rather than guessing. + assertReject("ws::addr=[::1]9000;", + "expected ':' after ']' in IPv6 addr entry: [::1]9000"); + } + + @Test + public void testAddrIpv6BracketedWithPortAccepted() { + // Per RFC 3986, IPv6 addresses must be bracketed when carrying a port. + assertParses("ws::addr=[::1]:9000;"); + } + + @Test + public void testAddrIpv6BracketedWithoutPortAccepted() { + assertParses("ws::addr=[fe80::1];"); + } + + @Test + public void testAddrIpv6MissingClosingBracketRejected() { + assertReject("ws::addr=[::1:9000;", + "missing closing ']' in IPv6 addr entry: [::1:9000"); + } + + @Test + public void testAddrPortAbove65535Rejected() { + assertReject("ws::addr=db:65536;", + "port out of range in addr: db:65536 (must be 1-65535)"); + } + + @Test + public void testAddrPortIpv6BracketedOutOfRangeRejected() { + assertReject("ws::addr=[::1]:0;", + "port out of range in addr: [::1]:0 (must be 1-65535)"); + } + + @Test + public void testAddrPortMaxValueRejected() { + // Integer.MAX_VALUE parses successfully but is well above 65535. + assertReject("ws::addr=db:2147483647;", + "port out of range in addr: db:2147483647 (must be 1-65535)"); + } + + @Test + public void testAddrPortNegativeRejected() { + assertReject("ws::addr=db:-1;", "port out of range in addr: db:-1 (must be 1-65535)"); + } + + @Test + public void testAddrPortWhitespaceTolerated() { + // Hand-edited config strings sometimes pick up a stray space around + // the port. Tolerate it rather than surface as opaque "invalid port". + assertParses("ws::addr=host: 9000;"); + } + + @Test + public void testAddrPortZeroRejected() { + assertReject("ws::addr=db:0;", "port out of range in addr: db:0 (must be 1-65535)"); + } + + @Test + public void testAddrSingleWhitespaceTrimmedAroundHostPort() { + // The parser splits on commas and trims; a single leading space on a + // valid entry must therefore be tolerated rather than rejected as + // "empty". Pin so a future refactor that drops trim() breaks here. + assertParses("ws::addr= db1:9000 , db2:9000 ;"); + } + + @Test + public void testAuthAndBasicMutuallyExclusive() { + assertReject( + "ws::addr=db:9000;auth=Bearer xyz;username=admin;password=quest;", + "auth, username/password, and token are mutually exclusive" + ); + } + + @Test + public void testAuthAndTokenMutuallyExclusive() { + assertReject( + "ws::addr=db:9000;auth=Bearer xyz;token=ey.xyz;", + "auth, username/password, and token are mutually exclusive" + ); + } + + @Test + public void testAuthHeaderAcceptedAlone() { + // Each of the three auth modes has a dedicated mutual-exclusion test; + // the positive happy path is asserted here so the parser's per-key + // dispatch and the post-loop "no auth set" path both have coverage. + assertParses("ws::addr=db:9000;auth=Bearer xyz;"); + } + + @Test + public void testBasicAuthAcceptedAlone() { + assertParses("ws::addr=db:9000;username=alice;password=secret;"); + } + + @Test + public void testBasicAuthAndTokenMutuallyExclusive() { + assertReject( + "ws::addr=db:9000;username=admin;password=quest;token=ey.xyz;", + "auth, username/password, and token are mutually exclusive" + ); + } + + @Test + public void testBasicAuthWithPasswordOnlyRejected() { + assertReject( + "ws::addr=db:9000;password=quest;", + "both username and password must be provided together" + ); + } + + @Test + public void testBasicAuthWithUsernameOnlyRejected() { + assertReject( + "ws::addr=db:9000;username=admin;", + "both username and password must be provided together" + ); + } + + @Test + public void testBufferPoolSizeLowerBoundRejected() { + assertReject("ws::addr=db:9000;buffer_pool_size=0;", "buffer_pool_size must be >= 1"); + } + + @Test + public void testBufferPoolSizeNegativeRejected() { + assertReject("ws::addr=db:9000;buffer_pool_size=-1;", "buffer_pool_size must be >= 1"); + } + + @Test + public void testBufferPoolSizeNonNumericRejected() { + assertReject("ws::addr=db:9000;buffer_pool_size=big;", "invalid buffer_pool_size: big"); + } + + @Test + public void testBufferPoolSizeValidAccepted() { + assertParses("ws::addr=db:9000;buffer_pool_size=8;"); + } + + @Test + public void testClientIdAcceptedAlone() { + // Sent as the X-QWP-Client-Id header on the upgrade request; useful for + // server-side telemetry. No format constraints from the parser. + assertParses("ws::addr=db:9000;client_id=batch-job/42;"); + } + + @Test + public void testCompressionAutoAccepted() { + try (QwpQueryClient c = QwpQueryClient.fromConfig("ws::addr=db:9000;compression=auto;")) { + Assert.assertEquals("auto", c.getCompressionPreference()); + } + } + + @Test + public void testCompressionDefaultIsRaw() { + try (QwpQueryClient c = QwpQueryClient.fromConfig("ws::addr=db:9000;")) { + Assert.assertEquals("raw", c.getCompressionPreference()); + } + } + + @Test + public void testCompressionInvalidRejected() { + assertReject( + "ws::addr=db:9000;compression=gzip;", + "unsupported compression: gzip (expected zstd, raw, or auto)" + ); + } + + @Test + public void testCompressionLevelAtLowerBoundAccepted() { + assertParses("ws::addr=db:9000;compression=zstd;compression_level=1;"); + } + + @Test + public void testCompressionLevelAtUpperBoundAccepted() { + // Parse-time cap is [1, 22]; server-side runtime clamp to [1, 9] is a separate concern. + assertParses("ws::addr=db:9000;compression=zstd;compression_level=22;"); + } + + @Test + public void testCompressionLevelNegativeRejected() { + assertReject("ws::addr=db:9000;compression_level=-1;", "compression_level must be in [1, 22]"); + } + + @Test + public void testCompressionLevelNonNumericRejected() { + assertReject("ws::addr=db:9000;compression_level=high;", "invalid compression_level: high"); + } + + @Test + public void testCompressionLevelOverUpperBoundRejected() { + assertReject("ws::addr=db:9000;compression_level=23;", "compression_level must be in [1, 22]"); + } + + @Test + public void testCompressionLevelZeroRejected() { + assertReject("ws::addr=db:9000;compression_level=0;", "compression_level must be in [1, 22]"); + } + + @Test + public void testCompressionRawAccepted() { + try (QwpQueryClient c = QwpQueryClient.fromConfig("ws::addr=db:9000;compression=raw;")) { + Assert.assertEquals("raw", c.getCompressionPreference()); + } + } + + @Test + public void testCompressionZstdAccepted() { + try (QwpQueryClient c = QwpQueryClient.fromConfig("ws::addr=db:9000;compression=zstd;")) { + Assert.assertEquals("zstd", c.getCompressionPreference()); + } + } + + @Test + public void testEmptyStringRejected() { + assertReject("", "configuration string cannot be empty"); + } + + @Test + public void testFailoverBackoffInitialAtZeroAccepted() { + assertParses("ws::addr=db:9000;failover_backoff_initial_ms=0;"); + } + + @Test + public void testFailoverBackoffInitialNegativeRejected() { + assertReject( + "ws::addr=db:9000;failover_backoff_initial_ms=-1;", + "failover_backoff_initial_ms must be >= 0" + ); + } + + @Test + public void testFailoverBackoffInitialNonNumericRejected() { + assertReject( + "ws::addr=db:9000;failover_backoff_initial_ms=soon;", + "invalid failover_backoff_initial_ms: soon" + ); + } + + @Test + public void testFailoverBackoffMaxAndInitialBothAccepted() { + assertParses("ws::addr=db:9000;failover_backoff_initial_ms=100;failover_backoff_max_ms=500;"); + } + + @Test + public void testFailoverBackoffMaxLessThanInitialRejected() { + assertReject( + "ws::addr=db:9000;failover_backoff_initial_ms=500;failover_backoff_max_ms=100;", + "failover_backoff_max_ms must be >= failover_backoff_initial_ms" + ); + } + + @Test + public void testFailoverBackoffMaxNegativeRejected() { + assertReject( + "ws::addr=db:9000;failover_backoff_max_ms=-1;", + "failover_backoff_max_ms must be >= 0" + ); + } + + @Test + public void testFailoverBackoffMaxNonNumericRejected() { + assertReject( + "ws::addr=db:9000;failover_backoff_max_ms=later;", + "invalid failover_backoff_max_ms: later" + ); + } + + @Test + public void testFailoverDefaultIsOn() { + // No failover= key: happy path, no throw. Internal default is on; not directly + // observable from the public API so we assert successful parse only. + assertParses("ws::addr=db:9000;"); + } + + @Test + public void testFailoverInvalidRejected() { + assertReject( + "ws::addr=db:9000;failover=maybe;", + "invalid failover: maybe (expected on or off)" + ); + } + + @Test + public void testFailoverMaxAttemptsAcceptedAtOne() { + assertParses("ws::addr=db:9000;failover_max_attempts=1;"); + } + + @Test + public void testFailoverMaxAttemptsNonNumericRejected() { + assertReject( + "ws::addr=db:9000;failover_max_attempts=many;", + "invalid failover_max_attempts: many" + ); + } + + @Test + public void testFailoverMaxAttemptsZeroRejected() { + assertReject( + "ws::addr=db:9000;failover_max_attempts=0;", + "failover_max_attempts must be >= 1" + ); + } + + @Test + public void testFailoverMaxBackoffEqualToInitialAccepted() { + // The "max < initial" rejection path is tested already; verify the + // boundary case (max == initial) is the lowest legal max and parses. + assertParses("ws::addr=db:9000;failover_backoff_initial_ms=100;failover_backoff_max_ms=100;"); + } + + @Test + public void testFailoverOffAccepted() { + assertParses("ws::addr=db:9000;failover=off;"); + } + + @Test + public void testFailoverOnAccepted() { + assertParses("ws::addr=db:9000;failover=on;"); + } + + @Test + public void testFullKitchenSinkAccepted() { + // Every optional key set to a valid non-default value on a wss:: schema. + // Verifies the parser's cross-key validation doesn't reject an otherwise + // legal combination, and that the happy-path client construction works. + String conf = "wss::addr=a.internal:9443,b.internal:9443,c.internal:9443;" + + "path=/read/v1;target=primary;failover=on;" + + "username=admin;password=quest;" + + "client_id=batch-job/42;buffer_pool_size=8;" + + "compression=zstd;compression_level=5;" + + "max_batch_rows=512;" + + "tls_verify=on;"; + assertParses(conf); + } + + @Test + public void testMalformedKeyValueRejected() { + // Missing = sign in a key=value pair -- ConfStringParser.nextKey / value + // bails out; fromConfig surfaces the parser's scratch sink verbatim in + // the "invalid configuration string [error=...]" shape. + try { + QwpQueryClient.fromConfig("ws::addr=db:9000;bogus;").close(); + Assert.fail("expected IllegalArgumentException for malformed key=value"); + } catch (IllegalArgumentException e) { + Assert.assertTrue( + "expected 'invalid configuration string' message, got: " + e.getMessage(), + e.getMessage().startsWith("invalid configuration string") + ); + } + } + + @Test + public void testMaxBatchRowsAtLowerBoundAccepted() { + assertParses("ws::addr=db:9000;max_batch_rows=1;"); + } + + @Test + public void testMaxBatchRowsAtUpperBoundAccepted() { + assertParses("ws::addr=db:9000;max_batch_rows=1048576;"); + } + + @Test + public void testMaxBatchRowsNonNumericRejected() { + assertReject("ws::addr=db:9000;max_batch_rows=lots;", "invalid max_batch_rows: lots"); + } + + @Test + public void testMaxBatchRowsOverUpperBoundRejected() { + assertReject( + "ws::addr=db:9000;max_batch_rows=1048577;", + "max_batch_rows must be in [1, 1048576]" + ); + } + + @Test + public void testMaxBatchRowsZeroRejected() { + assertReject( + "ws::addr=db:9000;max_batch_rows=0;", + "max_batch_rows must be in [1, 1048576]" + ); + } + + @Test + public void testMinimalWsConfigAccepted() { + assertParses("ws::addr=localhost:9000;"); + } + + @Test + public void testMinimalWssConfigAccepted() { + assertParses("wss::addr=secure.internal:9443;"); + } + + @Test + public void testMissingAddrRejected() { + assertReject("ws::client_id=test;", "missing required key: addr"); + } + + @Test + public void testMissingSchemaRejected() { + // No "::" separator -- the schema parser returns negative and fromConfig + // reports the parser's error sink. + try { + QwpQueryClient.fromConfig("addr=db:9000;").close(); + Assert.fail("expected IllegalArgumentException for missing schema"); + } catch (IllegalArgumentException e) { + // ConfStringParser either surfaces as "unsupported schema" (if it + // parses "addr=db" as a schema-like token) or "invalid configuration + // string". Accept either since both are actionable. + String msg = e.getMessage(); + Assert.assertTrue( + "expected schema-related error, got: " + msg, + msg.startsWith("invalid configuration string") + || msg.startsWith("unsupported schema") + ); + } + } + + @Test + public void testNullStringRejected() { + assertReject(null, "configuration string cannot be empty"); + } + + @Test + public void testPathOverrideAccepted() { + assertParses("ws::addr=db:9000;path=/custom/read;"); + } + + @Test + public void testTargetAnyAccepted() { + assertParses("ws::addr=db:9000;target=any;"); + } + + @Test + public void testTargetInvalidRejected() { + assertReject( + "ws::addr=db:9000;target=leader;", + "invalid target: leader (expected any, primary, or replica)" + ); + } + + @Test + public void testTargetPrimaryAccepted() { + assertParses("ws::addr=db:9000;target=primary;"); + } + + @Test + public void testTargetReplicaAccepted() { + assertParses("ws::addr=db:9000;target=replica;"); + } + + @Test + public void testTlsRootsOnWsRejected() { + assertReject( + "ws::addr=db:9000;tls_roots=/etc/qdb/ca.p12;tls_roots_password=secret;", + "tls_verify/tls_roots/tls_roots_password require the wss:: schema" + ); + } + + @Test + public void testTlsRootsPasswordWithoutPathRejected() { + assertReject( + "wss::addr=db:9000;tls_roots_password=secret;", + "tls_roots and tls_roots_password must be provided together" + ); + } + + @Test + public void testTlsRootsWithPasswordAccepted() { + assertParses("wss::addr=db:9000;tls_roots=/etc/qdb/ca.p12;tls_roots_password=secret;"); + } + + @Test + public void testTlsRootsWithoutPasswordRejected() { + assertReject( + "wss::addr=db:9000;tls_roots=/etc/qdb/ca.p12;", + "tls_roots and tls_roots_password must be provided together" + ); + } + + @Test + public void testTlsVerifyInvalidRejected() { + assertReject( + "wss::addr=db:9000;tls_verify=strict;", + "invalid tls_verify: strict (expected on or unsafe_off)" + ); + } + + @Test + public void testTlsVerifyOnWsRejected() { + assertReject( + "ws::addr=db:9000;tls_verify=on;", + "tls_verify/tls_roots/tls_roots_password require the wss:: schema" + ); + } + + @Test + public void testTlsVerifyOnWssAccepted() { + assertParses("wss::addr=db:9000;tls_verify=on;"); + } + + @Test + public void testTlsVerifyUnsafeOffOnWssAccepted() { + assertParses("wss::addr=db:9000;tls_verify=unsafe_off;"); + } + + @Test + public void testTokenAcceptedAlone() { + assertParses("ws::addr=db:9000;token=ey.payload.sig;"); + } + + @Test + public void testTokenRequestEncodesAsBearer() { + // We can't easily snoop the request header without a server, but the + // parser must at least accept the configuration end-to-end, then exit + // through close() without leaking the I/O thread (which never spun up). + try (QwpQueryClient c = QwpQueryClient.fromConfig("ws::addr=db:9000;token=abc.def.ghi;")) { + Assert.assertFalse(c.isConnected()); + Assert.assertFalse(c.wasLastCloseTimedOut()); + } + } + + @Test + public void testUnknownKeyRejected() { + assertReject( + "ws::addr=db:9000;mystery_knob=1;", + "unknown configuration key: mystery_knob" + ); + } + + @Test + public void testUnsupportedSchemaRejected() { + assertReject( + "http::addr=db:9000;", + "unsupported schema [schema=http, supported-schemas=[ws, wss]]" + ); + } + + @Test + public void testWhitespaceOnlyAddrEntryRejected() { + // A single-space entry between commas collapses to "empty" by the + // parser's .isEmpty() check after trim(). The exact rejection message + // is "empty addr entry". + assertReject("ws::addr=a:9000, ,b:9000;", "empty addr entry"); + } + + /** + * Asserts that {@code conf} parses into a non-null {@link QwpQueryClient} + * and closes the result on the way out. Centralising both checks here + * stops every happy-path test from leaking the I/O scaffolding the + * client allocates eagerly in fromConfig(). + */ + private static void assertParses(String conf) { + try (QwpQueryClient c = QwpQueryClient.fromConfig(conf)) { + Assert.assertNotNull(c); + } + } + + /** + * Asserts that {@link QwpQueryClient#fromConfig(CharSequence)} rejects + * {@code conf} with an {@link IllegalArgumentException} whose message + * equals {@code expectedMessage} exactly. The strict match is deliberate: + * downstream tooling consumes these strings verbatim in diagnostics, so a + * silent refactor that tweaks wording must break this test. + */ + private static void assertReject(String conf, String expectedMessage) { + try { + QwpQueryClient.fromConfig(conf).close(); + Assert.fail("expected IllegalArgumentException for: " + conf); + } catch (IllegalArgumentException e) { + Assert.assertEquals(expectedMessage, e.getMessage()); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientPostConnectGuardTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientPostConnectGuardTest.java new file mode 100644 index 00000000..69847682 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientPostConnectGuardTest.java @@ -0,0 +1,111 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; + +/** + * Verifies the post-connect mutation guard on every {@code with*} setter. + * Uses reflection to flip the {@code connected} field to {@code true} -- + * avoids the cost of booting a real server just to exercise the guard, and + * keeps the coverage dense (one assertion per setter, ~20 setters). + *

+ * The test asserts two things per setter: + * 1. The exception is {@link IllegalStateException} (so the check is + * reachable and typed correctly). + * 2. The exception message names the setter, so a stack-free error surface + * tells the caller which setter was at fault. Downstream UIs and CLIs + * surface the message verbatim. + */ +public class QwpQueryClientPostConnectGuardTest { + + @Test + public void testAllSettersRejectAfterConnect() throws Exception { + // withAuthorization + assertRejects(c -> c.withAuthorization("Bearer x"), "withAuthorization"); + // withBasicAuth + assertRejects(c -> c.withBasicAuth("u", "p"), "withBasicAuth"); + // withBearerToken + assertRejects(c -> c.withBearerToken("tok"), "withBearerToken"); + // withBufferPoolSize + assertRejects(c -> c.withBufferPoolSize(2), "withBufferPoolSize"); + // withClientId + assertRejects(c -> c.withClientId("id"), "withClientId"); + // withCompression + assertRejects(c -> c.withCompression("zstd", 3), "withCompression"); + // withEndpointPath + assertRejects(c -> c.withEndpointPath("/x"), "withEndpointPath"); + // withFailover + assertRejects(c -> c.withFailover(false), "withFailover"); + // withFailoverBackoff + assertRejects(c -> c.withFailoverBackoff(100L, 500L), "withFailoverBackoff"); + // withFailoverMaxAttempts + assertRejects(c -> c.withFailoverMaxAttempts(3), "withFailoverMaxAttempts"); + // withInitialCredit + assertRejects(c -> c.withInitialCredit(1024L), "withInitialCredit"); + // withInsecureTls + assertRejects(QwpQueryClient::withInsecureTls, "withInsecureTls"); + // withMaxBatchRows + assertRejects(c -> c.withMaxBatchRows(128), "withMaxBatchRows"); + // withServerInfoTimeout + assertRejects(c -> c.withServerInfoTimeout(500), "withServerInfoTimeout"); + // withTarget + assertRejects(c -> c.withTarget(QwpQueryClient.TARGET_PRIMARY), "withTarget"); + // withTls + assertRejects(QwpQueryClient::withTls, "withTls"); + // withTrustStore + assertRejects(c -> c.withTrustStore("/tmp/ts.p12", "pw".toCharArray()), "withTrustStore"); + } + + private static QwpQueryClient fakeConnected() throws Exception { + QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000); + Field connected = QwpQueryClient.class.getDeclaredField("connected"); + connected.setAccessible(true); + connected.setBoolean(c, true); + return c; + } + + private static void assertRejects(Setter setter, String setterName) throws Exception { + QwpQueryClient c = fakeConnected(); + try { + setter.apply(c); + Assert.fail(setterName + " must throw IllegalStateException after connect()"); + } catch (IllegalStateException e) { + Assert.assertTrue( + "exception message for " + setterName + " must name the setter, was: " + e.getMessage(), + e.getMessage() != null && e.getMessage().startsWith(setterName + " must be called before connect()") + ); + } + } + + @FunctionalInterface + private interface Setter { + void apply(QwpQueryClient client); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUnitTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUnitTest.java new file mode 100644 index 00000000..f5ee7bdb --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUnitTest.java @@ -0,0 +1,231 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import org.junit.Assert; +import org.junit.Test; + +/** + * Unit coverage for {@link QwpQueryClient} surface that does not require a + * live socket: the pre-connect getters, the cancel/close idempotency + * guarantees, the routing-target constants, and the bind-encoding error + * surface. Live failover, role-mismatch, SERVER_INFO negotiation and the rest + * of the connect()-time state machine are intentionally NOT covered here -- + * they belong to the parent QuestDB egress integration suite per the + * package-level testing policy. See {@link QwpQueryClientPostConnectGuardTest} + * for setter guard coverage. + */ +public class QwpQueryClientUnitTest { + + private static final QwpColumnBatchHandler NOOP_HANDLER = new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + } + }; + + @Test + public void testTargetConstants() { + // The connection-string parser keys on these literal strings, and the + // role-matching logic compares the configured target against them. + // Pin the values so a typo refactor doesn't silently break the parse + // and routing paths simultaneously. + Assert.assertEquals("any", QwpQueryClient.TARGET_ANY); + Assert.assertEquals("primary", QwpQueryClient.TARGET_PRIMARY); + Assert.assertEquals("replica", QwpQueryClient.TARGET_REPLICA); + } + + @Test + public void testNewPlainTextDoesNotConnect() { + // Constructor must not perform I/O. Pin this so a future refactor + // doesn't accidentally make `new` block or throw on a bad host. + try (QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000)) { + Assert.assertFalse(c.isConnected()); + Assert.assertNull("getServerInfo must be null before connect()", c.getServerInfo()); + Assert.assertEquals(0, c.getNegotiatedQwpVersion()); + Assert.assertFalse(c.wasLastCloseTimedOut()); + } + } + + @Test + public void testCompressionPreferenceDefaultsToRaw() { + // Library default per the connection-string docs: no compression unless + // the caller opts in via compression=zstd|auto. + try (QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000)) { + Assert.assertEquals("raw", c.getCompressionPreference()); + } + } + + @Test + public void testCompressionPreferencePropagatedFromConfig() { + try (QwpQueryClient c = QwpQueryClient.fromConfig("ws::addr=localhost:9000;compression=auto;")) { + Assert.assertEquals("auto", c.getCompressionPreference()); + } + try (QwpQueryClient c = QwpQueryClient.fromConfig("ws::addr=localhost:9000;compression=zstd;")) { + Assert.assertEquals("zstd", c.getCompressionPreference()); + } + } + + @Test + public void testExecuteBeforeConnectThrowsIllegalState() { + try (QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000)) { + try { + c.execute("SELECT 1", NOOP_HANDLER); + Assert.fail("execute() before connect() must throw"); + } catch (IllegalStateException expected) { + Assert.assertTrue("error must mention connect(): " + expected.getMessage(), + expected.getMessage().contains("connect()")); + } + } + } + + @Test + public void testExecuteWithBindsBeforeConnectThrowsIllegalState() { + try (QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000)) { + try { + c.execute("SELECT 1", binds -> binds.setLong(0, 42L), NOOP_HANDLER); + Assert.fail("execute(binds) before connect() must throw"); + } catch (IllegalStateException expected) { + // expected + } + } + } + + @Test + public void testCancelOnFreshClientIsNoOp() { + // cancel() must be safe to call before connect (e.g., a watchdog timer + // that fires before the connect handshake completed). Documented on + // QwpQueryClient.cancel as "No-op if no query is in flight". + try (QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000)) { + c.cancel(); + c.cancel(); // idempotent + } + } + + @Test + public void testCloseOnNeverConnectedClientIsNoOp() { + QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000); + c.close(); + Assert.assertFalse(c.wasLastCloseTimedOut()); + // Second close must also not throw. + c.close(); + } + + /** + * Regression: concurrent close() calls must not double-free the shared + * bindValues native scratch (or re-enter the I/O thread shutdown path). + * Prior to the AtomicBoolean gate on close(), two threads could both + * walk the shutdown body; the internal close() of each native resource + * is individually idempotent, but stacking the whole sequence twice + * wastes work and makes state transitions harder to reason about. + */ + @Test(timeout = 10_000L) + public void testConcurrentCloseIsSafe() throws Exception { + QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000); + int threads = 8; + java.util.concurrent.CountDownLatch ready = new java.util.concurrent.CountDownLatch(threads); + java.util.concurrent.CountDownLatch start = new java.util.concurrent.CountDownLatch(1); + java.util.concurrent.atomic.AtomicInteger failed = new java.util.concurrent.atomic.AtomicInteger(); + Thread[] workers = new Thread[threads]; + for (int i = 0; i < threads; i++) { + workers[i] = new Thread(() -> { + ready.countDown(); + try { + start.await(); + c.close(); + } catch (Throwable t) { + failed.incrementAndGet(); + } + }, "close-race-" + i); + workers[i].start(); + } + ready.await(); + start.countDown(); + for (Thread w : workers) w.join(); + Assert.assertEquals("no close() invocation should have thrown", 0, failed.get()); + // A third serial close for good measure -- also a no-op. + c.close(); + } + + @Test + public void testWasLastCloseTimedOutDefaultFalse() { + // No connection attempt -> no I/O thread to time out joining; the flag + // must read false. (The flag is set true only by close() when the I/O + // thread fails to exit within shutdownJoinMs.) + try (QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000)) { + Assert.assertFalse(c.wasLastCloseTimedOut()); + } + } + + @Test + public void testFromConfigReturnsUsableClient() { + // Sanity round-trip: a minimal valid config string parses, applies, and + // leaves the client ready (but not connected). Detailed parse-branch + // coverage lives in QwpQueryClientFromConfigTest; this is the smoke test + // that ties fromConfig to the rest of the unit-level guarantees here. + try (QwpQueryClient c = QwpQueryClient.fromConfig("ws::addr=localhost:9000;")) { + Assert.assertFalse(c.isConnected()); + Assert.assertNull(c.getServerInfo()); + } + } + + @Test + public void testGetServerInfoNullableBeforeConnect() { + // The accessor is documented as returning null before connect(); a + // QwpServerInfo instance materialises only after the v2 SERVER_INFO + // frame is decoded during connect(). + try (QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000)) { + QwpServerInfo info = c.getServerInfo(); + Assert.assertNull(info); + } + } + + @Test + public void testQwpMaxVersionConstant() { + // The negotiated-version cap is the v2 protocol; pin it so a future + // version bump that adds a new control frame must explicitly update + // both the constant and any callers asserting the cap. + Assert.assertEquals(2, QwpQueryClient.QWP_MAX_VERSION); + } + + @Test + public void testMaxBatchRowsUpperBoundMirrorsDecoderCap() { + // The setter clamps to this; the cap matches QwpResultBatchDecoder's + // own MAX_ROWS_PER_BATCH so a user can't ask for a per-batch row count + // the decoder would itself refuse. + Assert.assertEquals(1_048_576, QwpQueryClient.MAX_BATCH_ROWS_UPPER_BOUND); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderColumnTypesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderColumnTypesTest.java new file mode 100644 index 00000000..eed16251 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderColumnTypesTest.java @@ -0,0 +1,1102 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpBatchBuffer; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpDecodeException; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpResultBatchDecoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; + +/** + * Per-wire-type and per-feature coverage for {@link QwpResultBatchDecoder}. + * Sister to {@link QwpResultBatchDecoderHardeningTest}: the hardening file + * pins the rejection paths for malformed frames, this file pins the + * happy-path decoders for every wire type plus the connection-scoped state + * machines (delta SYMBOL dict, FLAG_GORILLA TIMESTAMP, FLAG_DELTA_SYMBOL_DICT, + * CACHE_RESET, SCHEMA_MODE_REFERENCE). Each test crafts a single-column or + * minimal multi-column RESULT_BATCH in native memory and verifies the values + * surface correctly through {@link io.questdb.client.cutlass.qwp.client.QwpColumnBatch} + * accessors, mirroring the production end-to-end flow without a server. + */ +public class QwpResultBatchDecoderColumnTypesTest { + + @Test + public void testBadMagicRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(64); + long staging = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + long p = staging; + p = putInt(p, 0xDEADBEEF); // wrong magic + for (int i = 4; i < 32; i++) p = putByte(p, (byte) 0); + buffer.copyFromPayload(staging, 32); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject bad magic"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions magic: " + expected.getMessage(), + expected.getMessage().contains("magic")); + } + } finally { + Unsafe.free(staging, 64, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testBooleanColumnDecodesBitPacked() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { + // 10 booleans across 2 bytes (LSB-first within each byte). Bits: + // row: 0 1 2 3 4 5 6 7 8 9 + // value: T F T F F T T F T F + // byte 0 = 0b01100101 = 0x65, byte 1 = 0b00000001 = 0x01 + int rows = 10; + long p = startSingleColumnFrame(staging, "b", QwpConstants.TYPE_BOOLEAN, rows); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0x65); + p = putByte(p, (byte) 0x01); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + boolean[] expected = {true, false, true, false, false, true, true, false, true, false}; + for (int i = 0; i < rows; i++) { + Assert.assertEquals("row " + i, expected[i], batchOf(buffer).getBoolValue(0, i)); + } + })); + } + + @Test + public void testByteColumnDecodes() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { + byte[] values = {Byte.MIN_VALUE, -1, 0, 7, Byte.MAX_VALUE}; + long p = startSingleColumnFrame(staging, "b", QwpConstants.TYPE_BYTE, values.length); + p = putByte(p, (byte) 0); + for (byte v : values) p = putByte(p, v); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + for (int i = 0; i < values.length; i++) { + Assert.assertEquals("row " + i, values[i], batchOf(buffer).getByteValue(0, i)); + } + })); + } + + @Test + public void testCacheResetDictMaskClearsConnectionDict() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + // Seed the dict with one entry via a delta-mode batch. + long p = staging; + p = writeHeader(p, QwpConstants.FLAG_DELTA_SYMBOL_DICT); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + byte[] aaa = "aaa".getBytes(StandardCharsets.UTF_8); + p = putVarint(p, aaa.length); + for (byte b : aaa) p = putByte(p, b); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putVarint(p, 1L); + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putByte(p, (byte) 's'); + p = putByte(p, QwpConstants.TYPE_SYMBOL); + p = putByte(p, (byte) 0); + p = putVarint(p, 0L); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals("aaa", batchOf(buffer).getSymbol(0, 0)); + + // Apply CACHE_RESET with the dict mask -- the connection dict + // empties and the next batch must restart deltaStart from 0. + decoder.applyCacheReset(QwpEgressMsgKind.RESET_MASK_DICT); + + // Batch 2: deltaStart = 0 again. Re-add a different entry. + p = staging; + p = writeHeader(p, QwpConstants.FLAG_DELTA_SYMBOL_DICT); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 1L); + p = putVarint(p, 0L); // deltaStart = 0 after reset + p = putVarint(p, 1L); + byte[] bbb = "bbb".getBytes(StandardCharsets.UTF_8); + p = putVarint(p, bbb.length); + for (byte b : bbb) p = putByte(p, b); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putVarint(p, 1L); + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putByte(p, (byte) 's'); + p = putByte(p, QwpConstants.TYPE_SYMBOL); + p = putByte(p, (byte) 0); + p = putVarint(p, 0L); + + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals("bbb", batchOf(buffer).getSymbol(0, 0)); + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testCacheResetSchemasMaskClearsRegistry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + // Register schema_id=3 with one INT column, then CACHE_RESET schemas, + // then attempt to reference schema_id=3 -- decoder must reject. + long p = startSingleColumnFrame(staging, "n", QwpConstants.TYPE_INT, 1, 3L); + p = putByte(p, (byte) 0); + p = putInt(p, 1); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + + decoder.applyCacheReset(QwpEgressMsgKind.RESET_MASK_SCHEMAS); + + p = staging; + p = writeHeader(p, (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putByte(p, QwpConstants.SCHEMA_MODE_REFERENCE); + p = putVarint(p, 3L); + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("schema ref must be rejected after CACHE_RESET schemas mask"); + } catch (QwpDecodeException expected) { + Assert.assertTrue(expected.getMessage().contains("not registered")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testCharColumnDecodes() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { + char[] values = {'a', 'Z', '0', '', '￿'}; + long p = startSingleColumnFrame(staging, "c", QwpConstants.TYPE_CHAR, values.length); + p = putByte(p, (byte) 0); + for (char v : values) p = putShort(p, (short) v); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + for (int i = 0; i < values.length; i++) { + Assert.assertEquals("row " + i, values[i], batchOf(buffer).getCharValue(0, i)); + } + })); + } + + @Test + public void testDecimal128ColumnDecodesScaleByte() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { + int rows = 1; + long p = startSingleColumnFrame(staging, "d", QwpConstants.TYPE_DECIMAL128, rows); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 4); // scale + p = putLong(p, 0xCAFEBABEDEADBEEFL); // low 8 bytes + p = putLong(p, 0x0102030405060708L); // high 8 bytes + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(4, batchOf(buffer).getDecimalScale(0)); + Assert.assertEquals(0xCAFEBABEDEADBEEFL, batchOf(buffer).getDecimal128Low(0, 0)); + Assert.assertEquals(0x0102030405060708L, batchOf(buffer).getDecimal128High(0, 0)); + })); + } + + @Test + public void testDoubleColumnDecodes() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { + double[] values = {Double.NEGATIVE_INFINITY, -1.5, 0.0, 1.5, Double.POSITIVE_INFINITY, Double.NaN}; + long p = startSingleColumnFrame(staging, "d", QwpConstants.TYPE_DOUBLE, values.length); + p = putByte(p, (byte) 0); + for (double v : values) { + p = putLong(p, Double.doubleToRawLongBits(v)); + } + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + for (int i = 0; i < values.length; i++) { + double got = batchOf(buffer).getDoubleValue(0, i); + if (Double.isNaN(values[i])) { + Assert.assertTrue("row " + i + " NaN", Double.isNaN(got)); + } else { + Assert.assertEquals("row " + i, values[i], got, 0.0); + } + } + })); + } + + @Test + public void testEmptyBatchDecodes() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(128, (decoder, buffer, staging) -> { + long p = staging; + p = writeHeader(p, /*flags=*/(byte) 0); + p = putByte(p, (byte) 0x11); // RESULT_BATCH + p = putLong(p, 1L); // request_id + p = putVarint(p, 0L); // batch_seq + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 0L); // row_count = 0 + p = putVarint(p, 0L); // column_count = 0 + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); // schema_id + + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(0, batchOf(buffer).getRowCount()); + Assert.assertEquals(0, batchOf(buffer).getColumnCount()); + })); + } + + @Test + public void testGeohashColumnReadsPrecisionVarint() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { + int rows = 2; + long p = startSingleColumnFrame(staging, "g", QwpConstants.TYPE_GEOHASH, rows); + p = putByte(p, (byte) 0); + // Precision = 20 bits -> ceil(20/8) = 3 bytes per value. + p = putVarint(p, 20L); + // Two values, 3 bytes each. + p = putByte(p, (byte) 0x01); + p = putByte(p, (byte) 0x02); + p = putByte(p, (byte) 0x03); + p = putByte(p, (byte) 0x04); + p = putByte(p, (byte) 0x05); + p = putByte(p, (byte) 0x06); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(rows, batchOf(buffer).getRowCount()); + // We only need to verify the decoder accepted the layout; the + // GEOHASH accessor is not in scope here. + })); + } + + @Test + public void testIntColumnDecodes() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { + int[] values = {Integer.MIN_VALUE, -1, 0, 42, Integer.MAX_VALUE}; + long p = startSingleColumnFrame(staging, "n", QwpConstants.TYPE_INT, values.length); + p = putByte(p, (byte) 0); + for (int v : values) p = putInt(p, v); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + for (int i = 0; i < values.length; i++) { + Assert.assertEquals("row " + i, values[i], batchOf(buffer).getIntValue(0, i)); + } + })); + } + + @Test + public void testLongArrayColumnDecodes() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(512, (decoder, buffer, staging) -> { + // 2 rows, each a 1-D array of 3 longs. + int rows = 2; + long p = startSingleColumnFrame(staging, "a", QwpConstants.TYPE_LONG_ARRAY, rows); + p = putByte(p, (byte) 0); + // row 0: nDims=1, dim=3, [10, 20, 30] + p = putByte(p, (byte) 1); + p = putInt(p, 3); + p = putLong(p, 10L); + p = putLong(p, 20L); + p = putLong(p, 30L); + // row 1: nDims=1, dim=3, [40, 50, 60] + p = putByte(p, (byte) 1); + p = putInt(p, 3); + p = putLong(p, 40L); + p = putLong(p, 50L); + p = putLong(p, 60L); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(rows, batchOf(buffer).getRowCount()); + })); + } + + @Test + public void testLongColumnDecodes() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(512, (decoder, buffer, staging) -> { + long[] values = {Long.MIN_VALUE, -1L, 0L, 1L, Long.MAX_VALUE}; + long p = startSingleColumnFrame(staging, "v", QwpConstants.TYPE_LONG, values.length); + p = putByte(p, (byte) 0); // null_flag = 0 (no nulls) + for (long v : values) p = putLong(p, v); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + for (int i = 0; i < values.length; i++) { + Assert.assertFalse(batchOf(buffer).isNull(0, i)); + Assert.assertEquals("row " + i, values[i], batchOf(buffer).getLongValue(0, i)); + } + })); + } + + @Test + public void testNegativeArrayDimRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + long p = startSingleColumnFrame(staging, "a", QwpConstants.TYPE_LONG_ARRAY, 1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, -1); // hostile negative dim + + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject negative ARRAY dim"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions negative dim: " + expected.getMessage(), + expected.getMessage().contains("ARRAY dim")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testNullBitmapPopulatesNonNullIdx() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { + // 4 rows: row 0 non-null, row 1 NULL, row 2 non-null, row 3 NULL. + // bitmap byte 0: bit 1 + bit 3 set = 0b00001010 = 0x0A. + // nonNullCount = 2, dense values = 2 longs. + int rows = 4; + long p = startSingleColumnFrame(staging, "v", QwpConstants.TYPE_LONG, rows); + p = putByte(p, (byte) 1); // null_flag = 1 -> bitmap follows + p = putByte(p, (byte) 0x0A); // bitmap byte + p = putLong(p, 100L); // dense[0] -> row 0 + p = putLong(p, 200L); // dense[1] -> row 2 + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + + Assert.assertFalse(batchOf(buffer).isNull(0, 0)); + Assert.assertTrue(batchOf(buffer).isNull(0, 1)); + Assert.assertFalse(batchOf(buffer).isNull(0, 2)); + Assert.assertTrue(batchOf(buffer).isNull(0, 3)); + Assert.assertEquals(100L, batchOf(buffer).getLongValue(0, 0)); + Assert.assertEquals(200L, batchOf(buffer).getLongValue(0, 2)); + })); + } + + @Test + public void testSchemaReferenceModeReusesEarlierSchema() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + // Batch 1 registers schema_id=7 with one INT column. + long p = startSingleColumnFrame(staging, "n", QwpConstants.TYPE_INT, 1, /*schemaId=*/ 7L); + p = putByte(p, (byte) 0); + p = putInt(p, 99); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(99, batchOf(buffer).getIntValue(0, 0)); + + // Batch 2 references schema_id=7 (REFERENCE mode) -- no schema bytes inline. + p = staging; + p = writeHeader(p, /*flags=*/ (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 1L); + p = putVarint(p, 0L); + p = putVarint(p, 1L); // row_count + p = putVarint(p, 1L); // column_count + p = putByte(p, QwpConstants.SCHEMA_MODE_REFERENCE); + p = putVarint(p, 7L); // schema_id + p = putByte(p, (byte) 0); // null_flag + p = putInt(p, 1234); // value + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(1234, batchOf(buffer).getIntValue(0, 0)); + Assert.assertEquals("n", batchOf(buffer).getColumnName(0)); + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testSchemaReferenceModeUnregisteredRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + long p = staging; + p = writeHeader(p, (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putByte(p, QwpConstants.SCHEMA_MODE_REFERENCE); + p = putVarint(p, 99L); // never registered + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject unregistered schema reference"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions schema id: " + expected.getMessage(), + expected.getMessage().contains("not registered")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testSymbolColumnDeltaModeAccumulatesAcrossBatches() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + // Batch 1: dict starts empty, add ["alpha", "beta"], 2 rows referencing them. + long p = staging; + p = writeHeader(p, QwpConstants.FLAG_DELTA_SYMBOL_DICT); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); // batch_seq + // Delta dict section: deltaStart=0, deltaCount=2, then len+bytes. + p = putVarint(p, 0L); + p = putVarint(p, 2L); + byte[] alpha = "alpha".getBytes(StandardCharsets.UTF_8); + byte[] beta = "beta".getBytes(StandardCharsets.UTF_8); + p = putVarint(p, alpha.length); + for (byte b : alpha) p = putByte(p, b); + p = putVarint(p, beta.length); + for (byte b : beta) p = putByte(p, b); + // Table block. + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 2L); // row_count + p = putVarint(p, 1L); // column_count + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); // schema_id + p = putVarint(p, 1L); // colName length + p = putByte(p, (byte) 's'); + p = putByte(p, QwpConstants.TYPE_SYMBOL); + // Column body: null_flag = 0, then per-row varint dict ids. + p = putByte(p, (byte) 0); + p = putVarint(p, 0L); // row 0 -> "alpha" + p = putVarint(p, 1L); // row 1 -> "beta" + + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals("alpha", batchOf(buffer).getSymbol(0, 0)); + Assert.assertEquals("beta", batchOf(buffer).getSymbol(0, 1)); + + // Batch 2: dict already has 2 entries; add 1 more ["gamma"], 1 row referencing it. + p = staging; + p = writeHeader(p, QwpConstants.FLAG_DELTA_SYMBOL_DICT); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 1L); + p = putVarint(p, 2L); // deltaStart = current size + p = putVarint(p, 1L); + byte[] gamma = "gamma".getBytes(StandardCharsets.UTF_8); + p = putVarint(p, gamma.length); + for (byte b : gamma) p = putByte(p, b); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putVarint(p, 1L); + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putByte(p, (byte) 's'); + p = putByte(p, QwpConstants.TYPE_SYMBOL); + p = putByte(p, (byte) 0); + p = putVarint(p, 2L); // row 0 -> "gamma" (id=2) + + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals("gamma", batchOf(buffer).getSymbol(0, 0)); + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testSymbolColumnNonDeltaInlineDictionary() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(512, (decoder, buffer, staging) -> { + long p = staging; + p = writeHeader(p, /*flags=*/ (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 3L); // row_count + p = putVarint(p, 1L); // column_count + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putByte(p, (byte) 's'); + p = putByte(p, QwpConstants.TYPE_SYMBOL); + // Non-delta SYMBOL: null_flag, dict_size varint, len+bytes per entry, then per-row ids. + p = putByte(p, (byte) 0); + p = putVarint(p, 2L); // dictSize + byte[] x = "xx".getBytes(StandardCharsets.UTF_8); + byte[] y = "yyyy".getBytes(StandardCharsets.UTF_8); + p = putVarint(p, x.length); + for (byte b : x) p = putByte(p, b); + p = putVarint(p, y.length); + for (byte b : y) p = putByte(p, b); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putVarint(p, 0L); + + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals("xx", batchOf(buffer).getSymbol(0, 0)); + Assert.assertEquals("yyyy", batchOf(buffer).getSymbol(0, 1)); + Assert.assertEquals("xx", batchOf(buffer).getSymbol(0, 2)); + })); + } + + @Test + public void testSymbolIndexOutOfRangeRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + long p = staging; + p = writeHeader(p, /*flags=*/ (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putVarint(p, 1L); + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putByte(p, (byte) 's'); + p = putByte(p, QwpConstants.TYPE_SYMBOL); + p = putByte(p, (byte) 0); + p = putVarint(p, 1L); // dictSize = 1 (only id 0 valid) + byte[] only = "only".getBytes(StandardCharsets.UTF_8); + p = putVarint(p, only.length); + for (byte b : only) p = putByte(p, b); + p = putVarint(p, 5L); // hostile id 5 + + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject symbol id beyond dictSize"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions symbol index: " + expected.getMessage(), + expected.getMessage().contains("symbol index")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testTimestampColumnGorillaEncoding() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + // Pre-encode timestamps via the matching encoder, then splice the + // result into a RESULT_BATCH frame whose flags advertise FLAG_GORILLA. + long[] values = {1_700_000_000L, 1_700_000_100L, 1_700_000_205L, + 1_700_000_320L, 1_700_000_440L}; + int srcLen = values.length * 8; + int gorillaCap = 256; + long src = Unsafe.malloc(srcLen, MemoryTag.NATIVE_DEFAULT); + long gorillaScratch = Unsafe.malloc(gorillaCap, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < values.length; i++) { + Unsafe.getUnsafe().putLong(src + (long) i * 8, values[i]); + } + QwpGorillaEncoder enc = new QwpGorillaEncoder(); + int gorillaWritten = enc.encodeTimestamps(gorillaScratch, gorillaCap, src, values.length); + + long p = staging; + p = writeHeader(p, QwpConstants.FLAG_GORILLA); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, values.length); + p = putVarint(p, 1L); + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putByte(p, (byte) 't'); + p = putByte(p, QwpConstants.TYPE_TIMESTAMP); + p = putByte(p, (byte) 0); // null_flag + p = putByte(p, (byte) 0x01); // encoding = Gorilla + for (int i = 0; i < gorillaWritten; i++) { + p = putByte(p, Unsafe.getUnsafe().getByte(gorillaScratch + i)); + } + + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + for (int i = 0; i < values.length; i++) { + Assert.assertEquals("row " + i, values[i], batchOf(buffer).getLongValue(0, i)); + } + } finally { + Unsafe.free(src, srcLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(gorillaScratch, gorillaCap, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testTimestampColumnRawWithoutGorillaFlag() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { + long[] values = {1_000_000L, 2_000_000L, 3_000_000L}; + long p = startSingleColumnFrame(staging, "t", QwpConstants.TYPE_TIMESTAMP, values.length); + p = putByte(p, (byte) 0); + for (long v : values) p = putLong(p, v); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + for (int i = 0; i < values.length; i++) { + Assert.assertEquals("row " + i, values[i], batchOf(buffer).getLongValue(0, i)); + } + })); + } + + @Test + public void testTimestampUnknownEncodingByteRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + long p = staging; + p = writeHeader(p, QwpConstants.FLAG_GORILLA); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 3L); + p = putVarint(p, 1L); + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); + p = putVarint(p, 1L); + p = putByte(p, (byte) 't'); + p = putByte(p, QwpConstants.TYPE_TIMESTAMP); + p = putByte(p, (byte) 0); // null_flag + p = putByte(p, (byte) 0x77); // bogus encoding byte + + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject unknown TIMESTAMP encoding byte"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions TIMESTAMP encoding: " + expected.getMessage(), + expected.getMessage().contains("TIMESTAMP encoding")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testTooManyArrayDimsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + long p = startSingleColumnFrame(staging, "a", QwpConstants.TYPE_LONG_ARRAY, 1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) (ColumnType.ARRAY_NDIMS_LIMIT + 1)); + + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject out-of-range ARRAY nDims"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions invalid dimensions: " + expected.getMessage(), + expected.getMessage().contains("invalid array dimensions")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testTruncatedFixedColumnRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + // Claim 3 LONG rows but supply only 2 longs of payload. + long p = startSingleColumnFrame(staging, "v", QwpConstants.TYPE_LONG, 3); + p = putByte(p, (byte) 0); + p = putLong(p, 1L); + p = putLong(p, 2L); + // (no third long) + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject truncated fixed-width column"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions truncation: " + expected.getMessage(), + expected.getMessage().contains("truncated")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testUnknownSchemaModeRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + long p = staging; + p = writeHeader(p, (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putByte(p, (byte) 0x42); // unknown schema_mode + p = putVarint(p, 0L); + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject unknown schema mode"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions schema mode: " + expected.getMessage(), + expected.getMessage().contains("schema mode")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testUnsupportedVersionRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(64); + long staging = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + long p = staging; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, (byte) 99); // future version client doesn't speak + for (int i = 5; i < 32; i++) p = putByte(p, (byte) 0); + buffer.copyFromPayload(staging, 32); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject unsupported version"); + } catch (QwpDecodeException expected) { + Assert.assertTrue(expected.getMessage().contains("unsupported version")); + } + } finally { + Unsafe.free(staging, 64, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testUnsupportedWireTypeRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + long p = startSingleColumnFrame(staging, "x", (byte) 0x7F, 1); + p = putByte(p, (byte) 0); + p = putLong(p, 0L); + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject unknown wire type"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions unsupported wire type: " + expected.getMessage(), + expected.getMessage().contains("unsupported wire type")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testUuidColumnDecodes() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { + int rows = 2; + long p = startSingleColumnFrame(staging, "u", QwpConstants.TYPE_UUID, rows); + p = putByte(p, (byte) 0); + // UUID 0: low=0x1122334455667788, high=0x99AABBCCDDEEFF00 + p = putLong(p, 0x1122334455667788L); + p = putLong(p, 0x99AABBCCDDEEFF00L); + // UUID 1: low=0, high=0 + p = putLong(p, 0L); + p = putLong(p, 0L); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(0x1122334455667788L, batchOf(buffer).getUuidLo(0, 0)); + Assert.assertEquals(0x99AABBCCDDEEFF00L, batchOf(buffer).getUuidHi(0, 0)); + Assert.assertEquals(0L, batchOf(buffer).getUuidLo(0, 1)); + Assert.assertEquals(0L, batchOf(buffer).getUuidHi(0, 1)); + })); + } + + @Test + public void testVarcharColumnRoundTrip() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(512, (decoder, buffer, staging) -> { + String[] strings = {"alpha", "", "héllo", "🚀"}; + long p = startSingleColumnFrame(staging, "s", QwpConstants.TYPE_VARCHAR, strings.length); + p = putByte(p, (byte) 0); // no nulls + int totalBytes = 0; + int[] offsets = new int[strings.length + 1]; + offsets[0] = 0; + byte[][] payloads = new byte[strings.length][]; + for (int i = 0; i < strings.length; i++) { + payloads[i] = strings[i].getBytes(StandardCharsets.UTF_8); + totalBytes += payloads[i].length; + offsets[i + 1] = totalBytes; + } + for (int o : offsets) p = putInt(p, o); + for (byte[] payload : payloads) { + for (byte b : payload) p = putByte(p, b); + } + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + + for (int i = 0; i < strings.length; i++) { + Assert.assertFalse(batchOf(buffer).isNull(0, i)); + String got = batchOf(buffer).getStrA(0, i).toString(); + Assert.assertEquals("row " + i, strings[i], got); + } + })); + } + + @Test + public void testZeroArrayDimsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + long p = startSingleColumnFrame(staging, "a", QwpConstants.TYPE_LONG_ARRAY, 1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); // nDims = 0 (null on the wire is signalled via the bitmap, not 0-dim) + + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject 0-dimensional ARRAY value"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions invalid dimensions: " + expected.getMessage(), + expected.getMessage().contains("invalid array dimensions")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Reaches into {@link QwpBatchBuffer} via reflection to grab the + * package-private {@code batch} view. Production callers receive the same + * batch via the {@code QwpColumnBatchHandler.onBatch} callback dispatched + * by {@code QwpEgressIoThread}; we don't have that path available in a + * unit test, so we read the field directly. + */ + private static QwpColumnBatch batchOf(QwpBatchBuffer buffer) { + try { + Field f = QwpBatchBuffer.class.getDeclaredField("batch"); + f.setAccessible(true); + return (QwpColumnBatch) f.get(buffer); + } catch (ReflectiveOperationException e) { + throw new AssertionError("could not access QwpBatchBuffer.batch via reflection", e); + } + } + + /** + * Wire-format helpers. + */ + private static long putByte(long p, byte v) { + Unsafe.getUnsafe().putByte(p, v); + return p + 1; + } + + private static long putInt(long p, int v) { + Unsafe.getUnsafe().putInt(p, v); + return p + 4; + } + + private static long putLong(long p, long v) { + Unsafe.getUnsafe().putLong(p, v); + return p + 8; + } + + private static long putShort(long p, short v) { + Unsafe.getUnsafe().putShort(p, v); + return p + 2; + } + + private static long putVarint(long p, long value) { + while ((value & ~0x7FL) != 0) { + Unsafe.getUnsafe().putByte(p++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(p++, (byte) (value & 0x7F)); + return p; + } + + /** + * Begins a single-column RESULT_BATCH frame and writes everything up to but + * NOT including the column body (caller writes null_flag + values). The + * returned position points at where the column body starts. + */ + private static long startSingleColumnFrame(long buf, String colName, byte wireType, int rowCount) { + return startSingleColumnFrame(buf, colName, wireType, rowCount, 0L); + } + + private static long startSingleColumnFrame(long buf, String colName, byte wireType, int rowCount, long schemaId) { + long p = buf; + p = writeHeader(p, (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); // batch_seq + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, rowCount); + p = putVarint(p, 1L); // column_count + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, schemaId); + byte[] nameBytes = colName.getBytes(StandardCharsets.UTF_8); + p = putVarint(p, nameBytes.length); + for (byte b : nameBytes) p = putByte(p, b); + p = putByte(p, wireType); + return p; + } + + private static void withDecoder(int bufferSize, BatchTest body) throws Exception { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(bufferSize); + long staging = Unsafe.malloc(bufferSize, MemoryTag.NATIVE_DEFAULT); + try { + body.run(decoder, buffer, staging); + } finally { + Unsafe.free(staging, bufferSize, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + } + + private static long writeHeader(long p, byte flags) { + // Magic (4) + version (1) + flags (1) + msg_kind-in-header unused (1) + // + table_count (1) + payload_length (4). + // The decoder reads flags at HEADER_OFFSET_FLAGS == 5, so flags MUST sit + // immediately after version. Putting them later (e.g. at offset 6) makes + // the decoder see flags == 0 and silently take the non-delta / non-gorilla + // / non-zstd path, leaving the delta/gorilla payload bytes at the position + // the table-block parser then advances over -- producing inscrutable + // "unknown schema mode 0x..." errors mid-frame. + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, flags); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, 0); + return p; + } + + @FunctionalInterface + private interface BatchTest { + void run(QwpResultBatchDecoder decoder, QwpBatchBuffer buffer, long staging) throws Exception; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java new file mode 100644 index 00000000..dc4ee05a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java @@ -0,0 +1,1208 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QueryEvent; +import io.questdb.client.cutlass.qwp.client.QwpBatchBuffer; +import io.questdb.client.cutlass.qwp.client.QwpDecodeException; +import io.questdb.client.cutlass.qwp.client.QwpEgressIoThread; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpResultBatchDecoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Hardening tests for {@link QwpResultBatchDecoder} against malformed RESULT_BATCH + * frames from a hostile or buggy server. Each test crafts a wire payload directly + * in native memory and asserts that the decoder rejects it cleanly with a + * {@link QwpDecodeException} rather than reading out of bounds, growing the + * schema registry without bound, or returning negative offsets that propagate + * into accessors. + */ +public class QwpResultBatchDecoderHardeningTest { + + /** + * Regression: an ARRAY column row whose dimension list contains a zero + * must be rejected. Without the fix the multiplicative cap is + * short-circuited the moment {@code elements} hits zero, so subsequent + * dimensions can hold any value (including {@code Integer.MAX_VALUE}) + * without the decoder ever firing the {@code MAX_ARRAY_ELEMENTS} + * guard. The encoder side never produces dl == 0 -- the existing + * nDims >= 1 check at the start of {@code parseArrayColumn} confirms + * the design intent that arrays are non-empty in every dimension. + */ + @Test + public void testArrayDimZeroIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeArrayResultBatchWithDims(staging, new int[]{0, 5}); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject ARRAY with dim == 0"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error must blame the ARRAY dim: " + expected.getMessage(), + expected.getMessage().contains("ARRAY dim")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Sanity: a well-formed single-row ARRAY column with all dimensions + * >= 1 still decodes successfully. Pins the wire layout so the + * dim-zero rejection above is testing the right code path rather + * than a generic frame-shape bug. + */ + @Test + public void testArrayValidDimensionsAreAccepted() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeArrayResultBatchWithDims(staging, new int[]{2, 3}); + buffer.copyFromPayload(staging, len); + decoder.decode(buffer); + // no exception => decoder accepts the well-formed wire bytes + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression: a delta SYMBOL dict entry whose length exceeds + * {@link Integer#MAX_VALUE} must be rejected. Prior to the fix, the + * per-entry guard checked {@code entryLen < 0} (on the long varint value) + * and {@code p + entryLen > limit}; neither catches a value that is + * positive-as-long but wraps to a negative int after the subsequent + * {@code (int) entryLen} cast, which then fed a negative length into + * {@code ensureConnDictHeapCapacity} (no-op) and finally into + * {@code copyMemory} (undefined behaviour). + */ + @Test + public void testDeltaSymbolDictHugeEntryLenIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + // 5-byte varint for 0x1_0000_0000L (2^32). That is positive as + // long but (int) wraps to 0. A wider check is needed to catch + // the int-cast hazard; a varint of 2^32 exceeds Integer.MAX_VALUE + // and must be rejected before the cast. + byte[] entryLenVarint = new byte[]{ + (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x10 + }; + int len = writeDeltaSymbolDictFrame(staging, entryLenVarint); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject delta dict entryLen > Integer.MAX_VALUE"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error must mention truncated delta symbol entry: " + expected.getMessage(), + expected.getMessage().contains("truncated delta symbol entry")); + } + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression: a hostile delta-symbol section that picks + * {@code (deltaStart, deltaCount)} so their long sum overflows negative + * must be rejected by the range guard, NOT silently bypassed and caught + * later by the secondary "out of sync" check (or, on a connection where + * deltaStart matches an already-grown connDictSize, written into native + * memory past connDictEntriesAddr). + *

+ * With deltaStart=4M and deltaCount=Long.MAX_VALUE: the additive form + * {@code deltaStart + deltaCount > MAX_CONN_DICT_SIZE} wraps to a + * negative long, fails the {@code > MAX} comparison, and lets the value + * fall through to the sync check (which, on a fresh decoder, throws + * "out of sync"). The fix validates each operand separately, so the + * range guard fires first with "out of range" -- which this test pins. + */ + @Test + public void testDeltaSymbolDictStartPlusCountOverflowIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeDeltaSymbolDictFrameWithRange(staging, + /*deltaStart=*/ 4L * 1024L * 1024L, + /*deltaCount=*/ Long.MAX_VALUE); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject overflowing delta range"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("range guard must fire before sync guard so the additive " + + "overflow is caught up-front: " + expected.getMessage(), + expected.getMessage().contains("out of range")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression: an EXEC_DONE frame whose {@code rows_affected} varint runs + * past the end of the payload (continuation bit set on the last byte we + * have, with no more bytes to follow) must be rejected as a transport + * failure. Prior to the fix the loop exited via {@code p < limit} + * without observing a clear continuation bit and emitted the + * partially-decoded value to the user as a successful EXEC_DONE. + */ + @Test + public void testExecDoneTruncatedRowsAffectedVarintIsRejected() throws Exception { + AtomicReference failure = new AtomicReference<>(); + TestUtils.assertMemoryLeak(() -> { + QwpEgressIoThread io = new QwpEgressIoThread(null, /*bufferPoolSize=*/ 2, + (status, message) -> failure.compareAndSet(null, message)); + try { + int cap = 64; + long buf = Unsafe.malloc(cap, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeExecDoneTruncatedRowsAffected(buf); + io.onBinaryMessage(buf, len); + } finally { + Unsafe.free(buf, cap, MemoryTag.NATIVE_DEFAULT); + } + } finally { + closePool(io); + } + }); + Assert.assertNotNull("terminal failure listener must fire on truncated EXEC_DONE varint", + failure.get()); + Assert.assertTrue("error must blame truncation: " + failure.get(), + failure.get().contains("truncated")); + } + + /** + * Regression: GEOHASH precisionBits on the wire must be in {@code [1, 60]}. + * A hostile varint decoding to 0, 61, or anything above 60 used to + * flow through as-is into {@code bytesPerValue = (precisionBits + 7) >>> 3}, + * generating nonsense bytesPerValue (e.g. 0 bytes per value for + * precision=0, or 8+ bytes for a large precision) that skewed the + * subsequent truncated-column check. + */ + @Test + public void testGeohashPrecisionAboveMaxIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeGeohashResultBatch(staging, /*precisionBits=*/ 61); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject GEOHASH precision above 60"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error must mention GEOHASH precision: " + expected.getMessage(), + expected.getMessage().contains("GEOHASH precision")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Complementary regression: precision 0 (below the [1, 60] range) must + * also be rejected. Without a lower-bound check the subsequent length + * computation degenerates to zero bytes per value, masking the corruption. + */ + @Test + public void testGeohashPrecisionBelowMinIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeGeohashResultBatch(staging, /*precisionBits=*/ 0); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject GEOHASH precision 0"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error must mention GEOHASH precision: " + expected.getMessage(), + expected.getMessage().contains("GEOHASH precision")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression for C5: a server-supplied {@code schema_id} above the per-connection + * cap must be rejected. Without the fix, {@code ensureSchemaSlot} would happily + * append nulls until OOM (or AIOOBE for negative ids cast from a high varint). + */ + @Test + public void testHugeSchemaIdIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + // schema_id = 1_000_000_000, well above the 65_535 cap. + int len = writeMinimalResultBatch(staging, /*schemaId=*/ 1_000_000_000L); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject huge schema_id"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error message should mention schema_id: " + expected.getMessage(), + expected.getMessage().contains("schema_id")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression for C5: a varint that long-to-int casts to a negative value + * (a hostile high varint with the sign bit set after the cast) must be + * rejected, not silently passed to {@code getQuick(negativeIndex)}. + */ + @Test + public void testNegativeSchemaIdIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + // 5-byte varint encoding 0x80000000 (which casts to Integer.MIN_VALUE). + // varint bytes for 0x80000000: + // value bits 7..0: 0x00 -> byte: 0x80 (continuation) + // value bits 14..8: 0x00 -> byte: 0x80 + // value bits 21..15:0x00 -> byte: 0x80 + // value bits 28..22:0x00 -> byte: 0x80 + // value bits 35..29:0x08 -> byte: 0x08 (no continuation) + int len = writeMinimalResultBatchWithRawSchemaIdVarint( + staging, new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x08}); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject huge/negative schema_id"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error message should mention schema_id: " + expected.getMessage(), + expected.getMessage().contains("schema_id")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression for C3: a hostile or buggy server can send a QUERY_ERROR frame + * that claims a 65535-byte message but supplies a tiny payload. Without the + * fix, the client reads up to ~65 KiB of native memory beyond the frame and + * surfaces it to the user callback as a String. With the fix, the client + * detects the overrun and reports a bounded error. + */ + @Test + public void testQueryErrorMsgLenOverrunIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Frame contents: + // 12 bytes header (uninspected by decodeError) + // 1 byte msg_kind + // 8 bytes request_id + // 1 byte status + // 2 bytes msgLen (we set 0xFFFF) + // 0 bytes of actual message body + // Total payload: 24 bytes; msgLen would otherwise force reading 65535 bytes. + int payloadLen = 12 + 1 + 8 + 1 + 2; + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Zero out + for (int i = 0; i < payloadLen; i++) Unsafe.getUnsafe().putByte(buf + i, (byte) 0); + // Write an obviously bogus msgLen at the right offset (header + msg_kind + reqId + status). + long msgLenOffset = buf + 12 + 1 + 8 + 1; + Unsafe.getUnsafe().putShort(msgLenOffset, (short) 0xFFFF); + + QueryEvent ev = QwpEgressIoThread.decodeError(buf, payloadLen); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertNotNull(ev.errorMessage); + Assert.assertTrue("error must mention msg_len overrun: " + ev.errorMessage, + ev.errorMessage.contains("msg_len") && ev.errorMessage.contains("exceeds")); + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + /** + * Regression for C3: a QUERY_ERROR frame with valid msgLen and bytes must be + * decoded correctly. Pins the wire format so the rejection test above is + * confirming a real defensive guard, not a broken decoder. + */ + @Test + public void testQueryErrorValidMessageDecodes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + byte[] msgBytes = "boom".getBytes(java.nio.charset.StandardCharsets.UTF_8); + int payloadLen = 12 + 1 + 8 + 1 + 2 + msgBytes.length; + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < payloadLen; i++) Unsafe.getUnsafe().putByte(buf + i, (byte) 0); + long statusOffset = buf + 12 + 1 + 8; + Unsafe.getUnsafe().putByte(statusOffset, (byte) 0x05); + long msgLenOffset = statusOffset + 1; + Unsafe.getUnsafe().putShort(msgLenOffset, (short) msgBytes.length); + long bytesOffset = msgLenOffset + 2; + for (int i = 0; i < msgBytes.length; i++) { + Unsafe.getUnsafe().putByte(bytesOffset + i, msgBytes[i]); + } + + QueryEvent ev = QwpEgressIoThread.decodeError(buf, payloadLen); + Assert.assertEquals(QueryEvent.KIND_ERROR, ev.kind); + Assert.assertEquals((byte) 0x05, ev.errorStatus); + Assert.assertEquals("boom", ev.errorMessage); + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + /** + * Regression: a RESULT_END frame whose {@code final_seq} varint is + * truncated (the frame ends before any byte with a clear continuation + * bit) must be rejected. The previous code's seq-skip loop exited on + * {@code p == limit} silently and the subsequent {@code total_rows} + * loop saw no bytes, so the user received a successful + * {@code asEnd(0)} for a malformed frame. + */ + @Test + public void testResultEndTruncatedFinalSeqVarintIsRejected() throws Exception { + AtomicReference failure = new AtomicReference<>(); + TestUtils.assertMemoryLeak(() -> { + QwpEgressIoThread io = new QwpEgressIoThread(null, /*bufferPoolSize=*/ 2, + (status, message) -> failure.compareAndSet(null, message)); + try { + int cap = 64; + long buf = Unsafe.malloc(cap, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeResultEndTruncatedFinalSeq(buf); + io.onBinaryMessage(buf, len); + } finally { + Unsafe.free(buf, cap, MemoryTag.NATIVE_DEFAULT); + } + } finally { + closePool(io); + } + }); + Assert.assertNotNull("terminal failure listener must fire on truncated RESULT_END final_seq", + failure.get()); + Assert.assertTrue("error must mention RESULT_END final_seq: " + failure.get(), + failure.get().contains("final_seq") && failure.get().contains("truncated")); + } + + /** + * Regression: a RESULT_END frame whose {@code total_rows} varint is + * truncated must be rejected as a transport failure rather than emitted + * as a successful end-of-result with a partially-decoded total. Before + * the fix, the old {@code decodeResultEnd} returned the partial value + * and the I/O thread happily delivered {@code asEnd(partial)} to the + * user. + */ + @Test + public void testResultEndTruncatedTotalRowsVarintIsRejected() throws Exception { + AtomicReference failure = new AtomicReference<>(); + TestUtils.assertMemoryLeak(() -> { + QwpEgressIoThread io = new QwpEgressIoThread(null, /*bufferPoolSize=*/ 2, + (status, message) -> failure.compareAndSet(null, message)); + try { + int cap = 64; + long buf = Unsafe.malloc(cap, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeResultEndTruncatedTotalRows(buf); + io.onBinaryMessage(buf, len); + } finally { + Unsafe.free(buf, cap, MemoryTag.NATIVE_DEFAULT); + } + } finally { + closePool(io); + } + }); + Assert.assertNotNull("terminal failure listener must fire on truncated RESULT_END total_rows", + failure.get()); + Assert.assertTrue("error must mention RESULT_END total_rows: " + failure.get(), + failure.get().contains("total_rows") && failure.get().contains("truncated")); + } + + /** + * Regression: a STRING/VARCHAR offset that is negative (cast from a + * hostile int32 with the sign bit set) must be rejected at parse time. + * Without per-offset validation, parseStringColumn only checks + * offset[nonNull]; the negative intermediate offset survives, and + * lookupStringBytes computes {@code stringBytesAddr + (negative)} -- a + * native address before the bytes region, leaking unrelated native + * memory through the user callback. + */ + @Test + public void testStringColumnNegativeIntermediateOffsetIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeStringResultBatchWithOffsets(staging, + new int[]{-16, 5}, + /*totalBytes=*/ 5, + /*bytesLen=*/ 5); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject negative intermediate offset"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error must blame an offset, not totalBytes: " + expected.getMessage(), + expected.getMessage().contains("offset[")); + } + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression for C4: STRING column with a negative {@code totalBytes} field. + * Without the fix, "stringBytesAddr + totalBytes > limit" passes (the sum + * stays below limit), and {@code parseStringColumn} returns a position + * before {@code stringBytesAddr} — subsequent column parsing reads native + * memory backwards. + */ + @Test + public void testStringColumnNegativeTotalBytesIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeStringResultBatch(staging, /*nonNull=*/ 1, /*totalBytes=*/ -1); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject negative totalBytes"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error message should describe invalid total bytes: " + expected.getMessage(), + expected.getMessage().contains("total bytes")); + } + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression: STRING/VARCHAR offsets must be monotonically non-decreasing. + * Without this, lookupBinaryBytes computes {@code endOff - startOff} as a + * negative int and {@code getBinary} surfaces NegativeArraySizeException. + * Worse, with a startOff in the safe zone and a tiny negative-cast + * endOff, the view's address range can wrap into unrelated native memory. + */ + @Test + public void testStringColumnNonMonotonicOffsetsAreRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeStringResultBatchWithOffsets(staging, + new int[]{3, 1}, + /*totalBytes=*/ 5, + /*bytesLen=*/ 5); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject non-monotonic offsets"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error must blame an offset: " + expected.getMessage(), + expected.getMessage().contains("offset[")); + } + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression: an intermediate offset that exceeds {@code totalBytes} must + * be rejected. Without it, the cell view at that row would read past the + * end of the bytes region into the next column or beyond the frame. + */ + @Test + public void testStringColumnOffsetExceedingTotalBytesIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + // offset[0] = 0, offset[1] = 99 (well past totalBytes = 5). + // offset[2] = totalBytes = 5 (but offset[1] > offset[2] also + // violates monotonicity). The intent of the test is the + // out-of-range value at offset[1]; the parser fires on the + // first violation, so accept either rejection wording. + int len = writeStringResultBatchWithOffsets(staging, + new int[]{0, 99}, + /*totalBytes=*/ 5, + /*bytesLen=*/ 5); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject offset > totalBytes"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error must blame an offset: " + expected.getMessage(), + expected.getMessage().contains("offset[")); + } + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Sanity: with a sane (non-negative, in-range) {@code totalBytes}, the same + * wire layout decodes successfully (no exception). Pins the wire format so + * the negative-value rejection above is testing the right code path. + */ + @Test + public void testStringColumnValidTotalBytesIsAccepted() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeStringResultBatch(staging, /*nonNull=*/ 1, /*totalBytes=*/ 5); + buffer.copyFromPayload(staging, len); + decoder.decode(buffer); + // no exception => the decoder accepts the valid wire bytes + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression: a non-delta SYMBOL column with a dict size above + * {@code rowCount} must be rejected. Prior to the fix, {@code dictSize} + * was read as a raw int with no bound -- a hostile varint could drive + * {@code dictSize * 8} to overflow silently, causing + * {@code ensureOwnedEntriesAddr} to skip the realloc (because the + * overflowed-negative requested size is less than any non-negative + * capacity) and the subsequent {@code Unsafe.putLong} to write past the + * allocated buffer. + */ + @Test + public void testSymbolColumnNonDeltaHugeDictSizeIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + // rowCount=1 column carrying dictSize=Integer.MAX_VALUE (which + // would drive dictSize*8 to overflow). Must reject on the bound + // rather than letting the overflowed allocation through. + int len = writeSymbolResultBatch(staging, /*rowCount=*/ 1, /*dictSize=*/ Integer.MAX_VALUE); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject SYMBOL dictSize above rowCount"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error must mention SYMBOL dict size: " + expected.getMessage(), + expected.getMessage().contains("SYMBOL dict size")); + } + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression for CR-3: the table {@code name_len} varint must reject + * negative-when-cast values (10-byte varint with bit 63 set on the final + * data byte). Without the fix, the bound check {@code p + nameLen > limit} + * passes (negative addend wraps {@code p} below {@code limit}), the + * decoder advances {@code p} backwards, and the next decode reads garbage + * from already-consumed bytes -- silent corruption rather than a clean + * rejection. + */ + @Test + public void testTableNameLengthOverflowVarintIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeMinimalResultBatchWithRawNameLenVarint(staging, + // 10-byte varint encoding 0x8000_0000_0000_0000 (Long.MIN_VALUE). + // Each 0x80 byte sets the continuation bit and contributes + // 7 zero data bits; the final 0x01 byte sets bit 63. + new byte[]{ + (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, + (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x01 + }); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject negative-when-cast nameLen"); + } catch (QwpDecodeException expected) { + String msg = expected.getMessage(); + Assert.assertTrue("error must blame varint overflow or table name length: " + msg, + msg.contains("varint overflow") || msg.contains("table name length")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Regression for CR-2: a RESULT_BATCH frame with FLAG_ZSTD set and a body + * that is not a valid zstd frame must be rejected via the up-front + * frame-header check, NOT by the old grow-the-scratch-and-retry loop. The + * old code would double the scratch up to the 64 MiB cap on any negative + * decompress return -- a single corrupt frame caused ~127 MiB of native + * malloc/free churn and pinned 64 MiB resident for the rest of the + * connection. The fix reads ZSTD_getFrameContentSize before allocating + * anything; an invalid frame returns -2 and the decoder throws. + */ + @Test + public void testZstdCorruptBodyIsRejectedBeforeScratchGrowth() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeResultBatchWithCorruptZstdBody(staging); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject corrupt zstd frame"); + } catch (QwpDecodeException expected) { + String msg = expected.getMessage(); + Assert.assertTrue("error must blame the frame header (rejected before decompress): " + msg, + msg.contains("frame header") || msg.contains("frame missing")); + } + } finally { + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + /** + * Reflective wrapper for the package-private {@code closePool()} on + * {@link QwpEgressIoThread}, used to free the constructor-allocated + * {@link QwpBatchBuffer} pool and ensure {@link TestUtils#assertMemoryLeak} + * sees a clean delta. Mirrors the same pattern used by + * {@link QwpEgressIoThreadCloseRaceTest}. + */ + private static void closePool(QwpEgressIoThread io) { + try { + Method m = QwpEgressIoThread.class.getDeclaredMethod("closePool"); + m.setAccessible(true); + m.invoke(io); + } catch (ReflectiveOperationException e) { + throw new AssertionError("could not invoke QwpEgressIoThread.closePool via reflection", e); + } + } + + /** + * Writes a single byte and returns the advanced position. Part of the + * wire-format helper set used to hand-craft a minimal RESULT_BATCH frame + * in native memory, matching {@code QwpResultBatchDecoder.decodePayload + + * parseStringColumn}: + * 

+     *   header (12 bytes)
+     *   msg_kind (0x11)
+     *   request_id (8 bytes)
+     *   batch_seq (varint)
+     *   table-block:
+     *     name_len (varint), name bytes (none)
+     *     row_count (varint)
+     *     column_count (varint)
+     *     schema_mode (1 byte) + schema_id (varint)
+     *     [if FULL] per column: name_len varint, name bytes, wire_type byte
+     *     per column: null_flag byte (+optional bitmap), then column body
+     *
+ */ + private static long putByte(long p, byte v) { + Unsafe.getUnsafe().putByte(p, v); + return p + 1; + } + + private static long putInt(long p, int v) { + Unsafe.getUnsafe().putInt(p, v); + return p + 4; + } + + private static long putLong(long p, long v) { + Unsafe.getUnsafe().putLong(p, v); + return p + 8; + } + + private static long putVarint(long p, long value) { + while ((value & ~0x7FL) != 0) { + Unsafe.getUnsafe().putByte(p++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(p++, (byte) (value & 0x7F)); + return p; + } + + /** + * Crafts a RESULT_BATCH carrying a single TYPE_DOUBLE_ARRAY column with + * one row whose dimensions are caller-supplied. Each dimension takes + * 4 bytes on the wire; the row's payload follows as + * {@code product(dims)} 8-byte slots, all zero. With {@code dims} + * containing a 0 entry the helper writes 0 payload bytes, exercising + * the dim-zero rejection in {@link QwpResultBatchDecoder}. + */ + private static int writeArrayResultBatchWithDims(long buf, int[] dims) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); // batch_seq + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 1L); // row_count = 1 + p = putVarint(p, 1L); // column_count + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); // schema_id + p = putVarint(p, 1L); // column name length + p = putByte(p, (byte) 'a'); + p = putByte(p, QwpConstants.TYPE_DOUBLE_ARRAY); + p = putByte(p, (byte) 0); // null_flag = 0 + p = putByte(p, (byte) dims.length); // nDims + long elements = 1; + for (int dl : dims) { + p = putInt(p, dl); + elements *= dl; + } + for (long e = 0; e < elements; e++) { + p = putLong(p, 0L); // zero-valued payload + } + return (int) (p - buf); + } + + /** + * Crafts a RESULT_BATCH frame with FLAG_DELTA_SYMBOL_DICT set and a + * delta-dict section of one entry whose length varint is supplied as + * raw bytes. No further table block is emitted; the decoder must fail + * in {@code parseDeltaSymbolDict} before reaching it. + */ + private static int writeDeltaSymbolDictFrame(long buf, byte[] entryLenVarint) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + // Flags byte lives at HEADER_OFFSET_FLAGS (=5). Existing helpers with + // flags=0 don't care about the exact slot; a delta-mode frame does. + p = putByte(p, QwpConstants.FLAG_DELTA_SYMBOL_DICT); + p = putByte(p, (byte) 0); // reserved slot at offset 6 + p = putByte(p, (byte) 1); // table_count + p = putInt(p, 0); // payload_length placeholder + p = putByte(p, (byte) 0x11); // msg_kind = RESULT_BATCH + p = putLong(p, 1L); // request_id + p = putVarint(p, 0L); // batch_seq + // Delta dict section: deltaStart=0, deltaCount=1, then the raw entry + // length varint the test wants to probe. + p = putVarint(p, 0L); + p = putVarint(p, 1L); + for (byte b : entryLenVarint) p = putByte(p, b); + return (int) (p - buf); + } + + /** + * Crafts a RESULT_BATCH frame with FLAG_DELTA_SYMBOL_DICT set, carrying + * caller-supplied {@code deltaStart} / {@code deltaCount} as varints. No + * entry payload follows; the decoder must fail in + * {@code parseDeltaSymbolDict} on the range guard before reaching it. + * Used to probe the range-guard for additive-overflow attacks. + */ + private static int writeDeltaSymbolDictFrameWithRange(long buf, long deltaStart, long deltaCount) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.FLAG_DELTA_SYMBOL_DICT); // flags at HEADER_OFFSET_FLAGS + p = putByte(p, (byte) 0); // reserved + p = putByte(p, (byte) 1); // table_count + p = putInt(p, 0); // payload_length placeholder + p = putByte(p, (byte) 0x11); // msg_kind = RESULT_BATCH + p = putLong(p, 1L); // request_id + p = putVarint(p, 0L); // batch_seq + p = putVarint(p, deltaStart); + p = putVarint(p, deltaCount); + return (int) (p - buf); + } + + /** + * Header (12 bytes) + msg_kind(1) + request_id(8) + op_type(1) + a single + * 0x80 byte with the continuation bit set and no terminator. The varint + * loop must reject this rather than emit the partial rows_affected value + * to the user. + */ + private static int writeExecDoneTruncatedRowsAffected(long buf) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, 0); + p = putByte(p, QwpEgressMsgKind.EXEC_DONE); + p = putLong(p, 1L); + p = putByte(p, (byte) 0); // op_type + p = putByte(p, (byte) 0x80); // continuation set, no terminator follows + return (int) (p - buf); + } + + /** + * Crafts a minimal RESULT_BATCH frame carrying a single GEOHASH column + * with {@code rowCount=0} and caller-chosen {@code precisionBits}. + * Because rowCount is zero, no per-row value bytes follow the precision + * varint; the decoder still decodes the precision up front and the range + * check must fire there. + */ + private static int writeGeohashResultBatch(long buf, long precisionBits) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); // header msg_kind (unused) + p = putByte(p, (byte) 0); // flags (no delta dict) + p = putByte(p, (byte) 1); // table_count + p = putInt(p, 0); // payload_length placeholder + p = putByte(p, (byte) 0x11); // msg_kind = RESULT_BATCH + p = putLong(p, 1L); // request_id + p = putVarint(p, 0L); // batch_seq + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 0L); // row_count (no data rows) + p = putVarint(p, 1L); // column_count + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); // schema_id + // Schema: one column "g" of TYPE_GEOHASH. + p = putVarint(p, 1L); + p = putByte(p, (byte) 'g'); + p = putByte(p, QwpConstants.TYPE_GEOHASH); + // Column body: null_flag=0 (no nulls), then precision_bits varint. + // With row_count=0 no value bytes follow. + p = putByte(p, (byte) 0); // null_flag + p = putVarint(p, precisionBits); + return (int) (p - buf); + } + + private static int writeMinimalResultBatch(long buf, long schemaId) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); // batch_seq + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 0L); // row_count = 0 (no body needed) + p = putVarint(p, 0L); // column_count = 0 + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, schemaId); + return (int) (p - buf); + } + + /** + * Variant of {@link #writeMinimalResultBatch} that injects a raw varint + * sequence for the table {@code name_len} field. Used by the CR-3 + * regression test to drive the negative-nameLen bound-check bypass. + */ + private static int writeMinimalResultBatchWithRawNameLenVarint(long buf, byte[] nameLenVarint) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); // batch_seq + for (byte b : nameLenVarint) p = putByte(p, b); + return (int) (p - buf); + } + + /** + * Variant that writes a custom raw varint sequence for schema_id. Lets us + * inject a multi-byte varint that decodes to a value with the int sign bit + * set after long-to-int truncation. + */ + private static int writeMinimalResultBatchWithRawSchemaIdVarint(long buf, byte[] schemaIdVarint) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + for (byte b : schemaIdVarint) p = putByte(p, b); + return (int) (p - buf); + } + + /** + * Crafts a RESULT_BATCH frame whose flags byte advertises FLAG_ZSTD but + * whose body is junk (not a valid zstd frame). Used by + * {@link #testZstdCorruptBodyIsRejectedBeforeScratchGrowth}. + */ + private static int writeResultBatchWithCorruptZstdBody(long buf) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.FLAG_ZSTD); // flags byte (offset 5) -- FLAG_ZSTD set + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, 0); + p = putByte(p, (byte) 0x11); // msg_kind = RESULT_BATCH + p = putLong(p, 1L); // request_id + p = putVarint(p, 0L); // batch_seq + // Following bytes claim to be a zstd frame but aren't -- not the magic + // (0x28 0xB5 0x2F 0xFD), no header, just a few junk bytes. + for (int i = 0; i < 16; i++) p = putByte(p, (byte) (0xAA ^ i)); + return (int) (p - buf); + } + + /** + * Header (12 bytes) + msg_kind(1) + request_id(8) + final_seq truncated + * (single 0x80 byte, frame ends before any clear-continuation byte). The + * final_seq skip loop must reject the truncation rather than fall + * through to total_rows with a desynced cursor. + */ + private static int writeResultEndTruncatedFinalSeq(long buf) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, 0); + p = putByte(p, QwpEgressMsgKind.RESULT_END); + p = putLong(p, 1L); + p = putByte(p, (byte) 0x80); // final_seq truncated mid-varint + return (int) (p - buf); + } + + /** + * Header (12 bytes) + msg_kind(1) + request_id(8) + final_seq(1 byte + * varint = 0) + total_rows truncated (single 0x80 byte). The total_rows + * loop must reject the truncation. + */ + private static int writeResultEndTruncatedTotalRows(long buf) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, 0); + p = putByte(p, QwpEgressMsgKind.RESULT_END); + p = putLong(p, 1L); + p = putVarint(p, 0L); // final_seq = 0 (single 0x00 byte) + p = putByte(p, (byte) 0x80); // total_rows truncated mid-varint + return (int) (p - buf); + } + + private static int writeStringResultBatch(long buf, int nonNull, int totalBytes) { + long p = buf; + // Header: magic + version + msg_kind + flags + table_count + payload_length + p = putInt(p, QwpConstants.MAGIC_MESSAGE); // 4 + p = putByte(p, QwpConstants.VERSION_1); // 1 + p = putByte(p, (byte) 0); // msg_kind in header (unused by client) + p = putByte(p, (byte) 0); // flags + p = putByte(p, (byte) 1); // table_count + p = putInt(p, 0); // payload_length placeholder (unused) + + // Body: + p = putByte(p, (byte) 0x11); // msg_kind = RESULT_BATCH + p = putLong(p, 7L); // request_id + p = putVarint(p, 0L); // batch_seq + p = putVarint(p, 0L); // table_name_len = 0 + p = putVarint(p, nonNull); // row_count + p = putVarint(p, 1L); // column_count + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); // schema_id + // Schema entries (full): one column "s" of TYPE_VARCHAR + p = putVarint(p, 1L); // column name length + p = putByte(p, (byte) 's'); + p = putByte(p, QwpConstants.TYPE_VARCHAR); + // Column body: null_flag = 0 (no nulls), offsets[nonNull+1] u32, then bytes. + p = putByte(p, (byte) 0); // null_flag + for (int i = 0; i < nonNull; i++) { + p = putInt(p, i * 5); // offset[i] + } + p = putInt(p, totalBytes); // offset[nonNull] = totalBytes + // Followed by 'totalBytes' string bytes — for the success case we write "hello" + // (5 bytes). For the negative-totalBytes case we still write 5 bytes; the + // decoder must reject before consuming them. + byte[] s = "hello".getBytes(java.nio.charset.StandardCharsets.UTF_8); + for (byte b : s) p = putByte(p, b); + return (int) (p - buf); + } + + /** + * Crafts a RESULT_BATCH carrying a single VARCHAR column with + * caller-supplied per-row offsets, an explicit {@code totalBytes}, and a + * payload tail of {@code bytesLen} arbitrary bytes. Lets tests probe the + * per-offset validation in {@link QwpResultBatchDecoder} (negative, + * non-monotonic, or out-of-range entries). + *

+ * {@code intOffsets.length} must equal the column's non-null row count; + * the helper appends offset[N] = totalBytes for the (N+1)th slot the + * decoder requires. + */ + private static int writeStringResultBatchWithOffsets(long buf, int[] intOffsets, int totalBytes, int bytesLen) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 1); + p = putInt(p, 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 7L); + p = putVarint(p, 0L); + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, intOffsets.length); // row_count = nonNull + p = putVarint(p, 1L); // column_count + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); // schema_id + p = putVarint(p, 1L); // column name length + p = putByte(p, (byte) 's'); + p = putByte(p, QwpConstants.TYPE_VARCHAR); + p = putByte(p, (byte) 0); // null_flag = 0 + for (int off : intOffsets) { + p = putInt(p, off); // offset[i] + } + p = putInt(p, totalBytes); // offset[nonNull] + for (int i = 0; i < bytesLen; i++) { + p = putByte(p, (byte) 'a'); + } + return (int) (p - buf); + } + + /** + * Crafts a RESULT_BATCH carrying a single SYMBOL column in non-delta mode + * (FLAG_DELTA_SYMBOL_DICT unset). The caller supplies the raw + * {@code dictSize} varint so bounds tests can push values that would + * otherwise flow through untouched. + */ + private static int writeSymbolResultBatch(long buf, long rowCount, long dictSize) { + long p = buf; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, (byte) 0); + p = putByte(p, (byte) 0); // flags: no delta dict, so non-delta SYMBOL path + p = putByte(p, (byte) 1); + p = putInt(p, 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); // batch_seq + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, rowCount); + p = putVarint(p, 1L); // column_count + p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); + p = putVarint(p, 0L); // schema_id + // Schema: one column "s" of TYPE_SYMBOL. + p = putVarint(p, 1L); + p = putByte(p, (byte) 's'); + p = putByte(p, QwpConstants.TYPE_SYMBOL); + // Column body: null_flag=0 (no nulls), then the dict_size varint. + // The decoder reads dict_size and must reject before attempting to + // decode dict_size entries (which don't even exist in this frame). + p = putByte(p, (byte) 0); + p = putVarint(p, dictSize); + return (int) (p - buf); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpServerInfoDecoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpServerInfoDecoderTest.java new file mode 100644 index 00000000..440fa101 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpServerInfoDecoderTest.java @@ -0,0 +1,328 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpDecodeException; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import io.questdb.client.cutlass.qwp.client.QwpServerInfoDecoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +/** + * Hardening + happy-path coverage for {@link QwpServerInfoDecoder}. Each test + * crafts a SERVER_INFO frame in native memory with a controlled layout, then + * asserts that the decoder either returns a {@link QwpServerInfo} matching the + * encoded fields or throws {@link QwpDecodeException} with a diagnostic that + * mentions the failing field. The frame layout is: + * 

+ *  [HEADER_SIZE bytes] [msg_kind:u8] [role:u8] [epoch:u64] [capabilities:u32]
+ *  [server_wall_ns:i64] [cluster_id_len:u16] [cluster_id_bytes]
+ *  [node_id_len:u16] [node_id_bytes]
+ *
+ */ +public class QwpServerInfoDecoderTest { + + /** + * Layout offsets after HEADER_SIZE bytes of opaque header. Mirrors + * {@link QwpServerInfoDecoder#decode(long, int)}. + */ + private static final int OFF_MSG_KIND = QwpConstants.HEADER_SIZE; + private static final int OFF_ROLE = OFF_MSG_KIND + 1; + private static final int OFF_EPOCH = OFF_ROLE + 1; + private static final int OFF_CAPABILITIES = OFF_EPOCH + 8; + private static final int OFF_SERVER_WALL_NS = OFF_CAPABILITIES + 4; + private static final int OFF_CLUSTER_ID_LEN = OFF_SERVER_WALL_NS + 8; + + @Test + public void testHappyPathRoundTrip() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String clusterId = "prod-cluster"; + String nodeId = "node-42"; + int len = totalLen(clusterId, nodeId); + long buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + try { + writeFrame(buf, QwpEgressMsgKind.ROLE_PRIMARY, 100L, 0xCAFEBABE, + 1_700_000_000_000_000_000L, clusterId, nodeId); + QwpServerInfo info = QwpServerInfoDecoder.decode(buf, len); + Assert.assertEquals(QwpEgressMsgKind.ROLE_PRIMARY, info.getRole()); + Assert.assertEquals(100L, info.getEpoch()); + Assert.assertEquals(0xCAFEBABE, info.getCapabilities()); + Assert.assertEquals(1_700_000_000_000_000_000L, info.getServerWallNs()); + Assert.assertEquals(clusterId, info.getClusterId()); + Assert.assertEquals(nodeId, info.getNodeId()); + } finally { + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testEmptyClusterAndNodeStrings() throws Exception { + TestUtils.assertMemoryLeak(() -> { + int len = totalLen("", ""); + long buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + try { + writeFrame(buf, QwpEgressMsgKind.ROLE_STANDALONE, 0L, 0, 0L, "", ""); + QwpServerInfo info = QwpServerInfoDecoder.decode(buf, len); + Assert.assertEquals("", info.getClusterId()); + Assert.assertEquals("", info.getNodeId()); + Assert.assertEquals(QwpEgressMsgKind.ROLE_STANDALONE, info.getRole()); + } finally { + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testUtf8MultiByteRoundTrip() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Mix ASCII, accented Latin, CJK, and an emoji to exercise 1/2/3/4-byte + // UTF-8 sequences through readUtf8U16's byte-by-byte copy. + String clusterId = "klïster-集群-🚀"; + String nodeId = "résumé-ु"; + int len = totalLen(clusterId, nodeId); + long buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + try { + writeFrame(buf, QwpEgressMsgKind.ROLE_REPLICA, 1L, 1, 1L, clusterId, nodeId); + QwpServerInfo info = QwpServerInfoDecoder.decode(buf, len); + Assert.assertEquals(clusterId, info.getClusterId()); + Assert.assertEquals(nodeId, info.getNodeId()); + } finally { + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testTruncatedBeforeFixedFieldsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // payloadLen too short to even fit the fixed prelude. + int len = QwpConstants.HEADER_SIZE + 1 + 1 + 8 + 4 + 8 + 1; // 1 byte short of fixedBytes + long buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + try { + zero(buf, len); + Unsafe.getUnsafe().putByte(buf + OFF_MSG_KIND, QwpEgressMsgKind.SERVER_INFO); + QwpServerInfoDecoder.decode(buf, len); + Assert.fail("decoder must reject undersized payload"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions truncation: " + expected.getMessage(), + expected.getMessage().contains("truncated") + && expected.getMessage().contains("payloadLen")); + } finally { + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testWrongMsgKindRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + int len = totalLen("c", "n"); + long buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + try { + writeFrame(buf, QwpEgressMsgKind.ROLE_PRIMARY, 0L, 0, 0L, "c", "n"); + // Stomp the msg_kind with something that isn't SERVER_INFO. + Unsafe.getUnsafe().putByte(buf + OFF_MSG_KIND, QwpEgressMsgKind.RESULT_BATCH); + QwpServerInfoDecoder.decode(buf, len); + Assert.fail("decoder must reject wrong msg_kind"); + } catch (QwpDecodeException expected) { + String msg = expected.getMessage(); + Assert.assertTrue("error mentions expected msg_kind: " + msg, + msg.contains("SERVER_INFO") || msg.contains(Integer.toHexString(QwpEgressMsgKind.SERVER_INFO & 0xFF))); + Assert.assertTrue("error mentions actual msg_kind hex: " + msg, + msg.contains(Integer.toHexString(QwpEgressMsgKind.RESULT_BATCH & 0xFF))); + } finally { + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testTruncatedBeforeClusterIdLengthRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // fixedBytes = HEADER_SIZE + 1+1+8+4+8+2 = OFF_CLUSTER_ID_LEN+2. Pass + // one byte less so the up-front fixed-prelude check rejects before + // ever reading the cluster_id length prefix. + int payloadLen = OFF_CLUSTER_ID_LEN + 1; + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + zero(buf, payloadLen); + Unsafe.getUnsafe().putByte(buf + OFF_MSG_KIND, QwpEgressMsgKind.SERVER_INFO); + QwpServerInfoDecoder.decode(buf, payloadLen); + Assert.fail("decoder must reject payload that ends before cluster_id length"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions truncation: " + expected.getMessage(), + expected.getMessage().contains("truncated")); + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testClusterIdLengthOvershootsPayloadRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Lay out a frame that claims a 5000-byte cluster_id but only carries 4. + int payloadLen = OFF_CLUSTER_ID_LEN + 2 + 4; // length prefix + 4 bytes + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + zero(buf, payloadLen); + Unsafe.getUnsafe().putByte(buf + OFF_MSG_KIND, QwpEgressMsgKind.SERVER_INFO); + Unsafe.getUnsafe().putShort(buf + OFF_CLUSTER_ID_LEN, (short) 5000); + QwpServerInfoDecoder.decode(buf, payloadLen); + Assert.fail("decoder must reject cluster_id length that overshoots remainder"); + } catch (QwpDecodeException expected) { + String msg = expected.getMessage(); + Assert.assertTrue("error mentions cluster_id: " + msg, msg.contains("cluster_id")); + Assert.assertTrue("error mentions overshoot length: " + msg, msg.contains("5000")); + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testNodeIdLengthOvershootsPayloadRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String clusterId = "ok"; + // Encode a normal cluster_id, then a node_id whose length prefix + // claims 9999 bytes but the payload ends after 1 byte. + int payloadLen = OFF_CLUSTER_ID_LEN + 2 + clusterId.length() + 2 + 1; + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + zero(buf, payloadLen); + Unsafe.getUnsafe().putByte(buf + OFF_MSG_KIND, QwpEgressMsgKind.SERVER_INFO); + int p = OFF_CLUSTER_ID_LEN; + byte[] cBytes = clusterId.getBytes(StandardCharsets.UTF_8); + Unsafe.getUnsafe().putShort(buf + p, (short) cBytes.length); + p += 2; + for (int i = 0; i < cBytes.length; i++) { + Unsafe.getUnsafe().putByte(buf + p + i, cBytes[i]); + } + p += cBytes.length; + Unsafe.getUnsafe().putShort(buf + p, (short) 9999); + QwpServerInfoDecoder.decode(buf, payloadLen); + Assert.fail("decoder must reject node_id length that overshoots remainder"); + } catch (QwpDecodeException expected) { + String msg = expected.getMessage(); + Assert.assertTrue("error mentions node_id: " + msg, msg.contains("node_id")); + Assert.assertTrue("error mentions overshoot length: " + msg, msg.contains("9999")); + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testTruncatedBeforeNodeIdLengthRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String clusterId = "abc"; + // Drop the buffer so the node_id length prefix has only 1 of its 2 bytes. + int payloadLen = OFF_CLUSTER_ID_LEN + 2 + clusterId.length() + 1; + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + zero(buf, payloadLen); + Unsafe.getUnsafe().putByte(buf + OFF_MSG_KIND, QwpEgressMsgKind.SERVER_INFO); + byte[] cBytes = clusterId.getBytes(StandardCharsets.UTF_8); + Unsafe.getUnsafe().putShort(buf + OFF_CLUSTER_ID_LEN, (short) cBytes.length); + for (int i = 0; i < cBytes.length; i++) { + Unsafe.getUnsafe().putByte(buf + OFF_CLUSTER_ID_LEN + 2 + i, cBytes[i]); + } + QwpServerInfoDecoder.decode(buf, payloadLen); + Assert.fail("decoder must reject payload that ends mid node_id length"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions node_id: " + expected.getMessage(), + expected.getMessage().contains("node_id")); + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testRoleByteIsCarriedThroughUnchanged() throws Exception { + // Decoder is intentionally tolerant of unknown role bytes -- it surfaces + // them via QwpServerInfo.roleName as UNKNOWN(n) rather than rejecting. + // Verify that here so a future refactor doesn't accidentally tighten it. + TestUtils.assertMemoryLeak(() -> { + int len = totalLen("c", "n"); + long buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + try { + writeFrame(buf, (byte) 0x77, 0L, 0, 0L, "c", "n"); + QwpServerInfo info = QwpServerInfoDecoder.decode(buf, len); + Assert.assertEquals((byte) 0x77, info.getRole()); + Assert.assertTrue(QwpServerInfo.roleName(info.getRole()).startsWith("UNKNOWN")); + } finally { + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static int totalLen(String clusterId, String nodeId) { + return OFF_CLUSTER_ID_LEN + + 2 + clusterId.getBytes(StandardCharsets.UTF_8).length + + 2 + nodeId.getBytes(StandardCharsets.UTF_8).length; + } + + private static void writeFrame(long buf, byte role, long epoch, int capabilities, + long serverWallNs, String clusterId, String nodeId) { + // Header bytes are not parsed by the SERVER_INFO decoder; zero them. + for (int i = 0; i < QwpConstants.HEADER_SIZE; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) 0); + } + Unsafe.getUnsafe().putByte(buf + OFF_MSG_KIND, QwpEgressMsgKind.SERVER_INFO); + Unsafe.getUnsafe().putByte(buf + OFF_ROLE, role); + Unsafe.getUnsafe().putLong(buf + OFF_EPOCH, epoch); + Unsafe.getUnsafe().putInt(buf + OFF_CAPABILITIES, capabilities); + Unsafe.getUnsafe().putLong(buf + OFF_SERVER_WALL_NS, serverWallNs); + + long p = buf + OFF_CLUSTER_ID_LEN; + byte[] cBytes = clusterId.getBytes(StandardCharsets.UTF_8); + Unsafe.getUnsafe().putShort(p, (short) cBytes.length); + p += 2; + for (byte cByte : cBytes) { + Unsafe.getUnsafe().putByte(p++, cByte); + } + byte[] nBytes = nodeId.getBytes(StandardCharsets.UTF_8); + Unsafe.getUnsafe().putShort(p, (short) nBytes.length); + p += 2; + for (byte nByte : nBytes) { + Unsafe.getUnsafe().putByte(p++, nByte); + } + } + + private static void zero(long buf, int len) { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) 0); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpServerInfoTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpServerInfoTest.java new file mode 100644 index 00000000..80b60882 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpServerInfoTest.java @@ -0,0 +1,124 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; +import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; +import io.questdb.client.cutlass.qwp.client.QwpServerInfo; +import org.junit.Assert; +import org.junit.Test; + +/** + * Pure data-class coverage for the {@link QwpServerInfo} accessors, + * the {@link QwpServerInfo#roleName(byte)} lookup, and the + * {@link QwpRoleMismatchException} getters. No I/O. + */ +public class QwpServerInfoTest { + + @Test + public void testAccessorsReturnConstructorArgs() { + QwpServerInfo info = new QwpServerInfo( + QwpEgressMsgKind.ROLE_PRIMARY, + 42L, + 0xCAFEBABE, + 1_700_000_000_000_000_000L, + "cluster-x", + "node-7" + ); + Assert.assertEquals(QwpEgressMsgKind.ROLE_PRIMARY, info.getRole()); + Assert.assertEquals(42L, info.getEpoch()); + Assert.assertEquals(0xCAFEBABE, info.getCapabilities()); + Assert.assertEquals(1_700_000_000_000_000_000L, info.getServerWallNs()); + Assert.assertEquals("cluster-x", info.getClusterId()); + Assert.assertEquals("node-7", info.getNodeId()); + } + + @Test + public void testRoleNameAllKnownValues() { + Assert.assertEquals("STANDALONE", QwpServerInfo.roleName(QwpEgressMsgKind.ROLE_STANDALONE)); + Assert.assertEquals("PRIMARY", QwpServerInfo.roleName(QwpEgressMsgKind.ROLE_PRIMARY)); + Assert.assertEquals("REPLICA", QwpServerInfo.roleName(QwpEgressMsgKind.ROLE_REPLICA)); + Assert.assertEquals("PRIMARY_CATCHUP", QwpServerInfo.roleName(QwpEgressMsgKind.ROLE_PRIMARY_CATCHUP)); + } + + @Test + public void testRoleNameUnknownReturnsTaggedDecimal() { + // 0xFE = 254 unsigned -- exercises the (role & 0xFF) widening so the + // diagnostic doesn't print -2. + Assert.assertEquals("UNKNOWN(254)", QwpServerInfo.roleName((byte) 0xFE)); + Assert.assertEquals("UNKNOWN(99)", QwpServerInfo.roleName((byte) 99)); + } + + @Test + public void testToStringContainsAllFields() { + QwpServerInfo info = new QwpServerInfo( + QwpEgressMsgKind.ROLE_REPLICA, + 7L, + 0x10, + 1234L, + "myCluster", + "myNode" + ); + String s = info.toString(); + Assert.assertTrue("expected role name in toString: " + s, s.contains("REPLICA")); + Assert.assertTrue("expected epoch in toString: " + s, s.contains("epoch=7")); + Assert.assertTrue("expected clusterId in toString: " + s, s.contains("myCluster")); + Assert.assertTrue("expected nodeId in toString: " + s, s.contains("myNode")); + Assert.assertTrue("expected hex capabilities in toString: " + s, s.contains("0x10")); + Assert.assertTrue("expected serverWallNs in toString: " + s, s.contains("1234")); + } + + @Test + public void testToStringHandlesUnknownRole() { + QwpServerInfo info = new QwpServerInfo((byte) 99, 0L, 0, 0L, "", ""); + Assert.assertTrue(info.toString().contains("UNKNOWN(99)")); + } + + @Test + public void testRoleMismatchExceptionGetters() { + QwpServerInfo lastObserved = new QwpServerInfo( + QwpEgressMsgKind.ROLE_REPLICA, 1L, 0, 0L, "c", "n"); + QwpRoleMismatchException ex = new QwpRoleMismatchException( + "primary", lastObserved, "no primary endpoint available"); + + Assert.assertEquals("primary", ex.getTargetRole()); + Assert.assertSame(lastObserved, ex.getLastObserved()); + Assert.assertTrue(ex.getMessage().contains("no primary endpoint available")); + } + + @Test + public void testRoleMismatchExceptionIsHttpClientException() { + QwpRoleMismatchException ex = new QwpRoleMismatchException("any", null, "msg"); + // The static-typed assignment below would fail to compile if + // QwpRoleMismatchException stopped extending HttpClientException -- + // a load-bearing relationship since callers catch the parent type. + // A runtime instanceof check would always be true here, so it adds + // nothing; this compile-time check is the meaningful guard. + @SuppressWarnings("unused") HttpClientException base = ex; + Assert.assertNull("nullable lastObserved must round-trip", ex.getLastObserved()); + Assert.assertEquals("any", ex.getTargetRole()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSpscQueueTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSpscQueueTest.java new file mode 100644 index 00000000..50a930b1 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSpscQueueTest.java @@ -0,0 +1,221 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.QwpSpscQueue; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Unit coverage for the lock-free SPSC queue used to hand events from the + * I/O thread to the user thread inside {@link io.questdb.client.cutlass.qwp.client.QwpQueryClient}. + * The queue's correctness rests on three behaviours: + *
    + *
  • {@code offer} returns false when the ring is full;
    • + *
  • {@code take} progresses through fast-poll → spin → park;
    • + *
  • a producer that {@code offer}s while the consumer is parked must call + * {@code LockSupport.unpark} — guarded by {@code consumerThread}.
    • + *
+ * Each test couples a producer and a consumer thread with a small {@link CountDownLatch} + * so the timing of the offer is observable from the test thread. + */ +public class QwpSpscQueueTest { + + @Test + public void testOfferAndPollFastPath() { + QwpSpscQueue q = new QwpSpscQueue<>(4); + Assert.assertNull("fresh queue must be empty", q.poll()); + Assert.assertTrue(q.offer("a")); + Assert.assertTrue(q.offer("b")); + Assert.assertEquals("a", q.poll()); + Assert.assertEquals("b", q.poll()); + Assert.assertNull("queue must be empty after draining", q.poll()); + } + + @Test + public void testCapacityRoundedUpToPowerOfTwo() { + // Requesting capacity 5 must round up to 8; we should be able to offer 8 items. + QwpSpscQueue q = new QwpSpscQueue<>(5); + for (int i = 0; i < 8; i++) { + Assert.assertTrue("expected ring size 8 after rounding 5 up, failed at i=" + i, q.offer(i)); + } + Assert.assertFalse("offer past power-of-two capacity must fail", q.offer(8)); + for (int i = 0; i < 8; i++) { + Assert.assertEquals(Integer.valueOf(i), q.poll()); + } + } + + @Test + public void testOfferReturnsFalseWhenFull() { + QwpSpscQueue q = new QwpSpscQueue<>(2); + Assert.assertTrue(q.offer(1)); + Assert.assertTrue(q.offer(2)); + Assert.assertFalse("ring of 2 must reject the third offer", q.offer(3)); + Assert.assertEquals(Integer.valueOf(1), q.poll()); + Assert.assertTrue("freed slot must be reusable", q.offer(3)); + } + + @Test + public void testTakeFastPathReturnsImmediately() throws InterruptedException { + QwpSpscQueue q = new QwpSpscQueue<>(4); + // Producer publishes before consumer asks -- no spin or park. + q.offer("ready"); + long start = System.nanoTime(); + Assert.assertEquals("ready", q.take()); + long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + Assert.assertTrue("take must return immediately when value present, took " + elapsedMs + " ms", + elapsedMs < 50); + } + + @Test + public void testTakeBlocksUntilProducerOffers() throws Exception { + QwpSpscQueue q = new QwpSpscQueue<>(4); + AtomicReference taken = new AtomicReference<>(); + CountDownLatch consumerStarted = new CountDownLatch(1); + CountDownLatch consumerDone = new CountDownLatch(1); + + Thread consumer = new Thread(() -> { + try { + consumerStarted.countDown(); + taken.set(q.take()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + consumerDone.countDown(); + } + }, "spsc-test-consumer"); + consumer.start(); + + Assert.assertTrue("consumer must start", consumerStarted.await(2, TimeUnit.SECONDS)); + // Sleep a touch longer than the spin window so the consumer is parked, + // exercising the unpark-from-offer path. SPIN_ITERATIONS is ~50 µs; + // 100 ms is overkill but safe under load. + Thread.sleep(100); + Assert.assertNull("consumer must not have completed before offer", taken.get()); + + Assert.assertTrue(q.offer("delivered")); + Assert.assertTrue("consumer must complete after offer + unpark", + consumerDone.await(2, TimeUnit.SECONDS)); + Assert.assertEquals("delivered", taken.get()); + consumer.join(1000); + } + + @Test + public void testTakeInterruptedDuringParkThrowsAndStaysAlive() throws Exception { + QwpSpscQueue q = new QwpSpscQueue<>(4); + AtomicReference caught = new AtomicReference<>(); + CountDownLatch consumerStarted = new CountDownLatch(1); + CountDownLatch consumerDone = new CountDownLatch(1); + + Thread consumer = new Thread(() -> { + try { + consumerStarted.countDown(); + q.take(); + caught.set(new AssertionError("expected InterruptedException")); + } catch (Throwable t) { + caught.set(t); + } finally { + consumerDone.countDown(); + } + }, "spsc-test-consumer-interrupt"); + consumer.start(); + + Assert.assertTrue(consumerStarted.await(2, TimeUnit.SECONDS)); + // Wait past the spin window so the consumer is in the park loop. + Thread.sleep(100); + consumer.interrupt(); + Assert.assertTrue("interrupt must release the parked consumer", + consumerDone.await(2, TimeUnit.SECONDS)); + Assert.assertTrue("expected InterruptedException, got " + caught.get(), + caught.get() instanceof InterruptedException); + consumer.join(1000); + } + + @Test + public void testProducerConsumerStreamRoundTrip() throws Exception { + // Larger end-to-end workload to exercise the wrap-around mask path + // (head and tail walk past Integer.MAX_VALUE within the (h & mask) cast). + QwpSpscQueue q = new QwpSpscQueue<>(8); + final int items = 5_000; + int[] received = new int[items]; + CountDownLatch done = new CountDownLatch(1); + + Thread consumer = new Thread(() -> { + try { + for (int i = 0; i < items; i++) { + received[i] = q.take(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + done.countDown(); + } + }, "spsc-stream-consumer"); + consumer.start(); + + for (int i = 0; i < items; i++) { + // Spin if the ring is full -- mirrors the real producer's behaviour + // when it ran into back-pressure from a slow consumer. + while (!q.offer(i)) { + Thread.yield(); + } + } + + Assert.assertTrue("consumer must finish the stream", done.await(5, TimeUnit.SECONDS)); + consumer.join(1000); + + for (int i = 0; i < items; i++) { + Assert.assertEquals("FIFO order must hold for SPSC", i, received[i]); + } + Assert.assertNull("queue must be empty after stream", q.poll()); + } + + @Test + public void testNoUnparkWhenConsumerNotParked() { + // offer() consults consumerThread; when it's null (consumer running), + // no unpark should occur -- this is the cheap fast path. Behaviourally + // we can only verify that offer() doesn't throw and the value is delivered. + QwpSpscQueue q = new QwpSpscQueue<>(4); + Assert.assertTrue(q.offer("x")); + Assert.assertTrue(q.offer("y")); + Assert.assertEquals("x", q.poll()); + Assert.assertEquals("y", q.poll()); + Assert.assertNull(q.poll()); + } + + @Test + public void testCapacityOneIsRoundedToOne() { + // Edge case: requesting capacity 1 still yields a power-of-two ring (1). + QwpSpscQueue q = new QwpSpscQueue<>(1); + Assert.assertTrue(q.offer("only")); + Assert.assertFalse("size-1 ring rejects a second offer", q.offer("nope")); + Assert.assertEquals("only", q.poll()); + Assert.assertTrue(q.offer("again")); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitReaderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitReaderTest.java new file mode 100644 index 00000000..790e1d2a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpBitReaderTest.java @@ -0,0 +1,390 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.client.QwpDecodeException; +import io.questdb.client.cutlass.qwp.protocol.QwpBitReader; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +/** + * Bit-level coverage for {@link QwpBitReader}, the LSB-first reader the QWP + * Gorilla decoder is built on. Bytes are pushed into the buffer LSB-first, so + * for the byte {@code 0b1010_0001} the reader yields {@code 1, 0, 0, 0, 0, 1, 0, 1} + * — bit 0 first. We craft small native byte arrays, exercise every path that + * the decoder hits in production (single-bit, multi-bit, refill, sign-extend, + * end-of-stream), and assert the bit position advances correctly so callers + * downstream can detect truncated columns. + */ +public class QwpBitReaderTest { + + @Test + public void testReadBitPastEndThrows() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0xFF); + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 1); + for (int i = 0; i < 8; i++) { + r.readBit(); + } + try { + r.readBit(); + Assert.fail("readBit past end must throw"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions read past end: " + expected.getMessage(), + expected.getMessage().contains("read past end")); + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadBitYieldsLsbFirstAcrossMultipleBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // 0b10100001 then 0b00000010 -- bits LSB-first: + // byte 0: 1 0 0 0 0 1 0 1 + // byte 1: 0 1 0 0 0 0 0 0 + int len = 2; + long buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0b10100001); + Unsafe.getUnsafe().putByte(buf + 1, (byte) 0b00000010); + QwpBitReader r = new QwpBitReader(); + r.reset(buf, len); + int[] expected = {1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0}; + for (int i = 0; i < expected.length; i++) { + Assert.assertEquals("bit " + i, expected[i], r.readBit()); + Assert.assertEquals("position after reading " + (i + 1) + " bits", + i + 1, r.getBitPosition()); + } + } finally { + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadBits64ReadsFullWord() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Eight bytes assembling, LSB-first, the long 0x0123456789ABCDEFL. + long buf = Unsafe.malloc(8, MemoryTag.NATIVE_DEFAULT); + try { + long value = 0x0123456789ABCDEFL; + for (int i = 0; i < 8; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) ((value >>> (8 * i)) & 0xFF)); + } + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 8); + long read = r.readBits(64); + Assert.assertEquals("64-bit read must hit the mask==-1L branch and reproduce input", + value, read); + Assert.assertEquals(64L, r.getBitPosition()); + } finally { + Unsafe.free(buf, 8, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + /** + * Regression: a full-width {@code readBits(64)} must clear the bit buffer + * so the next read sees a clean slate. Java masks the right operand of + * {@code >>>} by 0x3F for {@code long}, making {@code bitBuffer >>>= 64} + * a no-op. Without the special-case in the reader, the previous 64 bits + * remain in {@code bitBuffer}; the next ensureBits OR-fills a fresh byte + * at bit 0 of that stale buffer, silently corrupting every subsequent + * read. + *

+ * The two halves of the buffer are deliberately disjoint: 8 bytes of + * {@code 0xFF} followed by 8 bytes of {@code 0x00}. After reading the + * first 64 bits ({@code 0xFFFF_FFFF_FFFF_FFFFL}), the reader must report + * the second 64 bits as exactly {@code 0L}; today's bug surfaces them as + * {@code -1L} because the stale all-ones buffer is OR'd with the + * incoming zeros. + */ + @Test + public void testReadBits64TwiceDoesNotLeakStaleBuffer() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 8; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) 0xFF); + } + for (int i = 8; i < 16; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) 0x00); + } + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 16); + + long first = r.readBits(64); + Assert.assertEquals(-1L, first); + Assert.assertEquals(64L, r.getBitPosition()); + + long second = r.readBits(64); + Assert.assertEquals( + "second readBits(64) must reflect the bytes that follow, " + + "not the stale buffer left by the no-op shift", + 0L, second); + Assert.assertEquals(128L, r.getBitPosition()); + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadBitsAcrossLargeRefill() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // 16 bytes: read in arbitrary widths totalling 128 bits, verify position + // and that no exception fires. Smoke test for the refill loop's exit. + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 16; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) (i & 0xFF)); + } + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 16); + int[] widths = {1, 7, 13, 19, 23, 33, 32}; // sums to 128 + long totalBits = 0; + for (int w : widths) { + r.readBits(w); + totalBits += w; + Assert.assertEquals(totalBits, r.getBitPosition()); + } + // Now exhausted. + try { + r.readBit(); + Assert.fail("read past 16*8 bits must throw"); + } catch (QwpDecodeException expected) { + // expected + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadBitsArbitraryWidths() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Buffer: 0xFF 0x55 0xAA 0x00 -- 32 bits of mixed pattern. + long buf = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0xFF); + Unsafe.getUnsafe().putByte(buf + 1, (byte) 0x55); + Unsafe.getUnsafe().putByte(buf + 2, (byte) 0xAA); + Unsafe.getUnsafe().putByte(buf + 3, (byte) 0x00); + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 4); + + Assert.assertEquals(0b11111L, r.readBits(5)); // 5 bits from 0xFF + Assert.assertEquals(0b111L, r.readBits(3)); // remaining 3 bits of 0xFF + Assert.assertEquals(8L, r.getBitPosition()); + Assert.assertEquals(0x55L, r.readBits(8)); // whole 0x55 + Assert.assertEquals(16L, r.getBitPosition()); + // LSB-first: byte 0xAA contributes bits 0-7 of the result, byte 0x00 contributes bits 8-15. + Assert.assertEquals(0x00AAL, r.readBits(16)); + Assert.assertEquals(32L, r.getBitPosition()); + } finally { + Unsafe.free(buf, 4, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadBitsMoreThan64Throws() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 16; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) 0); + } + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 16); + try { + r.readBits(65); + Assert.fail("readBits(>64) must hit the AssertionError guard"); + } catch (AssertionError expected) { + Assert.assertTrue("error mentions 64-bit cap: " + expected.getMessage(), + expected.getMessage().contains("64")); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadBitsPastEndThrows() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // 1 byte = 8 bits available; asking for 9 must throw before any reads. + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0xFF); + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 1); + try { + r.readBits(9); + Assert.fail("readBits beyond available must throw"); + } catch (QwpDecodeException expected) { + Assert.assertTrue(expected.getMessage().contains("read past end")); + } + // Position should not have advanced. + Assert.assertEquals(0L, r.getBitPosition()); + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadBitsSpansBufferRefills() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Reading 24 bits in one call forces the inner ensureBits loop to + // refill across at least two boundary points, since the bit buffer + // is empty at start and refills 8 bits at a time. + long buf = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0x01); + Unsafe.getUnsafe().putByte(buf + 1, (byte) 0x02); + Unsafe.getUnsafe().putByte(buf + 2, (byte) 0x03); + Unsafe.getUnsafe().putByte(buf + 3, (byte) 0x00); + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 4); + long v = r.readBits(24); + // LSB-first: 0x01 | (0x02 << 8) | (0x03 << 16) = 0x030201 + Assert.assertEquals(0x030201L, v); + Assert.assertEquals(24L, r.getBitPosition()); + } finally { + Unsafe.free(buf, 4, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadBitsZeroBitsReturnsZeroWithoutAdvancing() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0xFF); + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 1); + Assert.assertEquals(0L, r.readBits(0)); + Assert.assertEquals(0L, r.getBitPosition()); + // Subsequent read still sees the byte intact. + Assert.assertEquals(1, r.readBit()); + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadSigned64BitsBehavesLikeReadBits() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Special-case in readSigned: numBits == 64 skips the sign-extend branch + // because the value already occupies the full long. + long buf = Unsafe.malloc(8, MemoryTag.NATIVE_DEFAULT); + try { + long value = 0xFFEEDDCCBBAA9988L; // negative as signed long + for (int i = 0; i < 8; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) ((value >>> (8 * i)) & 0xFF)); + } + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 8); + Assert.assertEquals(value, r.readSigned(64)); + } finally { + Unsafe.free(buf, 8, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadSignedDoesNotExtendWhenMsbClear() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Encode +5 in 5 bits (0b00101). MSB clear -> no sign extension. + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0b00000101); + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 1); + Assert.assertEquals(5L, r.readSigned(5)); + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testReadSignedExtendsWhenMsbSet() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Encode -1 in 5 bits (0b11111). Sign-extend should yield -1L. + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(buf, (byte) 0b00011111); + QwpBitReader r = new QwpBitReader(); + r.reset(buf, 1); + Assert.assertEquals(-1L, r.readSigned(5)); + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testResetClearsAllState() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long buf1 = Unsafe.malloc(2, MemoryTag.NATIVE_DEFAULT); + long buf2 = Unsafe.malloc(2, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(buf1, (byte) 0xAB); + Unsafe.getUnsafe().putByte(buf1 + 1, (byte) 0xCD); + Unsafe.getUnsafe().putByte(buf2, (byte) 0x12); + Unsafe.getUnsafe().putByte(buf2 + 1, (byte) 0x34); + + QwpBitReader r = new QwpBitReader(); + r.reset(buf1, 2); + r.readBits(10); + Assert.assertEquals(10L, r.getBitPosition()); + + // Reset to a fresh buffer; position must drop back to 0 and the + // first read must come from buf2, not the leftover bit buffer. + r.reset(buf2, 2); + Assert.assertEquals(0L, r.getBitPosition()); + Assert.assertEquals(0x12L, r.readBits(8)); + Assert.assertEquals(8L, r.getBitPosition()); + } finally { + Unsafe.free(buf1, 2, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(buf2, 2, MemoryTag.NATIVE_DEFAULT); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaDecoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaDecoderTest.java new file mode 100644 index 00000000..4bfede58 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpGorillaDecoderTest.java @@ -0,0 +1,291 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.client.QwpDecodeException; +import io.questdb.client.cutlass.qwp.protocol.QwpGorillaDecoder; +import io.questdb.client.cutlass.qwp.protocol.QwpGorillaEncoder; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Random; + +/** + * Coverage for {@link QwpGorillaDecoder}. We round-trip timestamp sequences + * through the matching server-side encoder ({@link QwpGorillaEncoder}) — the + * encoder is already heavily covered by {@link QwpGorillaEncoderTest}, so it + * doubles as a known-good wire-format generator that lets us focus on the + * decoder's read paths. Each test exercises one of the five DoD buckets the + * encoder picks based on the magnitude of the delta-of-delta value. + */ +public class QwpGorillaDecoderTest { + + @Test + public void testZeroDodBucket() throws Exception { + // Constant-stride sequence -> every DoD is 0 -> 1-bit prefix path. + roundTrip(new long[]{1000L, 2000L, 3000L, 4000L, 5000L, 6000L}); + } + + @Test + public void test7BitBucket() throws Exception { + // DoDs in [-64, 63]. Use small jitter on a stable stride. + roundTrip(new long[]{1000L, 2000L, 3010L, 4030L, 5040L, 6050L}); + } + + @Test + public void test9BitBucket() throws Exception { + // DoDs that overflow 7-bit but fit in 9-bit: aim for ~|200|. + roundTrip(new long[]{1000L, 2000L, 3200L, 4600L, 6200L, 7900L}); + } + + @Test + public void test12BitBucket() throws Exception { + // DoDs in (255, 2047]. Multi-thousand-scale jitter. + roundTrip(new long[]{1000L, 2000L, 4000L, 8000L, 13800L, 19500L, 25500L}); + } + + @Test + public void test32BitFallback() throws Exception { + // DoD beyond 12-bit bucket -> 36-bit fallback path. + roundTrip(new long[]{1_000_000L, 2_000_000L, 3_500_000L, 7_000_000L, 12_000_000L}); + } + + @Test + public void testNegativeDodSignExtension() throws Exception { + // Stride that decreases -> negative DoDs -> exercises sign-extension in readSigned. + roundTrip(new long[]{0L, 10_000L, 19_900L, 29_700L, 39_400L, 48_950L}); + } + + @Test + public void testMixedBuckets() throws Exception { + // A heterogeneous sequence forcing the decoder to walk every prefix path + // within a single column. Specifically: + // delta sequence 1000, 1000, 1010, 1210, 5210, 5208 + // DoD sequence 0, 10, 200, 4000, -2 + long[] timestamps = {0, 1000L, 2000L, 3010L, 4220L, 9430L, 14_638L}; + roundTrip(timestamps); + } + + @Test + public void testRandomLongSequenceRoundTrips() throws Exception { + // Stress-test: 1000 timestamps with a noisy walk. Validates that the + // decoder produces bit-identical output to the encoder's input across + // a wide range of bucket transitions and ensureBits refills. + Random rng = new Random(0xC0FFEE); + long[] timestamps = new long[1000]; + long t = 1_700_000_000_000_000L; + long stride = 100; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = t; + // jitter the stride to drive different buckets + stride += rng.nextInt(2001) - 1000; + t += Math.max(1, stride); + } + roundTrip(timestamps); + } + + @Test + public void testGetBitPositionPropagatesFromBitReader() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long[] timestamps = {0L, 100L, 200L, 300L}; + int srcLen = timestamps.length * 8; + long src = Unsafe.malloc(srcLen, MemoryTag.NATIVE_DEFAULT); + int destCap = 256; + long dest = Unsafe.malloc(destCap, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < timestamps.length; i++) { + Unsafe.getUnsafe().putLong(src + (long) i * 8, timestamps[i]); + } + QwpGorillaEncoder enc = new QwpGorillaEncoder(); + int written = enc.encodeTimestamps(dest, destCap, src, timestamps.length); + // First two timestamps written raw, bitstream starts at offset 16. + int bitstreamLen = written - 16; + QwpGorillaDecoder dec = new QwpGorillaDecoder(); + dec.reset(timestamps[0], timestamps[1], dest + 16, bitstreamLen); + Assert.assertEquals(0L, dec.getBitPosition()); + dec.decodeNext(); // third timestamp + long posAfterFirst = dec.getBitPosition(); + Assert.assertTrue("bit position must advance after a decode, was " + posAfterFirst, + posAfterFirst > 0); + dec.decodeNext(); + Assert.assertTrue("bit position must keep advancing on subsequent decodes", + dec.getBitPosition() > posAfterFirst); + } finally { + Unsafe.free(src, srcLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(dest, destCap, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testDecodePastEndOfEmptyBitstreamThrows() throws Exception { + // Reset to a zero-length bitstream and verify the very first decodeNext + // surfaces the bit reader's "read past end" exception. Asking for a + // value when there are no bytes at all is the unambiguous past-end case; + // the bit-byte-aligned trailing-zero scenario is not (the trailing 0s + // happen to be a valid 1-bit "DoD == 0" prefix). + TestUtils.assertMemoryLeak(() -> { + QwpGorillaDecoder dec = new QwpGorillaDecoder(); + dec.reset(0L, 100L, 0L, 0); + try { + dec.decodeNext(); + Assert.fail("decodeNext on empty bitstream must throw"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions read past end: " + expected.getMessage(), + expected.getMessage().contains("read past end")); + } + }); + } + + @Test + public void testDecodePastEndOfLargeBucketBitstreamThrows() throws Exception { + // Sequence whose DoDs land in the 36-bit fallback bucket so each value + // consumes a known multi-byte chunk. After decoding the encoded values, + // we keep asking for more until the read past-end check fires. Two + // successful decodes is enough to land us at end-of-bitstream. + TestUtils.assertMemoryLeak(() -> { + long[] timestamps = {1_000_000L, 2_000_000L, 3_500_000L, 7_000_000L}; + int srcLen = timestamps.length * 8; + long src = Unsafe.malloc(srcLen, MemoryTag.NATIVE_DEFAULT); + int destCap = 256; + long dest = Unsafe.malloc(destCap, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < timestamps.length; i++) { + Unsafe.getUnsafe().putLong(src + (long) i * 8, timestamps[i]); + } + QwpGorillaEncoder enc = new QwpGorillaEncoder(); + int written = enc.encodeTimestamps(dest, destCap, src, timestamps.length); + int bitstreamLen = written - 16; + QwpGorillaDecoder dec = new QwpGorillaDecoder(); + dec.reset(timestamps[0], timestamps[1], dest + 16, bitstreamLen); + for (int i = 2; i < timestamps.length; i++) { + Assert.assertEquals(timestamps[i], dec.decodeNext()); + } + // Past-end guarantee depends on the trailing-bit pattern of the + // last byte. Loop until the bit reader runs out of payload -- + // the bound caps overrun at < 64 attempts, well before a real + // decoder would ever loop forever. + boolean threw = false; + for (int i = 0; i < 64; i++) { + try { + dec.decodeNext(); + } catch (QwpDecodeException expected) { + threw = true; + Assert.assertTrue("error mentions read past end: " + expected.getMessage(), + expected.getMessage().contains("read past end")); + break; + } + } + Assert.assertTrue("decodeNext must eventually throw past end of bitstream", threw); + } finally { + Unsafe.free(src, srcLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(dest, destCap, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testResetBetweenColumnsForgetsPreviousState() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Encode-and-decode column A. Then reset to column B with completely + // different seed timestamps and verify B decodes correctly -- i.e. + // that prevDelta / prevTimestamp from A leaked into B. + long[] colA = {1L, 2L, 3L, 4L}; + long[] colB = {10_000L, 20_000L, 30_000L, 40_000L}; + + int srcLen = 8 * Math.max(colA.length, colB.length); + long src = Unsafe.malloc(srcLen, MemoryTag.NATIVE_DEFAULT); + int destCap = 64; + long destA = Unsafe.malloc(destCap, MemoryTag.NATIVE_DEFAULT); + long destB = Unsafe.malloc(destCap, MemoryTag.NATIVE_DEFAULT); + try { + QwpGorillaEncoder encA = new QwpGorillaEncoder(); + QwpGorillaEncoder encB = new QwpGorillaEncoder(); + for (int i = 0; i < colA.length; i++) { + Unsafe.getUnsafe().putLong(src + (long) i * 8, colA[i]); + } + int writtenA = encA.encodeTimestamps(destA, destCap, src, colA.length); + for (int i = 0; i < colB.length; i++) { + Unsafe.getUnsafe().putLong(src + (long) i * 8, colB[i]); + } + int writtenB = encB.encodeTimestamps(destB, destCap, src, colB.length); + + QwpGorillaDecoder dec = new QwpGorillaDecoder(); + dec.reset(colA[0], colA[1], destA + 16, writtenA - 16); + for (int i = 2; i < colA.length; i++) { + Assert.assertEquals(colA[i], dec.decodeNext()); + } + // Re-bind the decoder to column B with its own seed timestamps. + dec.reset(colB[0], colB[1], destB + 16, writtenB - 16); + for (int i = 2; i < colB.length; i++) { + Assert.assertEquals("col B index " + i, colB[i], dec.decodeNext()); + } + } finally { + Unsafe.free(src, srcLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(destA, destCap, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(destB, destCap, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + /** + * Encodes {@code timestamps} with {@link QwpGorillaEncoder}, decodes with + * {@link QwpGorillaDecoder}, and asserts every value round-trips. Skips the + * first two timestamps because the encoder ships them uncompressed at the + * head of the buffer; the decoder starts at the bitstream after them. + */ + private void roundTrip(long[] timestamps) throws Exception { + TestUtils.assertMemoryLeak(() -> { + int srcLen = timestamps.length * 8; + long src = Unsafe.malloc(srcLen, MemoryTag.NATIVE_DEFAULT); + // Bitstream worst-case: 36 bits per timestamp + 16 bytes of seeds. + int destCap = 16 + (timestamps.length * 36 + 7) / 8 + 16; + long dest = Unsafe.malloc(destCap, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < timestamps.length; i++) { + Unsafe.getUnsafe().putLong(src + (long) i * 8, timestamps[i]); + } + Assert.assertTrue("test sequence must satisfy canUseGorilla precondition", + QwpGorillaEncoder.canUseGorilla(src, timestamps.length)); + QwpGorillaEncoder enc = new QwpGorillaEncoder(); + int written = enc.encodeTimestamps(dest, destCap, src, timestamps.length); + int bitstreamLen = written - 16; + + QwpGorillaDecoder dec = new QwpGorillaDecoder(); + dec.reset(timestamps[0], timestamps[1], dest + 16, bitstreamLen); + for (int i = 2; i < timestamps.length; i++) { + long got = dec.decodeNext(); + Assert.assertEquals("timestamp index " + i, timestamps[i], got); + } + } finally { + Unsafe.free(src, srcLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(dest, destCap, MemoryTag.NATIVE_DEFAULT); + } + }); + } +} diff --git a/examples/src/main/java/com/example/query/BasicQueryExample.java b/examples/src/main/java/com/example/query/BasicQueryExample.java new file mode 100644 index 00000000..5b4fa9d9 --- /dev/null +++ b/examples/src/main/java/com/example/query/BasicQueryExample.java @@ -0,0 +1,64 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Minimal QWP egress query example. + *

+ * Connects to a QuestDB server over the /read/v1 WebSocket endpoint, + * runs a SELECT query, and prints each row as the batches arrive. + *

+ * Iterates rows via {@link QwpColumnBatch#forEachRow}, which hands a reusable + * row-pinned view to the lambda. Single-arg accessors keep the read path + * compact; the underlying batch is still column-major and the {@code (col, row)} + * primitives remain available on {@code batch} for callers that prefer them. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE trades (ts TIMESTAMP, sym SYMBOL, price DOUBLE, qty LONG)
+ *       TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class BasicQueryExample { + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + client.execute( + "SELECT ts, sym, price, qty FROM trades WHERE sym = 'AAPL' LIMIT 1000", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + // The RowView handed to the lambda is reusable and pinned to the + // current row; copy values out before the callback returns if you + // need to retain them past the surrounding onBatch call. + batch.forEachRow(row -> { + long timestamp = row.getLongValue(0); // TIMESTAMP -> microseconds since epoch + String symbol = row.getSymbol(1); // SYMBOL -> String + double price = row.getDoubleValue(2); // DOUBLE + long qty = row.getLongValue(3); // LONG + + System.out.printf( + "ts=%d sym=%s price=%.4f qty=%d%n", + timestamp, symbol, price, qty + ); + }); + } + + @Override + public void onEnd(long totalRows) { + System.out.println("query finished"); + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: status=" + status + " msg=" + message); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/BindAllTypesExample.java b/examples/src/main/java/com/example/query/BindAllTypesExample.java new file mode 100644 index 00000000..3121d814 --- /dev/null +++ b/examples/src/main/java/com/example/query/BindAllTypesExample.java @@ -0,0 +1,128 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; + +import java.util.UUID; + +/** + * Tour of every typed bind setter {@code QwpBindValues} exposes. Each + * placeholder is cast to the target SQL type in the projection so the server + * tags the returned column identically to the bind. + *

+ * Types covered: BOOLEAN, BYTE, SHORT, CHAR, INT, LONG, FLOAT, DOUBLE, DATE, + * TIMESTAMP, TIMESTAMP_NANOS, VARCHAR, UUID, LONG256, GEOHASH, DECIMAL64, + * DECIMAL128, DECIMAL256. + *

+ * Each bind goes over the wire as {@code type_code(1B) | null_flag(1B) | value} + * -- typed, not string-interpolated, so UUID / DECIMAL / TIMESTAMP_NANOS round + * trip without escaping and the server's SQL-text factory cache stays warm + * across calls that differ only in bind values. + */ +public class BindAllTypesExample { + + public static void main(String[] args) { + String sql = "SELECT " + + "$1::BOOLEAN AS c_bool, " + + "$2::BYTE AS c_byte, " + + "$3::SHORT AS c_short, " + + "$4::CHAR AS c_char, " + + "$5::INT AS c_int, " + + "$6::LONG AS c_long, " + + "$7::FLOAT AS c_float, " + + "$8::DOUBLE AS c_double, " + + "$9::DATE AS c_date, " + + "$10::TIMESTAMP AS c_ts, " + + "$11::TIMESTAMP_NS AS c_ts_ns, " + + "$12::VARCHAR AS c_varchar, " + + "$13::UUID AS c_uuid, " + + "$14::LONG256 AS c_long256, " + + "$15::GEOHASH(60b) AS c_geohash, " + + "$16::DECIMAL(18, 4) AS c_dec64, " + + "$17::DECIMAL(38, 6) AS c_dec128, " + + "$18::DECIMAL(76, 10) AS c_dec256 " + + "FROM long_sequence(1)"; + + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + client.execute( + sql, + binds -> binds + .setBoolean(0, true) + .setByte(1, (byte) 42) + .setShort(2, (short) 1234) + .setChar(3, 'Q') + .setInt(4, 2_000_000) + .setLong(5, 9_000_000_000L) + .setFloat(6, 3.14f) + .setDouble(7, 2.718281828) + .setDate(8, 1_700_000_000_000L) // ms since epoch + .setTimestampMicros(9, 1_700_000_000_000_000L) // us since epoch + .setTimestampNanos(10, 1_700_000_000_123_456_789L) // ns since epoch + .setVarchar(11, "café") + .setUuid(12, UUID.fromString("123e4567-e89b-12d3-a456-426614174000")) + .setLong256(13, 0x1111111111111111L, 0x2222222222222222L, + 0x3333333333333333L, 0x4444444444444444L) + .setGeohash(14, 60, 0x0FFF_FFFF_FFFF_FFFFL) + // Decimal unscaled values. + // 12345.6789 at scale 4 -> unscaled 123_456_789 + // 123456789.123456 at scale 6 -> unscaled 123_456_789_123_456 + // 42.0 at scale 10 -> unscaled 420_000_000_000 + .setDecimal64(15, 4, 123_456_789L) + .setDecimal128(16, 6, 123_456_789_123_456L, 0L) + .setDecimal256(17, 10, 420_000_000_000L, 0L, 0L, 0L), + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + System.out.println("row count: " + batch.getRowCount()); + System.out.println("columns: " + batch.getColumnCount()); + for (int c = 0; c < batch.getColumnCount(); c++) { + System.out.printf(" col %2d: wire type=0x%02X%n", + c, batch.getColumnWireType(c) & 0xFF); + } + System.out.println("bool : " + batch.getBoolValue(0, 0)); + System.out.println("byte : " + batch.getByteValue(1, 0)); + System.out.println("short : " + batch.getShortValue(2, 0)); + System.out.println("char : " + batch.getCharValue(3, 0)); + System.out.println("int : " + batch.getIntValue(4, 0)); + System.out.println("long : " + batch.getLongValue(5, 0)); + System.out.println("float : " + batch.getFloatValue(6, 0)); + System.out.println("double : " + batch.getDoubleValue(7, 0)); + System.out.println("date : " + batch.getLongValue(8, 0)); + System.out.println("ts_us : " + batch.getLongValue(9, 0)); + System.out.println("ts_ns : " + batch.getLongValue(10, 0)); + System.out.println("varchar : " + batch.getString(11, 0)); + System.out.printf("uuid : lo=%016x hi=%016x%n", + batch.getUuidLo(12, 0), batch.getUuidHi(12, 0)); + System.out.println("geohash wire type: 0x" + + Integer.toHexString(batch.getColumnWireType(14) & 0xFF)); + // Decimal128/Decimal256 surface via dedicated accessors: + System.out.printf("dec64 : scale=%d unscaled=%d%n", + batch.getDecimalScale(15), batch.getLongValue(15, 0)); + System.out.printf("dec128 : scale=%d lo=%d hi=%d%n", + batch.getDecimalScale(16), + batch.getDecimal128Low(16, 0), + batch.getDecimal128High(16, 0)); + System.out.println("dec256 wire type: 0x" + + Integer.toHexString(batch.getColumnWireType(17) & 0xFF)); + // Sanity -- dec256 wire type: + if (batch.getColumnWireType(17) != QwpConstants.TYPE_DECIMAL256) { + throw new AssertionError("expected DECIMAL256"); + } + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: status=" + status + " msg=" + message); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/BindDecimalExample.java b/examples/src/main/java/com/example/query/BindDecimalExample.java new file mode 100644 index 00000000..d6269f39 --- /dev/null +++ b/examples/src/main/java/com/example/query/BindDecimalExample.java @@ -0,0 +1,138 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; + +/** + * Binding DECIMAL64 / DECIMAL128 / DECIMAL256. + *

+ * QuestDB decimals are fixed-point: you ship a signed integer unscaled value + * plus a scale. For DECIMAL(precision, scale), the stored form is + * {@code unscaled / 10^scale}. DECIMAL64 holds up to 18 digits in a single + * 64-bit signed integer; DECIMAL128 uses two 64-bit limbs; DECIMAL256 uses + * four. The client's {@code setDecimal*} setters take the scale byte up + * front and validate it against the server's {@code MAX_SCALE = 76}. + *

+ * Two convenience overloads take an already-built {@code Decimal128} / + * {@code Decimal256} value and extract the scale + limbs automatically; + * useful when you already carry decimals as first-class objects. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE invoices (
+ *       id     LONG,
+ *       amount DECIMAL(18, 2),   -- fits DECIMAL64
+ *       tax    DECIMAL(38, 6),   -- requires DECIMAL128
+ *       ts     TIMESTAMP
+ *   ) TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class BindDecimalExample { + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + // --- Example 1: DECIMAL64 amount, unscaled literal. + // Looking up invoices where the total is exactly $1999.99 -- + // unscaled = 199_999 at scale 2. + System.out.println("lookup by DECIMAL64 amount = 1999.99"); + client.execute( + "SELECT id FROM invoices WHERE amount = $1", + binds -> binds.setDecimal64(0, 2, 199_999L), + printIdHandler() + ); + + // --- Example 2: DECIMAL128 tax, via explicit scale + limbs. + // Looking up invoices where tax equals 12.345678 -- + // unscaled = 12_345_678 at scale 6. Positive value fits the low + // limb alone; high limb is zero. + System.out.println("lookup by DECIMAL128 tax = 12.345678"); + client.execute( + "SELECT id FROM invoices WHERE tax = $1", + binds -> binds.setDecimal128(0, 6, 12_345_678L, 0L), + printIdHandler() + ); + + // --- Example 3: DECIMAL128 via Decimal128 convenience overload. + // If you already carry the value as a Decimal128, skip the scale + // + limb juggling -- the client reads them off the object. + Decimal128 tax = Decimal128.fromLong(12_345_678L, 6); + System.out.println("lookup by DECIMAL128 via Decimal128 convenience"); + client.execute( + "SELECT id FROM invoices WHERE tax = $1", + binds -> binds.setDecimal128(0, tax), + printIdHandler() + ); + + // --- Example 4: DECIMAL256 projection (no table needed). + // 42.0 at scale 10 -> unscaled 420_000_000_000. Positive value so + // only the lowest of the four limbs is set. + System.out.println("project DECIMAL256 42.0 at scale 10"); + client.execute( + "SELECT $1::DECIMAL(76, 10) AS v FROM long_sequence(1)", + binds -> binds.setDecimal256(0, 10, 420_000_000_000L, 0L, 0L, 0L), + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + System.out.println(" dec256 scale: " + batch.getDecimalScale(0)); + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: " + message); + } + } + ); + + // --- Example 5: DECIMAL256 via Decimal256 convenience. + Decimal256 big = new Decimal256(0L, 0L, 0L, 420_000_000_000L, 10); + System.out.println("project DECIMAL256 via Decimal256 convenience"); + client.execute( + "SELECT $1::DECIMAL(76, 10) AS v FROM long_sequence(1)", + binds -> binds.setDecimal256(0, big), + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + System.out.println(" dec256 scale: " + batch.getDecimalScale(0)); + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: " + message); + } + } + ); + } + } + + private static QwpColumnBatchHandler printIdHandler() { + return new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + batch.forEachRow(row -> System.out.println(" id=" + row.getLongValue(0))); + } + + @Override + public void onEnd(long totalRows) { + System.out.println(" total matched: " + totalRows); + } + + @Override + public void onError(byte status, String message) { + System.err.println(" query failed: " + message); + } + }; + } +} diff --git a/examples/src/main/java/com/example/query/BindErrorHandlingExample.java b/examples/src/main/java/com/example/query/BindErrorHandlingExample.java new file mode 100644 index 00000000..53e1abfc --- /dev/null +++ b/examples/src/main/java/com/example/query/BindErrorHandlingExample.java @@ -0,0 +1,113 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; + +/** + * How bind-encoding errors surface. + *

+ * The setters validate values at call time. If something's wrong -- scale + * out of range, geohash precision out of range, indexes out of order, too + * many binds, or an unsupported NULL wire-type code -- the setter throws. + * When the throw happens inside the {@code binds -> ...} lambda passed to + * {@code execute}, the client catches it and dispatches through + * {@code handler.onError} with a "bind encoding failed: ..." message. No + * query is sent and the client stays healthy for the next call. + *

+ * This example hits each failure mode intentionally. The calls that succeed + * show what valid usage looks like against the same scenarios. + */ +public class BindErrorHandlingExample { + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + // 1. Scale out of range. Max scale is 76 for every DECIMAL form. + System.out.println("bad: DECIMAL scale = 200"); + client.execute( + "SELECT $1::DECIMAL(18, 4) AS v FROM long_sequence(1)", + binds -> binds.setDecimal64(0, 200, 1L), // scale 200 > 76 + printErrorHandler() + ); + + // 2. Geohash precision out of range. Valid precisions: 1..60 bits. + System.out.println("bad: GEOHASH precision = 99"); + client.execute( + "SELECT $1::GEOHASH(60b) AS v FROM long_sequence(1)", + binds -> binds.setGeohash(0, 99, 0L), + printErrorHandler() + ); + + // 3. Out-of-order index. Indexes must be 0, 1, 2, ... dense. + System.out.println("bad: skip index 0"); + client.execute( + "SELECT $1::INT + $2::INT AS sum FROM long_sequence(1)", + binds -> binds.setInt(1, 10), // should be 0 first + printErrorHandler() + ); + + // 4. Duplicate index. Same index twice is rejected. + System.out.println("bad: duplicate index 0"); + client.execute( + "SELECT $1::INT AS v FROM long_sequence(1)", + binds -> binds.setInt(0, 1).setInt(0, 2), + printErrorHandler() + ); + + // 5. Unsupported NULL type code. The server doesn't accept + // BINARY, IPv4, or ARRAY as bind types -- the client mirrors + // that by rejecting them at the setter. + System.out.println("bad: setNull for BINARY (unsupported as bind)"); + client.execute( + "SELECT $1 AS v FROM long_sequence(1)", + binds -> binds.setNull(0, QwpConstants.TYPE_BINARY), + printErrorHandler() + ); + + // 6. For comparison: a good call. The client stays healthy + // after the errors above, so this still works. + System.out.println("good: valid DECIMAL64 bind"); + client.execute( + "SELECT $1::DECIMAL(18, 4) AS v FROM long_sequence(1)", + binds -> binds.setDecimal64(0, 4, 123_456L), + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + System.out.println(" v scale=" + batch.getDecimalScale(0)); + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println(" unexpected failure: " + message); + } + } + ); + } + } + + private static QwpColumnBatchHandler printErrorHandler() { + return new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + System.err.println(" unexpected batch -- encode was supposed to fail"); + } + + @Override + public void onEnd(long totalRows) { + System.err.println(" unexpected onEnd -- encode was supposed to fail"); + } + + @Override + public void onError(byte status, String message) { + System.out.println(" caught: " + message); + } + }; + } +} diff --git a/examples/src/main/java/com/example/query/BindNullExample.java b/examples/src/main/java/com/example/query/BindNullExample.java new file mode 100644 index 00000000..ac196941 --- /dev/null +++ b/examples/src/main/java/com/example/query/BindNullExample.java @@ -0,0 +1,118 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; + +/** + * Binding NULLs. + *

+ * Two ways to bind an explicit NULL: + *

    + *
  • {@code setNull(index, typeCode)} -- always works, takes the QWP wire + * type code from {@link QwpConstants}.
    • + *
  • {@code setVarchar(index, null)} and {@code setUuid(index, (UUID) null)} + * -- convenience shortcuts for the nullable reference types.
    • + *
+ *

+ * Note: SQL semantics mean you cannot normally match NULL with equality + * (col = NULL is itself NULL and filters nothing). Use the + * {@code IS NULL} / {@code IS NOT NULL} predicates when you want to find + * rows with NULL columns. The bind side here is only about transmitting a + * typed NULL value through the wire. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE logs (
+ *       id    LONG,
+ *       note  VARCHAR,
+ *       level INT,
+ *       ts    TIMESTAMP
+ *   ) TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class BindNullExample { + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + // --- Convenience: setVarchar(index, null). + // Use IS NULL / IS NOT NULL predicates since SQL equality with + // NULL always yields NULL. $1 still needs a typed NULL so the + // server can infer the intended type; the bind drives that. + System.out.println("rows where note IS NOT NULL and level < some-null-placeholder:"); + client.execute( + "SELECT id, note FROM logs WHERE note IS NOT NULL AND level < COALESCE($1::INT, 100) LIMIT 10", + binds -> binds.setNull(0, QwpConstants.TYPE_INT), + printRowsHandler() + ); + + // --- Convenience: setUuid(index, (UUID) null). + System.out.println("projection: NULL VARCHAR"); + client.execute( + "SELECT $1 AS v FROM long_sequence(1)", + binds -> binds.setVarchar(0, null), + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + System.out.println(" v is null? " + batch.isNull(0, 0)); + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println(" query failed: " + message); + } + } + ); + + // --- Explicit setNull for every scalar type (useful when you + // don't have a value-type overload, e.g. TIMESTAMP_NANOS). + System.out.println("projection: NULL TIMESTAMP_NANOS"); + client.execute( + "SELECT CAST($1 AS TIMESTAMP_NS) AS ts FROM long_sequence(1)", + binds -> binds.setNull(0, QwpConstants.TYPE_TIMESTAMP_NANOS), + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + System.out.println(" ts is null? " + batch.isNull(0, 0)); + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println(" query failed: " + message); + } + } + ); + } + } + + private static QwpColumnBatchHandler printRowsHandler() { + return new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + batch.forEachRow(row -> System.out.printf( + " id=%d note=%s%n", row.getLongValue(0), row.getString(1) + )); + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println(" query failed: " + message); + } + }; + } +} diff --git a/examples/src/main/java/com/example/query/BindParametersExample.java b/examples/src/main/java/com/example/query/BindParametersExample.java new file mode 100644 index 00000000..de37ab96 --- /dev/null +++ b/examples/src/main/java/com/example/query/BindParametersExample.java @@ -0,0 +1,71 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Demonstrates typed bind parameters. + *

+ * Placeholders in the SQL ({@code $1, $2, ...}) are filled by a lambda that + * receives a {@code QwpBindValues} sink. Values go over the wire as typed + * binary payloads, not interpolated strings -- no manual escaping, correct + * handling of UUID / DECIMAL / TIMESTAMP_NANOS, and the server's SQL-text- + * keyed factory cache hits on every repeated call because the SQL text is + * identical. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE trades (ts TIMESTAMP, sym SYMBOL, price DOUBLE, qty LONG)
+ *       TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class BindParametersExample { + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + String sql = "SELECT ts, sym, price, qty FROM trades " + + "WHERE sym = $1 AND price >= $2 AND ts >= $3 LIMIT 1000"; + + // Same SQL, three different parameter sets. Each call reuses the + // compiled factory on the server side because the text is identical. + String[] symbols = {"AAPL", "MSFT", "GOOG"}; + for (String symbol : symbols) { + System.out.println("fetching trades for " + symbol); + client.execute( + sql, + binds -> binds + .setVarchar(0, symbol) + .setDouble(1, 100.0) + .setTimestampMicros(2, 1_700_000_000_000_000L), + new PrintingHandler() + ); + } + } + } + + private static final class PrintingHandler implements QwpColumnBatchHandler { + @Override + public void onBatch(QwpColumnBatch batch) { + batch.forEachRow(row -> System.out.printf( + "ts=%d sym=%s price=%.4f qty=%d%n", + row.getLongValue(0), + row.getSymbol(1), + row.getDoubleValue(2), + row.getLongValue(3) + )); + } + + @Override + public void onEnd(long totalRows) { + System.out.println("batch done, total rows = " + totalRows); + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: status=" + status + " msg=" + message); + } + } +} diff --git a/examples/src/main/java/com/example/query/BindRepeatedLookupExample.java b/examples/src/main/java/com/example/query/BindRepeatedLookupExample.java new file mode 100644 index 00000000..58252954 --- /dev/null +++ b/examples/src/main/java/com/example/query/BindRepeatedLookupExample.java @@ -0,0 +1,82 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +import java.util.Arrays; +import java.util.List; + +/** + * The repeated-lookup / factory-cache-reuse pattern. + *

+ * Run the same SQL text many times with different bind values. The server + * compiles the factory on the first call and caches it keyed on the SQL + * text; every subsequent call with identical text hits the cache. Typed + * binds (not string interpolation) are what make this work -- the moment + * you splice values into the SQL text, each call becomes a new cache key. + *

+ * This pattern shows up in dashboards polling a fixed query per-entity, + * detail-page lookups by id, and any "one query shape, many parameter + * sets" workload. The cache stays warm for the lifetime of the query- + * execution plan cache on the server side. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE trades (
+ *       ts    TIMESTAMP,
+ *       sym   SYMBOL,
+ *       price DOUBLE,
+ *       qty   LONG
+ *   ) TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class BindRepeatedLookupExample { + + public static void main(String[] args) { + List instruments = Arrays.asList("AAPL", "MSFT", "GOOG", "AMZN", "META"); + + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + // SAME SQL TEXT across every iteration. Only the bind values differ. + String sql = "SELECT ts, price, qty FROM trades " + + "WHERE sym = $1 AND ts >= $2 ORDER BY ts DESC LIMIT 10"; + + long since = 1_700_000_000_000_000L; // micros since epoch + + for (String symbol : instruments) { + System.out.println("latest trades for " + symbol); + long[] rowCount = {0}; + client.execute( + sql, + binds -> binds + .setVarchar(0, symbol) + .setTimestampMicros(1, since), + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + rowCount[0] += batch.getRowCount(); + batch.forEachRow(row -> System.out.printf( + " ts=%d price=%.4f qty=%d%n", + row.getLongValue(0), + row.getDoubleValue(1), + row.getLongValue(2) + )); + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println(" query failed: " + message); + } + } + ); + System.out.println(" (" + rowCount[0] + " rows)"); + } + } + } +} diff --git a/examples/src/main/java/com/example/query/BindUuidExample.java b/examples/src/main/java/com/example/query/BindUuidExample.java new file mode 100644 index 00000000..e242a601 --- /dev/null +++ b/examples/src/main/java/com/example/query/BindUuidExample.java @@ -0,0 +1,118 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +import java.util.UUID; + +/** + * Binding UUIDs. The encoder offers two overloads: + *

+ * 1. {@code setUuid(index, lo, hi)} takes the two 64-bit limbs directly. + * The wire format is {@code lo | hi}, each little-endian. Use this when + * you carry UUIDs in long-pair form on the hot path and want zero + * allocation. + *

+ * 2. {@code setUuid(index, java.util.UUID)} takes a boxed UUID. The client + * extracts {@code getLeastSignificantBits()} as {@code lo} and + * {@code getMostSignificantBits()} as {@code hi} -- the mapping that + * QuestDB's UUID type uses internally. A {@code null} UUID is encoded + * as an explicit NULL bind. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE sessions (
+ *       session_id UUID,
+ *       user_name  VARCHAR,
+ *       started_at TIMESTAMP
+ *   ) TIMESTAMP(started_at) PARTITION BY DAY WAL;
+ *
+ */ +public class BindUuidExample { + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + UUID id = UUID.fromString("123e4567-e89b-12d3-a456-426614174000"); + + // Convenience overload: pass java.util.UUID directly. + System.out.println("lookup by java.util.UUID convenience overload"); + client.execute( + "SELECT user_name, started_at FROM sessions WHERE session_id = $1", + binds -> binds.setUuid(0, id), + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + batch.forEachRow(row -> System.out.printf( + " user=%s started_at=%d%n", + row.getString(0), row.getLongValue(1) + )); + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println(" query failed: " + message); + } + } + ); + + // Zero-alloc limb form: if you already carry the UUID as (lo, hi). + long lo = id.getLeastSignificantBits(); + long hi = id.getMostSignificantBits(); + System.out.println("lookup by (lo, hi) limb overload"); + client.execute( + "SELECT user_name, started_at FROM sessions WHERE session_id = $1", + binds -> binds.setUuid(0, lo, hi), + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + batch.forEachRow(row -> System.out.printf( + " user=%s started_at=%d%n", + row.getString(0), row.getLongValue(1) + )); + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println(" query failed: " + message); + } + } + ); + + // Projecting a UUID back via the batch's UUID accessors. + System.out.println("project a UUID back"); + client.execute( + "SELECT $1::UUID AS u FROM long_sequence(1)", + binds -> binds.setUuid(0, id), + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + long gotLo = batch.getUuidLo(0, 0); + long gotHi = batch.getUuidHi(0, 0); + UUID got = new UUID(gotHi, gotLo); + System.out.println(" got: " + got); + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println(" query failed: " + message); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/ColumnAggregationExample.java b/examples/src/main/java/com/example/query/ColumnAggregationExample.java new file mode 100644 index 00000000..3ec41866 --- /dev/null +++ b/examples/src/main/java/com/example/query/ColumnAggregationExample.java @@ -0,0 +1,70 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.ColumnView; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Single-column aggregate (min / max / sum / count) using {@code ColumnView}. + *

+ * Aggregates are textbook column-first work: the inner loop touches one + * column's values in tight succession. Pinning the column with + * {@link QwpColumnBatch#column(int)} resolves the column layout once per batch; + * the loop body is then a pure per-row read. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE trades (ts TIMESTAMP, sym SYMBOL, price DOUBLE, qty LONG)
+ *       TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class ColumnAggregationExample { + + public static void main(String[] args) { + final double[] min = {Double.POSITIVE_INFINITY}; + final double[] max = {Double.NEGATIVE_INFINITY}; + final double[] sum = {0.0}; + final long[] count = {0}; + + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + client.execute( + "SELECT price FROM trades WHERE sym = 'AAPL'", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + ColumnView prices = batch.column(0); + int rows = batch.getRowCount(); + for (int r = 0; r < rows; r++) { + if (prices.isNull(r)) continue; + double p = prices.getDoubleValue(r); + if (p < min[0]) min[0] = p; + if (p > max[0]) max[0] = p; + sum[0] += p; + count[0]++; + } + } + + @Override + public void onEnd(long totalRows) { + if (count[0] == 0) { + System.out.println("no rows"); + return; + } + System.out.printf( + "count=%d min=%.4f max=%.4f avg=%.4f%n", + count[0], min[0], max[0], sum[0] / count[0] + ); + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: status=" + status + " msg=" + message); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/ColumnScanExample.java b/examples/src/main/java/com/example/query/ColumnScanExample.java new file mode 100644 index 00000000..2865877a --- /dev/null +++ b/examples/src/main/java/com/example/query/ColumnScanExample.java @@ -0,0 +1,89 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.ColumnView; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Column-first iteration over a QWP egress result. + *

+ * The wire format is column-major, so analytical scans -- summing a price + * column, finding a min/max, building a histogram -- are most efficient when + * the loop pins one column and walks rows. {@link QwpColumnBatch#column(int)} + * returns a {@link ColumnView} flyweight that resolves the column layout once + * and exposes per-row accessors that take only the row index. + *

+ * For SIMD or JNI consumers, the same view exposes the raw native pointers + * underlying the batch: {@link ColumnView#valuesAddr()} for the packed values, + * {@link ColumnView#nullBitmapAddr()} for the null bitmap, and + * {@link ColumnView#bytesPerValue()} for the fixed-width stride. Two + * {@code ColumnView} instances over different columns can be held side-by-side + * (the batch caches one per column index), so a zip-style pass over two + * columns is a single inner loop. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE trades (ts TIMESTAMP, sym SYMBOL, price DOUBLE, qty LONG)
+ *       TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class ColumnScanExample { + + public static void main(String[] args) { + final double[] sumPrice = {0.0}; + final long[] sumQty = {0}; + final long[] rowCount = {0}; + final long[] nonNullPrice = {0}; + + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + client.execute( + "SELECT price, qty FROM trades WHERE sym = 'AAPL'", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + // Pin both columns once. The batch caches one ColumnView per + // column index, so prices and qtys remain valid simultaneously. + ColumnView prices = batch.column(0); + ColumnView qtys = batch.column(1); + + int rows = batch.getRowCount(); + rowCount[0] += rows; + nonNullPrice[0] += prices.nonNullCount(); + + // Typed column-first scan: one layout lookup, then per-row reads. + for (int r = 0; r < rows; r++) { + if (!prices.isNull(r)) { + sumPrice[0] += prices.getDoubleValue(r); + } + sumQty[0] += qtys.getLongValue(r); + } + + // The same view exposes raw addresses for SIMD/JNI consumers: + // long base = prices.valuesAddr(); + // int stride = prices.bytesPerValue(); // 8 for DOUBLE + // long bitmap = prices.nullBitmapAddr(); // 0 if no nulls + // int count = prices.nonNullCount(); + // The bytes follow the QWP wire format documented in + // QwpColumnBatch#valuesAddr(int). + } + + @Override + public void onEnd(long totalRows) { + System.out.printf( + "rows=%d nonNullPrice=%d sumPrice=%.2f sumQty=%d%n", + rowCount[0], nonNullPrice[0], sumPrice[0], sumQty[0] + ); + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: status=" + status + " msg=" + message); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/CompressionExample.java b/examples/src/main/java/com/example/query/CompressionExample.java new file mode 100644 index 00000000..f9d6e8c4 --- /dev/null +++ b/examples/src/main/java/com/example/query/CompressionExample.java @@ -0,0 +1,87 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Opting into zstd compression of {@code RESULT_BATCH} frames. + *

+ * QWP can compress the body of each result batch with zstd. Compression is + * negotiated once at WebSocket upgrade time via the + * {@code X-QWP-Accept-Encoding} header; once agreed, every batch on the + * connection ships compressed (unless the compressed form would be larger + * than the raw form, in which case the server sends that specific batch + * raw). + *

+ * Supported {@code compression=} values: + *

    + *
  • {@code auto} (default) -- advertise {@code zstd,raw}. The server + * uses zstd when it supports it and falls back to raw otherwise. + * Good default for most apps.
    • + *
  • {@code zstd} -- same advertisement as {@code auto}. Makes intent + * explicit in the connection string.
    • + *
  • {@code raw} -- skip compression entirely. No {@code X-QWP-Accept-Encoding} + * header is sent; the server ships batches uncompressed.
    • + *
+ *

+ * {@code compression_level=N} is a hint passed to the server; the server + * clamps it to {@code [1, 9]} because zstd levels 10 and above drop below + * ~20 MB/s compress speed and would pin a worker thread under a heavy load. + * The default is 3 -- roughly 400 MB/s compress, halves DOUBLE / LONG + * traffic on typical time-series data. + *

+ * When to turn it on: + *

    + *
  • Cross-region, cellular, or otherwise bandwidth-constrained clients.
    • + *
  • Highly repetitive data: rotating symbols, low-cardinality enums, + * adjacent timestamps. Expect 3-10x shrinkage.
    • + *
  • You'd rather spend server CPU than wire bandwidth.
    • + *
+ * When to leave it off ({@code compression=raw}): + *
    + *
  • Co-located client + server (localhost, same rack) -- the NIC is not + * the bottleneck; zstd eats CPU you could spend parsing results.
    • + *
  • Pre-compressed or encrypted-looking payloads -- BINARY columns + * holding images, compressed JSON, etc. zstd won't shrink them and + * you pay the CPU for nothing.
    • + *
+ */ +public final class CompressionExample { + + private CompressionExample() { + } + + public static void main(String[] args) { + // compression=zstd at a mid-tier level for a typical WAN consumer. + try (QwpQueryClient client = QwpQueryClient.fromConfig( + "ws::addr=127.0.0.1:9000;compression=zstd;compression_level=3;")) { + client.connect(); + + long[] rows = {0}; + + client.execute( + "SELECT symbol, price, timestamp " + + "FROM trades WHERE timestamp IN yesterday()", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + // The decoder hands us a view over the decompressed + // bytes; the rest of the row-consuming code is + // identical whether or not compression is on. + rows[0] += batch.getRowCount(); + } + + @Override + public void onEnd(long totalRows) { + System.out.printf("done: %d rows%n", totalRows); + } + + @Override + public void onError(byte status, String message) { + System.err.printf("query failed [status=%d]: %s%n", status, message); + } + }); + } + } +} diff --git a/examples/src/main/java/com/example/query/CreditFlowControlExample.java b/examples/src/main/java/com/example/query/CreditFlowControlExample.java new file mode 100644 index 00000000..0acd7a9a --- /dev/null +++ b/examples/src/main/java/com/example/query/CreditFlowControlExample.java @@ -0,0 +1,79 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Byte-budgeted flow control for large streaming queries. + *

+ * When a consumer is slow (throttled network, expensive per-row processing, + * transient back-pressure in a downstream sink), you can put the stream under + * explicit flow control by calling {@link QwpQueryClient#withInitialCredit} + * before {@code connect}. The server then streams at most that many bytes of + * result payload before pausing; the client's I/O thread auto-replenishes by + * the size of each batch as soon as the user's {@code onBatch} handler returns. + *

+ * Benefits: + *

    + *
  • Bounded client-side memory -- the client never holds more than + * roughly {@code initialCredit} bytes of unread result data, even if + * the handler pauses, sleeps, or does blocking I/O.
    • + *
  • Bounded server-side outstanding work -- the server parks its cursor + * when credit is exhausted instead of blasting bytes into a kernel + * buffer that a slow consumer can't drain.
    • + *
  • Back-pressure propagation that's explicit on the wire rather than + * relying solely on TCP window behaviour -- useful across proxies, + * load balancers, or WebSocket gateways that may or may not propagate + * TCP-level back-pressure cleanly.
    • + *
+ *

+ * Defaults ({@code withInitialCredit} not called, or {@code 0}): the stream + * is unbounded; no CREDIT frames are emitted. This is the Phase-1 behaviour + * and is the right choice for local / fast-network consumers that can keep + * up with whatever the server emits. + *

+ * Sizing guidance: make {@code initialCredit} at least a few times the batch + * size (server emits batches up to a few hundred KB). A value like + * {@code 256 * 1024} (256 KiB) is a reasonable starting point -- large enough + * that the server can send a couple of batches ahead, small enough to cap + * memory tightly on slow consumers. + */ +public class CreditFlowControlExample { + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000) + // 256 KiB send-ahead budget. Server pauses once it has streamed + // that many bytes of result payload; client auto-replenishes by + // the byte-size of each batch once the handler releases it. + .withInitialCredit(256 * 1024)) { + client.connect(); + + final long[] rowsSeen = {0}; + client.execute( + "SELECT ts, price FROM trades WHERE ts > dateadd('d', -30, now())", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + // Process the batch. If you block here (e.g. writing to a + // slow downstream), the server naturally parks until this + // returns and the credit-replenish frame catches up. + rowsSeen[0] += batch.getRowCount(); + } + + @Override + public void onEnd(long totalRows) { + System.out.printf("done: rows=%d%n", rowsSeen[0]); + } + + @Override + public void onError(byte status, String message) { + throw new RuntimeException( + "query failed (status=" + status + "): " + message + ); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/CsvExportExample.java b/examples/src/main/java/com/example/query/CsvExportExample.java new file mode 100644 index 00000000..16060f68 --- /dev/null +++ b/examples/src/main/java/com/example/query/CsvExportExample.java @@ -0,0 +1,82 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.std.str.StringSink; + +/** + * Streaming CSV export of a query result. + *

+ * Combines {@link QwpColumnBatch#forEachRow} for the per-row loop with the + * zero-allocation string sink ({@link QwpColumnBatch#getString(int, int, + * io.questdb.client.std.str.CharSink) batch.getString(col, row, sink)}) so the + * UTF-8 bytes for STRING / VARCHAR / SYMBOL columns transcode straight into a + * {@link StringSink} without intermediate {@link String} allocations. + *

+ * Note: this example does not handle CSV quoting -- a real exporter must + * escape commas, quotes, and newlines in string values. The point here is the + * shape of the loop, not the CSV format. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE trades (ts TIMESTAMP, sym SYMBOL, price DOUBLE, qty LONG)
+ *       TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class CsvExportExample { + + public static void main(String[] args) { + // One sink reused across every row -- clear() resets the position to 0 + // without releasing the backing array. + final StringSink line = new StringSink(); + final long[] rowsWritten = {0}; + + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + // Header row. + System.out.println("ts,sym,price,qty"); + + client.execute( + "SELECT ts, sym, price, qty FROM trades", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + batch.forEachRow(row -> { + line.clear(); + + // ts (TIMESTAMP -> microseconds since epoch). + if (!row.isNull(0)) line.put(row.getLongValue(0)); + line.put(','); + + // sym (SYMBOL) -- transcodes UTF-8 directly into the sink. + row.getString(1, line); + line.put(','); + + // price (DOUBLE). + if (!row.isNull(2)) line.put(row.getDoubleValue(2)); + line.put(','); + + // qty (LONG). + if (!row.isNull(3)) line.put(row.getLongValue(3)); + + System.out.println(line); + rowsWritten[0]++; + }); + } + + @Override + public void onEnd(long totalRows) { + System.err.println("exported " + rowsWritten[0] + " rows"); + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: status=" + status + " msg=" + message); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/ErrorHandlingExample.java b/examples/src/main/java/com/example/query/ErrorHandlingExample.java new file mode 100644 index 00000000..c3bc7707 --- /dev/null +++ b/examples/src/main/java/com/example/query/ErrorHandlingExample.java @@ -0,0 +1,74 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Surfacing server-side errors back to application code. + *

+ * When the server rejects a query (syntax error, missing table, unsupported + * statement, permission denied), it sends a {@code QUERY_ERROR} frame rather + * than data. The client delivers this via {@link QwpColumnBatchHandler#onError}, + * skipping {@code onBatch} / {@code onEnd} entirely. + *

+ * Status codes mirror the ingress namespace. For egress the common ones are: + *

    + *
  • {@code 0x03 SCHEMA_MISMATCH} — bind parameter type doesn't match the placeholder
    • + *
  • {@code 0x05 PARSE_ERROR} — SQL syntax error OR unsupported statement on the endpoint
    • + *
  • {@code 0x06 INTERNAL_ERROR} — unexpected server-side failure
    • + *
  • {@code 0x08 SECURITY_ERROR} — authorization failure
    • + *
  • {@code 0x0A CANCELLED} — query terminated in response to CANCEL
    • + *
  • {@code 0x0B LIMIT_EXCEEDED} — a protocol limit was hit
    • + *
+ * SQL-level errors carry the position embedded in the message, using QuestDB's + * standard "{@code [pos] text}" format, so you can point the user directly at + * the offending token. + *

+ * Note: DDL / INSERT / UPDATE are not errors over {@code /read/v1} -- + * the server executes them and replies with {@code EXEC_DONE}, surfaced via + * {@link QwpColumnBatchHandler#onExecDone}. See {@link ExecStatementExample}. + */ +public class ErrorHandlingExample { + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + // Malformed SQL — triggers a parse error at position 14 (just past "FROM"). + runAndReport(client, "SELECT * FROM"); + + // Nonexistent table — also reported as PARSE_ERROR with a "does not exist" message. + runAndReport(client, "SELECT * FROM nowhere"); + + // COPY ... FROM is the one non-SELECT still rejected on egress: + // bulk load belongs on the /write/v4 ingress endpoint. + runAndReport(client, "COPY trades FROM '/tmp/missing.csv'"); + } + } + + private static void runAndReport(QwpQueryClient client, final String sql) { + System.out.println("-- executing: " + sql); + client.execute(sql, new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + System.out.println("(unexpected) received " + batch.getRowCount() + " rows"); + } + + @Override + public void onEnd(long totalRows) { + System.out.println("query succeeded: rows=" + totalRows); + } + + @Override + public void onError(byte status, String message) { + System.out.printf("query failed: status=0x%02X, message=%s%n", status & 0xFF, message); + } + + @Override + public void onExecDone(short opType, long rowsAffected) { + System.out.printf("(unexpected) exec ok: opType=%d, rows=%d%n", opType, rowsAffected); + } + }); + } +} diff --git a/examples/src/main/java/com/example/query/ExecStatementExample.java b/examples/src/main/java/com/example/query/ExecStatementExample.java new file mode 100644 index 00000000..05eff92e --- /dev/null +++ b/examples/src/main/java/com/example/query/ExecStatementExample.java @@ -0,0 +1,128 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Running DDL / INSERT / UPDATE statements over QWP egress. + *

+ * The {@code /read/v1} endpoint accepts any SQL statement the compiler + * understands, not just {@code SELECT}. Non-SELECT statements skip the + * {@code onBatch} / {@code onEnd} callbacks entirely -- the server executes + * the statement and replies with a single {@code EXEC_DONE} frame that the + * client surfaces via {@link QwpColumnBatchHandler#onExecDone}. + *

+ * The callback carries two pieces of information: + *

    + *
  • {@code opType} -- one of {@code CompiledQuery.SELECT} / + * {@code INSERT} / {@code UPDATE} / {@code CREATE_TABLE} / + * {@code DROP} / etc. (see {@code io.questdb.griffin.CompiledQuery} + * for the full enum).
    • + *
  • {@code rowsAffected} -- number of rows inserted / updated / deleted. + * Pure DDL (CREATE / DROP / ALTER / RENAME / TRUNCATE / ...) reports 0.
    • + *
+ */ +public class ExecStatementExample { + + // Op-type codes, copied from io.questdb.griffin.CompiledQuery so this + // example stays compilable without the server-side module on the classpath. + // The comments are just for readability -- the server sends the numeric + // value in the EXEC_DONE frame and the client hands it to onExecDone as-is. + private static final short CREATE_TABLE = 9; + private static final short DROP = 7; + private static final short INSERT = 2; + private static final short UPDATE = 14; + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + // 1. DDL: CREATE TABLE. rowsAffected is 0 for pure DDL. + runExec(client, "CREATE TABLE trades_example (" + + "ts TIMESTAMP, sym SYMBOL, price DOUBLE, qty LONG" + + ") TIMESTAMP(ts) PARTITION BY DAY WAL", CREATE_TABLE); + + // 2. INSERT with multi-row VALUES. rowsAffected = 3. + runExec(client, + "INSERT INTO trades_example VALUES " + + "(0, 'AAPL', 150.25, 100), " + + "(1_000_000, 'AAPL', 150.30, 200), " + + "(2_000_000, 'MSFT', 420.10, 50)", + INSERT); + + // 3. UPDATE with a predicate. Server reports how many rows matched. + runExec(client, + "UPDATE trades_example SET qty = qty * 2 WHERE sym = 'AAPL'", + UPDATE); + + // 4. SELECT the data back to confirm the UPDATE landed. Uses the + // standard batch-streaming path (onBatch + onEnd). + System.out.println("-- verifying UPDATE via SELECT"); + client.execute( + "SELECT ts, sym, qty FROM trades_example", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + for (int row = 0; row < batch.getRowCount(); row++) { + System.out.printf( + " ts=%d sym=%s qty=%d%n", + batch.getLongValue(0, row), + batch.getString(1, row), + batch.getLongValue(2, row) + ); + } + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println("SELECT failed: " + message); + } + } + ); + + // 5. Clean up with DROP. Also pure DDL, rowsAffected=0. + runExec(client, "DROP TABLE trades_example", DROP); + } + } + + /** + * Executes a non-SELECT statement and prints the server's ack. Fails + * fast if the server unexpectedly streams rows or the op type doesn't + * match what we asked for. + */ + private static void runExec(QwpQueryClient client, String sql, short expectedOpType) { + System.out.println("-- executing: " + sql); + client.execute(sql, new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + System.err.println("(unexpected) batch with " + batch.getRowCount() + " rows"); + } + + @Override + public void onEnd(long totalRows) { + System.err.println("(unexpected) onEnd for a non-SELECT; totalRows=" + totalRows); + } + + @Override + public void onError(byte status, String message) { + System.err.printf(" failed: status=0x%02X, message=%s%n", status & 0xFF, message); + } + + @Override + public void onExecDone(short opType, long rowsAffected) { + System.out.printf( + " done: opType=%d (expected=%d), rowsAffected=%d%n", + opType, expectedOpType, rowsAffected + ); + if (opType != expectedOpType) { + System.err.println(" !! op type mismatch"); + } + } + }); + } +} diff --git a/examples/src/main/java/com/example/query/LargeResultStreamingExample.java b/examples/src/main/java/com/example/query/LargeResultStreamingExample.java new file mode 100644 index 00000000..7f7c16b8 --- /dev/null +++ b/examples/src/main/java/com/example/query/LargeResultStreamingExample.java @@ -0,0 +1,70 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Streaming over a large result set. + *

+ * For result sets that don't fit in memory, the column-batch consumer is the + * right entry point: each {@code onBatch} callback sees one {@code RESULT_BATCH} + * frame (up to a few thousand rows), letting you process the data incrementally + * without the whole result set ever being materialised in the client. + *

+ * The server streams continuously until the cursor is exhausted; batches arrive + * on the calling thread inside {@code client.execute(...)} as the WebSocket + * yields frames. Pacing naturally follows TCP back-pressure -- if the consumer + * is slower than the network, the kernel send buffer fills and the server + * parks between batches. For explicit byte-level flow control on top of that, + * see {@link CreditFlowControlExample}. + */ +public class LargeResultStreamingExample { + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + // Running totals we accumulate across all batches without ever holding + // the whole result set in memory. + final long[] rowsSeen = {0}; + final long[] batchCount = {0}; + final double[] priceSum = {0.0}; + + client.execute( + "SELECT ts, price FROM trades WHERE ts > dateadd('d', -7, now())", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + batchCount[0]++; + int rows = batch.getRowCount(); + for (int r = 0; r < rows; r++) { + priceSum[0] += batch.getDoubleValue(1, r); + } + rowsSeen[0] += rows; + + // Per-batch progress marker for long-running queries. + if (batchCount[0] % 10 == 0) { + System.out.println("received " + rowsSeen[0] + " rows so far"); + } + } + + @Override + public void onEnd(long totalRows) { + System.out.printf( + "done: rows=%d batches=%d priceSum=%.2f%n", + rowsSeen[0], batchCount[0], priceSum[0] + ); + } + + @Override + public void onError(byte status, String message) { + throw new RuntimeException( + "query failed (status=" + status + "): " + message + ); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/RawAddressScanExample.java b/examples/src/main/java/com/example/query/RawAddressScanExample.java new file mode 100644 index 00000000..fc90030f --- /dev/null +++ b/examples/src/main/java/com/example/query/RawAddressScanExample.java @@ -0,0 +1,96 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.ColumnView; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.std.Unsafe; + +/** + * Raw-address (SIMD-friendly) scan over a fixed-width column. + *

+ * For consumers who want to vectorise -- via the JDK Vector API, JNI to a + * native kernel, or hand-rolled {@code Unsafe} loops -- {@link ColumnView} + * exposes the underlying native pointers directly: + *

    + *
  • {@link ColumnView#valuesAddr()} -- base of the dense, non-null values.
    • + *
  • {@link ColumnView#bytesPerValue()} -- per-value stride for fixed-width + * types (8 for DOUBLE / LONG, 4 for INT / FLOAT, etc.); 0 for BOOLEAN + * (bit-packed) and -1 for variable-width types.
    • + *
  • {@link ColumnView#nonNullCount()} -- number of values packed in the + * dense array.
    • + *
  • {@link ColumnView#nullBitmapAddr()} -- the per-row null bitmap, or 0 + * when the column has no nulls in this batch (in which case dense index + * equals row index).
    • + *
+ * The bytes follow the QWP wire format documented on + * {@link QwpColumnBatch#valuesAddr(int)}. Treat this surface as an expert API: + * it ties the consumer to the wire layout, which may shift across versions. + *

+ * This example walks the dense values array directly with {@code Unsafe}, then + * cross-checks the result against the typed {@link ColumnView#getDoubleValue} + * accessor. In a real SIMD consumer you'd hand {@code valuesAddr} and + * {@code nonNullCount} to a vector kernel instead. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE trades (ts TIMESTAMP, sym SYMBOL, price DOUBLE, qty LONG)
+ *       TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class RawAddressScanExample { + + public static void main(String[] args) { + final double[] rawSum = {0.0}; + final double[] typedSum = {0.0}; + final long[] rawCount = {0}; + + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + client.execute( + "SELECT price FROM trades WHERE sym = 'AAPL'", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + ColumnView prices = batch.column(0); + + long base = prices.valuesAddr(); + int stride = prices.bytesPerValue(); // 8 for DOUBLE + int dense = prices.nonNullCount(); + + // Raw inner loop: pure pointer arithmetic, no per-row + // ObjList lookup or null check. The dense values array + // skips NULL rows entirely, so the loop trip count is + // nonNullCount, not rowCount. + for (int i = 0; i < dense; i++) { + rawSum[0] += Unsafe.getUnsafe().getDouble(base + (long) stride * i); + } + rawCount[0] += dense; + + // Cross-check: same result via the typed accessor. + int rows = batch.getRowCount(); + for (int r = 0; r < rows; r++) { + if (!prices.isNull(r)) { + typedSum[0] += prices.getDoubleValue(r); + } + } + } + + @Override + public void onEnd(long totalRows) { + System.out.printf( + "rawCount=%d rawSum=%.6f typedSum=%.6f match=%b%n", + rawCount[0], rawSum[0], typedSum[0], rawSum[0] == typedSum[0] + ); + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: status=" + status + " msg=" + message); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/RowFilterExample.java b/examples/src/main/java/com/example/query/RowFilterExample.java new file mode 100644 index 00000000..f67c186f --- /dev/null +++ b/examples/src/main/java/com/example/query/RowFilterExample.java @@ -0,0 +1,68 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Per-row filtering with {@code RowView}. + *

+ * Most of the value of a row-pinned facade shows up when the consumer is + * deciding what to do with each row based on a predicate over several columns. + * The lambda reads cleanly because the row index is implicit, and the same + * reusable view is handed back across every iteration -- zero allocations + * inside the loop. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE trades (ts TIMESTAMP, sym SYMBOL, price DOUBLE, qty LONG)
+ *       TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class RowFilterExample { + + public static void main(String[] args) { + final double threshold = 100.0; + final long[] kept = {0}; + final long[] skipped = {0}; + + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + client.execute( + "SELECT ts, sym, price, qty FROM trades", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + batch.forEachRow(row -> { + // Skip NULL prices and rows below threshold without ever + // materialising the full result set. + if (row.isNull(2) || row.getDoubleValue(2) < threshold) { + skipped[0]++; + return; + } + kept[0]++; + System.out.printf( + "ts=%d sym=%s price=%.2f qty=%d%n", + row.getLongValue(0), + row.getString(1), + row.getDoubleValue(2), + row.getLongValue(3) + ); + }); + } + + @Override + public void onEnd(long totalRows) { + System.out.printf("done: kept=%d skipped=%d%n", kept[0], skipped[0]); + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: status=" + status + " msg=" + message); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/SymbolHistogramExample.java b/examples/src/main/java/com/example/query/SymbolHistogramExample.java new file mode 100644 index 00000000..213ac12a --- /dev/null +++ b/examples/src/main/java/com/example/query/SymbolHistogramExample.java @@ -0,0 +1,97 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.ColumnView; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Per-symbol row counter that exploits the SYMBOL dictionary cache. + *

+ * SYMBOL columns ship as a per-batch (or connection-scoped) dictionary plus + * per-row dict ids. {@link ColumnView#getSymbolId(int)} returns the id without + * touching the dictionary heap, so the inner loop is a pure {@code int} read + * per row. The materialised String for each id is resolved lazily once per + * dict entry, regardless of how many rows reference it. + *

+ * Pattern: key the histogram on the dict id (an {@code int}), then resolve + * names once at the end. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE trades (ts TIMESTAMP, sym SYMBOL, price DOUBLE, qty LONG)
+ *       TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class SymbolHistogramExample { + + public static void main(String[] args) { + // The dict ids are batch-scoped, so we resolve to String per batch and + // accumulate against the resolved name across the whole query. + final Map counts = new HashMap<>(); + + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + client.execute( + "SELECT sym FROM trades", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + ColumnView syms = batch.column(0); + int rows = batch.getRowCount(); + + // Per-batch counter keyed on dict id -- only K HashMap + // operations, where K is the number of distinct symbols + // in this batch (typically << rows). + int dictSize = syms.symbolDictSize(); + long[] perDict = new long[dictSize]; + long nullCount = 0; + + for (int r = 0; r < rows; r++) { + int id = syms.getSymbolId(r); + if (id < 0) { + nullCount++; + } else { + perDict[id]++; + } + } + + // Resolve dict id -> String once per entry, then merge + // into the global histogram. + for (int id = 0; id < dictSize; id++) { + if (perDict[id] == 0) continue; + // getSymbolForId hits the per-dict String cache, so the + // same instance is reused across rows in this batch. + String name = batch.getSymbolForId(0, id); + counts.merge(name, perDict[id], Long::sum); + } + if (nullCount > 0) { + counts.merge("(null)", nullCount, Long::sum); + } + } + + @Override + public void onEnd(long totalRows) { + // Sort descending by count for readability. + counts.entrySet().stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .collect(LinkedHashMap::new, + (m, e) -> m.put(e.getKey(), e.getValue()), + Map::putAll) + .forEach((sym, n) -> System.out.printf("%-12s %d%n", sym, (int) n)); + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: status=" + status + " msg=" + message); + } + } + ); + } + } +} diff --git a/examples/src/main/java/com/example/query/TypedResultExample.java b/examples/src/main/java/com/example/query/TypedResultExample.java new file mode 100644 index 00000000..7d9474fc --- /dev/null +++ b/examples/src/main/java/com/example/query/TypedResultExample.java @@ -0,0 +1,143 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.Long256Impl; +import io.questdb.client.std.Uuid; + +/** + * Reading every supported wire type from a {@link QwpColumnBatch}. + *

+ * The batch exposes per-cell typed accessors and a {@code getColumnWireType(col)} + * helper so you can dispatch generically when the query's column set isn't known + * at compile time (e.g., a generic query runner). + *

+ * Assumes a table containing a representative set of columns, for example: + * 

+ *   CREATE TABLE demo (
+ *       b BOOLEAN, bt BYTE, sh SHORT, ch CHAR,
+ *       i INT, l LONG, f FLOAT, d DOUBLE,
+ *       dt DATE, ts TIMESTAMP,
+ *       s STRING, v VARCHAR, sy SYMBOL,
+ *       u UUID, l256 LONG256,
+ *       g GEOHASH(20b),
+ *       d64 DECIMAL(18,2)
+ *   );
+ *
+ */ +public class TypedResultExample { + + public static void main(String[] args) { + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + client.execute("SELECT * FROM demo LIMIT 5", new QwpColumnBatchHandler() { + // Sinks live at the handler level so they're reused across every + // (row, col) that needs a UUID or LONG256/DECIMAL256 decode. + final Long256Impl long256Sink = new Long256Impl(); + final Uuid uuidSink = new Uuid(); + + @Override + public void onBatch(QwpColumnBatch batch) { + int cols = batch.getColumnCount(); + int rows = batch.getRowCount(); + for (int row = 0; row < rows; row++) { + StringBuilder line = new StringBuilder(); + for (int col = 0; col < cols; col++) { + if (col > 0) line.append(" | "); + line.append(batch.getColumnName(col)).append('='); + if (batch.isNull(col, row)) { + line.append("NULL"); + } else { + appendCell(line, batch, col, row, uuidSink, long256Sink); + } + } + System.out.println(line); + } + } + + @Override + public void onEnd(long totalRows) { + } + + @Override + public void onError(byte status, String message) { + System.err.println("query error: " + message); + } + }); + } + } + + /** + * Appends a typed value to the builder using the column's wire type to pick + * the right accessor. The set of wire type codes is in {@link QwpConstants}. + * {@code uuidSink} and {@code long256Sink} are reusable sinks owned by the + * caller; passing them in keeps UUID / LONG256 / DECIMAL256 decoding + * allocation-free across all cells in the query. + */ + private static void appendCell(StringBuilder out, QwpColumnBatch batch, int col, int row, + Uuid uuidSink, Long256Impl long256Sink) { + byte type = batch.getColumnWireType(col); + switch (type) { + case QwpConstants.TYPE_BOOLEAN: + out.append(batch.getBoolValue(col, row)); + break; + case QwpConstants.TYPE_BYTE: + out.append(batch.getByteValue(col, row)); + break; + case QwpConstants.TYPE_SHORT: + out.append(batch.getShortValue(col, row)); + break; + case QwpConstants.TYPE_CHAR: + out.append(batch.getCharValue(col, row)); + break; + case QwpConstants.TYPE_INT: + case QwpConstants.TYPE_IPv4: + out.append(batch.getIntValue(col, row)); + break; + case QwpConstants.TYPE_LONG: + case QwpConstants.TYPE_DATE: + case QwpConstants.TYPE_TIMESTAMP: + case QwpConstants.TYPE_TIMESTAMP_NANOS: + case QwpConstants.TYPE_DECIMAL64: + out.append(batch.getLongValue(col, row)); + break; + case QwpConstants.TYPE_FLOAT: + out.append(batch.getFloatValue(col, row)); + break; + case QwpConstants.TYPE_DOUBLE: + out.append(batch.getDoubleValue(col, row)); + break; + case QwpConstants.TYPE_SYMBOL: + case QwpConstants.TYPE_VARCHAR: + out.append(batch.getString(col, row)); + break; + case QwpConstants.TYPE_UUID: + batch.getUuid(col, row, uuidSink); + out.append(String.format("%016x-%016x", uuidSink.getHi(), uuidSink.getLo())); + break; + case QwpConstants.TYPE_LONG256: + case QwpConstants.TYPE_DECIMAL256: + batch.getLong256(col, row, long256Sink); + out.append(String.format("0x%016x%016x%016x%016x", + long256Sink.getLong3(), + long256Sink.getLong2(), + long256Sink.getLong1(), + long256Sink.getLong0())); + break; + case QwpConstants.TYPE_GEOHASH: + out.append("geohash(").append(batch.getGeohashPrecisionBits(col)) + .append("b)=0x").append(Long.toHexString(batch.getGeohashValue(col, row))); + break; + case QwpConstants.TYPE_DECIMAL128: + out.append("decimal(") + .append(batch.getDecimal128Low(col, row)).append(',') + .append(batch.getDecimal128High(col, row)).append(')'); + break; + default: + out.append("(type 0x").append(Integer.toHexString(type & 0xFF)).append(")"); + break; + } + } +} diff --git a/examples/src/main/java/com/example/query/ZipColumnsExample.java b/examples/src/main/java/com/example/query/ZipColumnsExample.java new file mode 100644 index 00000000..8a3251d5 --- /dev/null +++ b/examples/src/main/java/com/example/query/ZipColumnsExample.java @@ -0,0 +1,62 @@ +package com.example.query; + +import io.questdb.client.cutlass.qwp.client.ColumnView; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatch; +import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; + +/** + * Zip-style scan over two pinned columns. + *

+ * Multi-column work that doesn't need every column -- compute notional + * (price * qty), correlate two series, build a derived column -- is well + * served by holding two {@link ColumnView} instances side-by-side. + * {@link QwpColumnBatch#column(int)} caches one view per column index, so a + * second call with a different column does not invalidate the first; both + * views stay valid for the lifetime of the surrounding {@code onBatch}. + *

+ * Assumes a table exists: + * 

+ *   CREATE TABLE trades (ts TIMESTAMP, sym SYMBOL, price DOUBLE, qty LONG)
+ *       TIMESTAMP(ts) PARTITION BY DAY WAL;
+ *
+ */ +public class ZipColumnsExample { + + public static void main(String[] args) { + final double[] notional = {0.0}; + final long[] rows = {0}; + + try (QwpQueryClient client = QwpQueryClient.newPlainText("localhost", 9000)) { + client.connect(); + + client.execute( + "SELECT price, qty FROM trades WHERE sym = 'AAPL'", + new QwpColumnBatchHandler() { + @Override + public void onBatch(QwpColumnBatch batch) { + ColumnView prices = batch.column(0); + ColumnView qtys = batch.column(1); + int n = batch.getRowCount(); + rows[0] += n; + for (int r = 0; r < n; r++) { + // NULL handling: if either side is NULL, skip the contribution. + if (prices.isNull(r) || qtys.isNull(r)) continue; + notional[0] += prices.getDoubleValue(r) * qtys.getLongValue(r); + } + } + + @Override + public void onEnd(long totalRows) { + System.out.printf("rows=%d notional=%.2f%n", rows[0], notional[0]); + } + + @Override + public void onError(byte status, String message) { + System.err.println("query failed: status=" + status + " msg=" + message); + } + } + ); + } + } +}